24.8 TMP fix: Disabled getting AZURE_CONNECTION_STRING from SSM

cmake/autogenerated_versions.txt

@@ -14,5 +14,4 @@ SET(VERSION_FLAVOUR altinitystable)
 SET(VERSION_DESCRIBE v24.8.8.18.altinitystable)
 SET(VERSION_STRING 24.8.8.18.altinitystable)
 
->>>>>>> altinity/customizations/24.8.7
 # end of autochange

docker/test/stateful/run.sh

@@ -22,8 +22,7 @@ source /utils.lib
 # install test configs
 /usr/share/clickhouse-test/config/install.sh
 
-azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence &
-
+azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 ./setup_minio.sh stateful
 ./mc admin trace clickminio > /test_output/minio.log &
 MC_ADMIN_PID=$!

docker/test/stateless/Dockerfile

@@ -87,7 +87,7 @@ ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py"
 
-RUN npm install -g azurite@3.30.0 \
+RUN npm install -g azurite@^3.33.0 \
     && npm install -g tslib && npm install -g node
 
 COPY run.sh /

docker/test/stateless/run.sh

@@ -50,6 +50,12 @@ source /utils.lib
 # install test configs
 /usr/share/clickhouse-test/config/install.sh
 
+if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    echo "Azure is disabled"
+else
+    azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
+fi
+
 ./setup_minio.sh stateless
 
 ./setup_hdfs_minicluster.sh

docker/test/stress/run.sh

@@ -55,6 +55,7 @@ export ZOOKEEPER_FAULT_INJECTION=1
 # available for dump via clickhouse-local
 configure
 
+azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 ./setup_minio.sh stateless # to have a proper environment
 
 config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml

tests/ci/ci.py

@@ -388,7 +388,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info):
             _get_ext_check_name(job_name),
         )
         ClickHouseHelper().insert_events_into(
-            db="default", table="checks", events=prepared_events
+            db="gh-data", table="checks", events=prepared_events
         )
     print(f"Pre action done. Report files [{reports_files}] have been downloaded")
 
@@ -966,7 +966,7 @@ def _add_build_to_version_history(
 
     print(f"::notice ::Log Adding record to versions history: {data}")
 
-    ch_helper.insert_event_into(db="default", table="version_history", event=data)
+    ch_helper.insert_event_into(db="gh-data", table="version_history", event=data)
 
 
 def _run_test(job_name: str, run_command: str) -> int:
@@ -1376,7 +1376,7 @@ def main() -> int:
                 _get_ext_check_name(args.job_name),
             )
             ClickHouseHelper().insert_events_into(
-                db="default", table="checks", events=prepared_events
+                db="gh-data", table="checks", events=prepared_events
             )
     ### POST action: end
 

tests/ci/functional_test_check.py

@@ -16,7 +16,6 @@
 from docker_images_helper import DockerImage, get_docker_image, pull_image
 from download_release_packages import download_last_release
 from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
-from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
 from report import (
     ERROR,
@@ -52,8 +51,9 @@ def get_additional_envs(
     check_name: str, run_by_hash_num: int, run_by_hash_total: int
 ) -> List[str]:
     result = []
-    azure_connection_string = get_parameter_from_ssm("azure_connection_string")
-    result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
+    # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one
+    # azure_connection_string = get_parameter_from_ssm("azure_connection_string")
+    # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
     if "DatabaseReplicated" in check_name:
         result.append("USE_DATABASE_REPLICATED=1")
     if "DatabaseOrdinary" in check_name:
@@ -67,9 +67,6 @@ def get_additional_envs(
         result.append("RANDOMIZE_OBJECT_KEY_TYPE=1")
     if "analyzer" in check_name:
         result.append("USE_OLD_ANALYZER=1")
-    if "azure" in check_name:
-        assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in result
-        result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1")
 
     if run_by_hash_total != 0:
         result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}")

tests/ci/stress_check.py

@@ -13,7 +13,6 @@
 from clickhouse_helper import CiLogsCredentials
 from docker_images_helper import DockerImage, get_docker_image, pull_image
 from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
-from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
 from report import ERROR, JobReport, TestResults, read_test_results
 from stopwatch import Stopwatch
@@ -34,14 +33,13 @@ def format(self, record):
 
 def get_additional_envs(check_name: str) -> List[str]:
     result = []
-    azure_connection_string = get_parameter_from_ssm("azure_connection_string")
-    result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
+    # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one
+    # azure_connection_string = get_parameter_from_ssm("azure_connection_string")
+    # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
     # some cloud-specific features require feature flags enabled
     # so we need this ENV to be able to disable the randomization
     # of feature flags
     result.append("RANDOMIZE_KEEPER_FEATURE_FLAGS=1")
-    if "azure" in check_name:
-        result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1")
 
     if "s3" in check_name:
         result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1")
@@ -167,7 +165,7 @@ def run_stress_test(docker_image_name: str) -> None:
         pr_info, stopwatch.start_time_str, check_name
     )
 
-    additional_envs = get_additional_envs(check_name)
+    additional_envs = get_additional_envs()
 
     run_command = get_run_command(
         packages_path,

tests/config/config.d/azure_storage_conf.xml

@@ -4,10 +4,13 @@
             <azure>
                 <type>object_storage</type>
                 <object_storage_type>azure</object_storage_type>
+                <storage_account_url>http://localhost:10000/devstoreaccount1</storage_account_url>
+                <container_name>cont</container_name>
                 <skip_access_check>false</skip_access_check>
+                <!-- default credentials for Azurite storage account -->
+                <account_name>devstoreaccount1</account_name>
+                <account_key>Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==</account_key>
                 <max_single_part_upload_size>33554432</max_single_part_upload_size>
-                <container_name>openbucketforpublicci</container_name>
-                <connection_string from_env="AZURE_CONNECTION_STRING"/>
             </azure>
             <cached_azure>
                 <type>cache</type>