Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 152 additions & 8 deletions .github/workflows/master.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2587,11 +2587,11 @@ jobs:
python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
fi

stateless_tests_arm_asan_azure_parallel:
stateless_tests_arm_asan_azure_parallel_1_4:
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_asan]
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbiwgYXp1cmUsIHBhcmFsbGVsKQ==') }}
name: "Stateless tests (arm_asan, azure, parallel)"
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbiwgYXp1cmUsIHBhcmFsbGVsLCAxLzQp') }}
name: "Stateless tests (arm_asan, azure, parallel, 1/4)"
outputs:
data: ${{ steps.run.outputs.DATA }}
pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
Expand All @@ -2606,7 +2606,7 @@ jobs:
- name: Docker setup
uses: ./.github/actions/docker_setup
with:
test_name: "Stateless tests (arm_asan, azure, parallel)"
test_name: "Stateless tests (arm_asan, azure, parallel, 1/4)"

- name: Prepare env script
run: |
Expand All @@ -2629,9 +2629,150 @@ jobs:
. ./ci/tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 1/4)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
else
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 1/4)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
fi

stateless_tests_arm_asan_azure_parallel_2_4:
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_asan]
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbiwgYXp1cmUsIHBhcmFsbGVsLCAyLzQp') }}
name: "Stateless tests (arm_asan, azure, parallel, 2/4)"
outputs:
data: ${{ steps.run.outputs.DATA }}
pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ env.CHECKOUT_REF }}

- name: Setup
uses: ./.github/actions/runner_setup
- name: Docker setup
uses: ./.github/actions/docker_setup
with:
test_name: "Stateless tests (arm_asan, azure, parallel, 2/4)"

- name: Prepare env script
run: |
rm -rf ./ci/tmp
mkdir -p ./ci/tmp
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.:

cat > ./ci/tmp/workflow_job.json << 'EOF'
${{ toJson(job) }}
EOF
cat > ./ci/tmp/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF

- name: Run
id: run
run: |
. ./ci/tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 2/4)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
else
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 2/4)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
fi

stateless_tests_arm_asan_azure_parallel_3_4:
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_asan]
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbiwgYXp1cmUsIHBhcmFsbGVsLCAzLzQp') }}
name: "Stateless tests (arm_asan, azure, parallel, 3/4)"
outputs:
data: ${{ steps.run.outputs.DATA }}
pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ env.CHECKOUT_REF }}

- name: Setup
uses: ./.github/actions/runner_setup
- name: Docker setup
uses: ./.github/actions/docker_setup
with:
test_name: "Stateless tests (arm_asan, azure, parallel, 3/4)"

- name: Prepare env script
run: |
rm -rf ./ci/tmp
mkdir -p ./ci/tmp
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.:

cat > ./ci/tmp/workflow_job.json << 'EOF'
${{ toJson(job) }}
EOF
cat > ./ci/tmp/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF

- name: Run
id: run
run: |
. ./ci/tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 3/4)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
else
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 3/4)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
fi

stateless_tests_arm_asan_azure_parallel_4_4:
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_asan]
if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbiwgYXp1cmUsIHBhcmFsbGVsLCA0LzQp') }}
name: "Stateless tests (arm_asan, azure, parallel, 4/4)"
outputs:
data: ${{ steps.run.outputs.DATA }}
pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ env.CHECKOUT_REF }}

- name: Setup
uses: ./.github/actions/runner_setup
- name: Docker setup
uses: ./.github/actions/docker_setup
with:
test_name: "Stateless tests (arm_asan, azure, parallel, 4/4)"

- name: Prepare env script
run: |
rm -rf ./ci/tmp
mkdir -p ./ci/tmp
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.:

cat > ./ci/tmp/workflow_job.json << 'EOF'
${{ toJson(job) }}
EOF
cat > ./ci/tmp/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF

- name: Run
id: run
run: |
. ./ci/tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 4/4)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
else
python3 -m praktika run 'Stateless tests (arm_asan, azure, parallel, 4/4)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
fi

stateless_tests_arm_asan_azure_sequential:
Expand Down Expand Up @@ -5033,7 +5174,7 @@ jobs:

finish_workflow:
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_binary, build_amd_release, build_arm_release, build_amd_coverage, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_amd_release, install_packages_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stateless_tests_arm_asan_azure_parallel, stateless_tests_arm_asan_azure_sequential, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, clickbench_amd_release, clickbench_arm_release, sqltest, stateless_tests_amd_coverage_1_8, stateless_tests_amd_coverage_2_8, stateless_tests_amd_coverage_3_8, stateless_tests_amd_coverage_4_8, stateless_tests_amd_coverage_5_8, stateless_tests_amd_coverage_6_8, stateless_tests_amd_coverage_7_8, stateless_tests_amd_coverage_8_8]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_amd_debug, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_asan, build_arm_binary, build_amd_release, build_arm_release, build_amd_coverage, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_amd_release, install_packages_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel, stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential, stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel, stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential, stateless_tests_amd_debug_asyncinsert_s3_storage_parallel, stateless_tests_amd_debug_asyncinsert_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_msan_parallel_1_2, stateless_tests_amd_msan_parallel_2_2, stateless_tests_amd_msan_sequential_1_2, stateless_tests_amd_msan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_amd_ubsan, stress_test_amd_msan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, clickbench_amd_release, clickbench_arm_release, sqltest, stateless_tests_amd_coverage_1_8, stateless_tests_amd_coverage_2_8, stateless_tests_amd_coverage_3_8, stateless_tests_amd_coverage_4_8, stateless_tests_amd_coverage_5_8, stateless_tests_amd_coverage_6_8, stateless_tests_amd_coverage_7_8, stateless_tests_amd_coverage_8_8]
if: ${{ always() }}
name: "Finish Workflow"
outputs:
Expand Down Expand Up @@ -5205,7 +5346,10 @@ jobs:
- stateless_tests_amd_tsan_s3_storage_sequential_2_2
- stateless_tests_arm_binary_parallel
- stateless_tests_arm_binary_sequential
- stateless_tests_arm_asan_azure_parallel
- stateless_tests_arm_asan_azure_parallel_1_4
- stateless_tests_arm_asan_azure_parallel_2_4
- stateless_tests_arm_asan_azure_parallel_3_4
- stateless_tests_arm_asan_azure_parallel_4_4
- stateless_tests_arm_asan_azure_sequential
- integration_tests_amd_asan_db_disk_old_analyzer_1_6
- integration_tests_amd_asan_db_disk_old_analyzer_2_6
Expand Down
16 changes: 10 additions & 6 deletions ci/defs/job_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -620,12 +620,16 @@ class JobConfigs:
)
functional_tests_jobs_azure_master_only = (
common_ft_job_config.set_allow_merge_on_failure(True).parametrize(
Job.ParamSet(
parameter="arm_asan, azure, parallel",
runs_on=RunnerLabels.FUNC_TESTER_ARM,
requires=[ArtifactNames.CH_ARM_ASAN],
timeout=3600 * 4,
),
*[
Job.ParamSet(
parameter=f"arm_asan, azure, parallel, {batch}/{total_batches}",
runs_on=RunnerLabels.FUNC_TESTER_ARM,
requires=[ArtifactNames.CH_ARM_ASAN],
timeout=3600 * 4,
)
for total_batches in (4,)
for batch in range(1, total_batches + 1)
],
Job.ParamSet(
parameter="arm_asan, azure, sequential",
runs_on=RunnerLabels.FUNC_TESTER_ARM,
Expand Down
3 changes: 3 additions & 0 deletions tests/clickhouse-test
Original file line number Diff line number Diff line change
Expand Up @@ -1731,6 +1731,9 @@ class TestCase:
self.base_client_options + client_options + " "
)

current_value = os.environ.get("CLICKHOUSE_CLIENT_OPT", "")
os.environ["CLICKHOUSE_CLIENT_OPT"] = f"{current_value} --allow_repeated_settings".strip()

return client_options + new_options

def remove_settings_from_env(self):
Expand Down
2 changes: 1 addition & 1 deletion tests/config/config.d/azure_storage_conf.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<disk>azure</disk>
<path>azure_cache/</path>
<cache_policy>LRU</cache_policy>
<max_size>100000000</max_size>
<max_size>8589934592</max_size>
<cache_on_write_operations>1</cache_on_write_operations>
</cached_azure>
<cached_azure_encrypted>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Tags: no-fasttest
# Tags: no-fasttest, no-replicated-database
# Tag no-fasttest: Depends on S3
# Tag no-replicated-database: the data can be inserted on a different node

# Regression test: S3Queue background threads read Parquet files without a
# query context on CurrentThread. Before the fix, the format factory lambda
Expand Down
Loading