Add SQLTest

.github/workflows/pull_request.yml

@@ -5182,3 +5182,39 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+##################################### SQL TEST ###############################################
+##############################################################################################
+  SQLTest:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqltest
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME="SQLTest"
+          REPO_COPY=${{runner.temp}}/sqltest/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: SQLTest
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqltest.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"

docker/images.json

@@ -125,6 +125,7 @@
             "docker/test/keeper-jepsen",
             "docker/test/server-jepsen",
             "docker/test/sqllogic",
+            "docker/test/sqltest",
             "docker/test/stateless"
          ]
     },
@@ -155,11 +156,14 @@
     },
     "docker/docs/builder": {
         "name": "clickhouse/docs-builder",
-        "dependent": [
-        ]
+        "dependent": []
     },
     "docker/test/sqllogic": {
         "name": "clickhouse/sqllogic-test",
         "dependent": []
+    },
+    "docker/test/sqltest": {
+        "name": "clickhouse/sqltest",
+        "dependent": []
     }
 }

docker/test/sqllogic/run.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 set -exu
 trap "exit" INT TERM
 

docker/test/sqltest/Dockerfile

@@ -0,0 +1,30 @@
+# docker build -t clickhouse/sqltest .
+ARG FROM_TAG=latest
+FROM clickhouse/test-base:$FROM_TAG
+
+RUN apt-get update --yes \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            wget \
+            git \
+            python3 \
+            python3-dev \
+            python3-pip \
+            sudo \
+    && apt-get clean
+
+RUN pip3 install \
+    pyyaml \
+    clickhouse-driver
+
+ARG sqltest_repo="https://github.com/elliotchance/sqltest/"
+
+RUN git clone ${sqltest_repo}
+
+ENV TZ=UTC
+ENV MAX_RUN_TIME=900
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY run.sh /
+COPY test.py /
+CMD ["/bin/bash", "/run.sh"]

docker/test/sqltest/run.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# shellcheck disable=SC2015
+
+set -x
+set -e
+set -u
+set -o pipefail
+
+BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
+BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
+
+function wget_with_retry
+{
+    for _ in 1 2 3 4; do
+        if wget -nv -nd -c "$1";then
+            return 0
+        else
+            sleep 0.5
+        fi
+    done
+    return 1
+}
+
+wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
+chmod +x clickhouse
+./clickhouse install --noninteractive
+
+echo "
+users:
+  default:
+    access_management: 1" > /etc/clickhouse-server/users.d/access_management.yaml
+
+clickhouse start
+
+# Wait for start
+for _ in {1..100}
+do
+    clickhouse-client --query "SELECT 1" && break ||:
+    sleep 1
+done
+
+# Run the test
+pushd sqltest/standards/2016/
+/test.py
+mv report.html test.log /
+popd
+
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.err.log

docker/test/sqltest/test.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+import html
+import random
+import string
+from clickhouse_driver import Client
+
+
+client = Client(host="localhost", port=9000)
+settings = {
+    "default_table_engine": "Memory",
+    "union_default_mode": "DISTINCT",
+    "calculate_text_stack_trace": 0,
+}
+
+database_name = "sqltest_" + "".join(
+    random.choice(string.ascii_lowercase) for _ in range(10)
+)
+
+client.execute(f"DROP DATABASE IF EXISTS {database_name}", settings=settings)
+client.execute(f"CREATE DATABASE {database_name}", settings=settings)
+
+client = Client(host="localhost", port=9000, database=database_name)
+
+summary = {"success": 0, "total": 0, "results": {}}
+
+log_file = open("test.log", "w")
+report_html_file = open("report.html", "w")
+
+with open("features.yml", "r") as file:
+    yaml_content = yaml.safe_load(file)
+
+    for category in yaml_content:
+        log_file.write(category.capitalize() + " features:\n")
+        summary["results"][category] = {"success": 0, "total": 0, "results": {}}
+
+        for test in yaml_content[category]:
+            log_file.write(test + ": " + yaml_content[category][test] + "\n")
+            summary["results"][category]["results"][test] = {
+                "success": 0,
+                "total": 0,
+                "description": yaml_content[category][test],
+            }
+
+            test_path = test[0] + "/" + test + ".tests.yml"
+            if os.path.exists(test_path):
+                with open(test_path, "r") as test_file:
+                    test_yaml_content = yaml.load_all(test_file, Loader=yaml.FullLoader)
+
+                    for test_case in test_yaml_content:
+                        queries = test_case["sql"]
+                        if not isinstance(queries, list):
+                            queries = [queries]
+
+                        for query in queries:
+                            # Example: E011-01
+                            test_group = ""
+                            if "-" in test:
+                                test_group = test.split("-", 1)[0]
+                                summary["results"][category]["results"][test_group][
+                                    "total"
+                                ] += 1
+                            summary["results"][category]["results"][test]["total"] += 1
+                            summary["results"][category]["total"] += 1
+                            summary["total"] += 1
+
+                            log_file.write(query + "\n")
+
+                            try:
+                                result = client.execute(query, settings=settings)
+                                log_file.write(str(result) + "\n")
+
+                                if test_group:
+                                    summary["results"][category]["results"][test_group][
+                                        "success"
+                                    ] += 1
+                                summary["results"][category]["results"][test][
+                                    "success"
+                                ] += 1
+                                summary["results"][category]["success"] += 1
+                                summary["success"] += 1
+
+                            except Exception as e:
+                                log_file.write(f"Error occurred: {str(e)}\n")
+
+client.execute(f"DROP DATABASE {database_name}", settings=settings)
+
+
+def enable_color(ratio):
+    if ratio == 0:
+        return "<b style='color: red;'>"
+    elif ratio < 0.5:
+        return "<b style='color: orange;'>"
+    elif ratio < 1:
+        return "<b style='color: gray;'>"
+    else:
+        return "<b style='color: green;'>"
+
+
+reset_color = "</b>"
+
+
+def print_ratio(indent, name, success, total, description):
+    report_html_file.write(
+        "{}{}: {}{} / {} ({:.1%}){}{}\n".format(
+            " " * indent,
+            name.capitalize(),
+            enable_color(success / total),
+            success,
+            total,
+            success / total,
+            reset_color,
+            f" - " + html.escape(description) if description else "",
+        )
+    )
+
+
+report_html_file.write(
+    "<html><body><pre style='font-size: 16pt; padding: 1em; line-height: 1.25;'>\n"
+)
+
+print_ratio(0, "Total", summary["success"], summary["total"], "")
+
+for category in summary["results"]:
+    cat_summary = summary["results"][category]
+
+    if cat_summary["total"] == 0:
+        continue
+
+    print_ratio(2, category, cat_summary["success"], cat_summary["total"], "")
+
+    for test in summary["results"][category]["results"]:
+        test_summary = summary["results"][category]["results"][test]
+
+        if test_summary["total"] == 0:
+            continue
+
+        print_ratio(
+            6 if "-" in test else 4,
+            test,
+            test_summary["success"],
+            test_summary["total"],
+            test_summary["description"],
+        )
+
+report_html_file.write("</pre></body></html>\n")

tests/ci/ast_fuzzer_check.py

@@ -79,7 +79,7 @@ def main():
             build_url = url
             break
     else:
-        raise Exception("Cannot binary clickhouse among build results")
+        raise Exception("Cannot find the clickhouse binary among build results")
 
     logging.info("Got build url %s", build_url)