From 6f32a0fc77c0a76732a5b610a761c5ec59293426 Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Sun, 17 Aug 2025 18:58:59 +0000 Subject: [PATCH 1/6] support embedded ducdb with fixed version --- build.py | 2 +- config.json | 2 +- duckdb/__init__.py | 0 duckdb/config.json | 11 +++++++++++ sqlancer/Dockerfile | 1 - sqlancer/entrypoint.sh | 16 +++++++++++++--- 6 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 duckdb/__init__.py create mode 100644 duckdb/config.json diff --git a/build.py b/build.py index 3cd0d8e..e6244a7 100644 --- a/build.py +++ b/build.py @@ -61,7 +61,7 @@ def build_db_image(cfg, use_cache, script_log, docker_log, custom=False, dockerf def build_environment(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""): script_log.info("==============================Building environment==============================") build_network(script_log, docker_log) - build_sqlancer_image(script_log, docker_log, force_rebuild=not use_cache) + build_sqlancer_image(script_log, docker_log, force_rebuild=False) if cfg["embedded"] == "no": build_db_image(cfg, use_cache, script_log, docker_log, custom, dockerfile_path) script_log.info("==============================Building environment==============================") diff --git a/config.json b/config.json index dc2b8f7..0158087 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "dbms_list": ["mysql", "postgres", "sqlite", "tidb", "cockroachdb"], + "dbms_list": ["mysql", "postgres", "sqlite", "tidb", "cockroachdb", "duckdb"], "dbms": "mysql", "container_name": "mysql-custom", "image": "mysql-custom", diff --git a/duckdb/__init__.py b/duckdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/duckdb/config.json b/duckdb/config.json new file mode 100644 index 0000000..148100b --- /dev/null +++ b/duckdb/config.json @@ -0,0 +1,11 @@ +{ + "embedded": "yes", + "dbms": "duckdb", + "username": "N/A", + "password": "N/A", + "container_name": "localhost", + "oracle": "NoREC", + "num_threads": 4, + "timeout_seconds": 60 + } + \ No newline at end of file diff --git a/sqlancer/Dockerfile b/sqlancer/Dockerfile index 7891414..2c08003 100644 --- a/sqlancer/Dockerfile +++ b/sqlancer/Dockerfile @@ -16,4 +16,3 @@ RUN chmod +x /root/entrypoint.sh CMD ["/root/entrypoint.sh"] - diff --git a/sqlancer/entrypoint.sh b/sqlancer/entrypoint.sh index d156301..185b6d2 100644 --- a/sqlancer/entrypoint.sh +++ b/sqlancer/entrypoint.sh @@ -18,12 +18,22 @@ for i in {1..60}; do sleep 1 done -CMD="java -jar sqlancer-*.jar --num-threads \"$SQLANCER_THREADS\" --timeout-seconds \"$SQLANCER_TIMEOUT\" --username \"$SQLANCER_USERNAME\" --password \"$SQLANCER_PASSWORD\" --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\"" -# Show command to console + log file +CMD="java -jar sqlancer-*.jar --num-threads \"$SQLANCER_THREADS\" --timeout-seconds \"$SQLANCER_TIMEOUT\" --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\"" + + +if [ "$(printf '%s' "${SQLANCER_USERNAME:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then + CMD="$CMD --username \"$SQLANCER_USERNAME\"" +fi + + +if [ "$(printf '%s' "${SQLANCER_PASSWORD:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then + CMD="$CMD --password \"$SQLANCER_PASSWORD\"" +fi + + echo "Running: $CMD" | tee -a "$LOG_FILE" -# Run command: log everything, show only stats to console eval "$CMD" 2>&1 | tee -a "$LOG_FILE" | grep --line-buffered "Executed" echo "[INFO] SQLancer finished. Logs saved to $LOG_DIR" From 7ee4c783ee7cd8450c88b9cc4cfaee2bd0789864 Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Sun, 17 Aug 2025 19:46:47 +0000 Subject: [PATCH 2/6] aupport test different version of duckdb by modifying pom.xml --- build.py | 40 ++++++++++++++++++++++++++++++++++------ duckdb/config.json | 1 + sqlancer/Dockerfile | 12 +++++++----- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/build.py b/build.py index e6244a7..0f298c5 100644 --- a/build.py +++ b/build.py @@ -4,10 +4,24 @@ import logging from utils import run_command -def build_sqlancer_image(script_log, docker_log, force_rebuild=False): +def build_sqlancer_image(script_log, docker_log, force_rebuild=False, version=None): if force_rebuild: - script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") - run_command(["docker", "build", "--no-cache", "-t", "sqlancer:latest", "./sqlancer"], docker_log) + if version: + script_log.info( + "Rebuilding SQLancer image: sqlancer:latest (DUCKDB_JDBC_VERSION=%s) ...", + version + ) + run_command([ + "docker", "build", "--no-cache", + "--build-arg", f"DUCKDB_JDBC_VERSION={version}", + "-t", "sqlancer:latest", "./sqlancer" + ], docker_log) + else: + script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") + run_command([ + "docker", "build", "--no-cache", + "-t", "sqlancer:latest", "./sqlancer" + ], docker_log) script_log.info("SQLancer image built: sqlancer:latest") return @@ -19,8 +33,22 @@ def build_sqlancer_image(script_log, docker_log, force_rebuild=False): if "sqlancer:latest" in images: script_log.info("SQLancer image exists: sqlancer:latest") else: - script_log.info("Building SQLancer image from cache: sqlancer:latest...") - run_command(["docker", "build", "-t", "sqlancer:latest", "./sqlancer"], docker_log) + if version: + script_log.info( + "Building SQLancer image from cache: sqlancer:latest (DUCKDB_JDBC_VERSION=%s) ...", + version + ) + run_command([ + "docker", "build", + "--build-arg", f"DUCKDB_JDBC_VERSION={version}", + "-t", "sqlancer:latest", "./sqlancer" + ], docker_log) + else: + script_log.info("Building SQLancer image from cache: sqlancer:latest ...") + run_command([ + "docker", "build", + "-t", "sqlancer:latest", "./sqlancer" + ], docker_log) script_log.info("SQLancer image built: sqlancer:latest") def build_network(script_log, docker_log, network_name="sqlancer-net"): @@ -61,7 +89,7 @@ def build_db_image(cfg, use_cache, script_log, docker_log, custom=False, dockerf def build_environment(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""): script_log.info("==============================Building environment==============================") build_network(script_log, docker_log) - build_sqlancer_image(script_log, docker_log, force_rebuild=False) + build_sqlancer_image(script_log, docker_log, not use_cache, cfg.get("version")) if cfg["embedded"] == "no": build_db_image(cfg, use_cache, script_log, docker_log, custom, dockerfile_path) script_log.info("==============================Building environment==============================") diff --git a/duckdb/config.json b/duckdb/config.json index 148100b..d44ec81 100644 --- a/duckdb/config.json +++ b/duckdb/config.json @@ -1,6 +1,7 @@ { "embedded": "yes", "dbms": "duckdb", + "version": "1.2.2.0", "username": "N/A", "password": "N/A", "container_name": "localhost", diff --git a/sqlancer/Dockerfile b/sqlancer/Dockerfile index 2c08003..63a9674 100644 --- a/sqlancer/Dockerfile +++ b/sqlancer/Dockerfile @@ -1,18 +1,20 @@ FROM ubuntu:20.04 - ENV DEBIAN_FRONTEND=noninteractive - RUN apt-get update && \ apt-get install -y openjdk-17-jdk maven git jq netcat mysql-client && \ rm -rf /var/lib/apt/lists/* WORKDIR /root/sqlancer +ARG DUCKDB_JDBC_VERSION=1.3.0.0 + RUN git clone https://github.com/sqlancer/sqlancer.git . && \ - mvn clean package -DskipTests + mvn -q versions:use-dep-version \ + -Dincludes=org.duckdb:duckdb_jdbc \ + -DdepVersion=${DUCKDB_JDBC_VERSION} \ + -DforceVersion=true && \ + mvn -q clean package -DskipTests COPY entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh - CMD ["/root/entrypoint.sh"] - From 4cdcee7b8cb1291723a761a49a76e2c7fd0b6add Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Wed, 3 Sep 2025 08:09:32 +0000 Subject: [PATCH 3/6] add customzied dockerfile to duckdb/ --- build.py | 34 +++++++++++++++++++++++++++++----- duckdb/Dockerfile | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 duckdb/Dockerfile diff --git a/build.py b/build.py index e6244a7..0d4434a 100644 --- a/build.py +++ b/build.py @@ -4,10 +4,29 @@ import logging from utils import run_command -def build_sqlancer_image(script_log, docker_log, force_rebuild=False): +def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=False): + context_dir = "./sqlancer" + if embedded == "yes": + dockerfile_path = f"./{dbms}/Dockerfile" + script_log.info(f"Using embedded DBMS Dockerfile: {dockerfile_path}") + script_log.info("Embedded mode: rebuilding SQLancer image unconditionally ...") + run_command( + ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], + docker_log + ) + script_log.info("SQLancer image built: sqlancer:latest") + return + else: + dockerfile_path = "./sqlancer/Dockerfile" + context_dir = "./sqlancer" + script_log.info(f"Using default SQLancer Dockerfile: {dockerfile_path}") + if force_rebuild: script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") - run_command(["docker", "build", "--no-cache", "-t", "sqlancer:latest", "./sqlancer"], docker_log) + run_command( + ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], + docker_log + ) script_log.info("SQLancer image built: sqlancer:latest") return @@ -19,10 +38,15 @@ def build_sqlancer_image(script_log, docker_log, force_rebuild=False): if "sqlancer:latest" in images: script_log.info("SQLancer image exists: sqlancer:latest") else: - script_log.info("Building SQLancer image from cache: sqlancer:latest...") - run_command(["docker", "build", "-t", "sqlancer:latest", "./sqlancer"], docker_log) + script_log.info("Building SQLancer image from cache: sqlancer:latest ...") + run_command( + ["docker", "build", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], + docker_log + ) script_log.info("SQLancer image built: sqlancer:latest") + + def build_network(script_log, docker_log, network_name="sqlancer-net"): try: output = subprocess.check_output([ @@ -61,7 +85,7 @@ def build_db_image(cfg, use_cache, script_log, docker_log, custom=False, dockerf def build_environment(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""): script_log.info("==============================Building environment==============================") build_network(script_log, docker_log) - build_sqlancer_image(script_log, docker_log, force_rebuild=False) + build_sqlancer_image(script_log, docker_log, cfg["embedded"], cfg["dbms"], force_rebuild=False) if cfg["embedded"] == "no": build_db_image(cfg, use_cache, script_log, docker_log, custom, dockerfile_path) script_log.info("==============================Building environment==============================") diff --git a/duckdb/Dockerfile b/duckdb/Dockerfile new file mode 100644 index 0000000..d7b2020 --- /dev/null +++ b/duckdb/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y openjdk-17-jdk maven git jq netcat mysql-client && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /root/sqlancer + +ARG DUCKDB_JDBC_VERSION=1.2.2.0 + +RUN git clone https://github.com/sqlancer/sqlancer.git . && \ + mvn -q versions:use-dep-version \ + -Dincludes=org.duckdb:duckdb_jdbc \ + -DdepVersion=${DUCKDB_JDBC_VERSION} \ + -DforceVersion=true && \ + mvn -q clean package -DskipTests + +COPY entrypoint.sh /root/entrypoint.sh +RUN chmod +x /root/entrypoint.sh +CMD ["/root/entrypoint.sh"] From 85c3ee2c5bc3edb68cb4ec114f57ad96006da326 Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Wed, 3 Sep 2025 08:15:50 +0000 Subject: [PATCH 4/6] add customized dckerfile --- build.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/build.py b/build.py index cd66835..f906a14 100644 --- a/build.py +++ b/build.py @@ -12,7 +12,7 @@ def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=F script_log.info("Embedded mode: rebuilding SQLancer image unconditionally ...") run_command( ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], - docker_log + docker_log ) script_log.info("SQLancer image built: sqlancer:latest") return @@ -22,22 +22,11 @@ def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=F script_log.info(f"Using default SQLancer Dockerfile: {dockerfile_path}") if force_rebuild: - if version: - script_log.info( - "Rebuilding SQLancer image: sqlancer:latest (DUCKDB_JDBC_VERSION=%s) ...", - version - ) - run_command([ - "docker", "build", "--no-cache", - "--build-arg", f"DUCKDB_JDBC_VERSION={version}", - "-t", "sqlancer:latest", "./sqlancer" - ], docker_log) - else: - script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") - run_command([ - "docker", "build", "--no-cache", - "-t", "sqlancer:latest", "./sqlancer" - ], docker_log) + script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") + run_command( + ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], + docker_log + ) script_log.info("SQLancer image built: sqlancer:latest") return @@ -58,6 +47,7 @@ def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=F + def build_network(script_log, docker_log, network_name="sqlancer-net"): try: output = subprocess.check_output([ From ea5f77eeb3461c844a100efeee10658a311a778d Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Wed, 3 Sep 2025 08:16:53 +0000 Subject: [PATCH 5/6] return to original dockerfile --- sqlancer/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sqlancer/Dockerfile b/sqlancer/Dockerfile index 63a9674..2c08003 100644 --- a/sqlancer/Dockerfile +++ b/sqlancer/Dockerfile @@ -1,20 +1,18 @@ FROM ubuntu:20.04 + ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update && \ apt-get install -y openjdk-17-jdk maven git jq netcat mysql-client && \ rm -rf /var/lib/apt/lists/* WORKDIR /root/sqlancer -ARG DUCKDB_JDBC_VERSION=1.3.0.0 - RUN git clone https://github.com/sqlancer/sqlancer.git . && \ - mvn -q versions:use-dep-version \ - -Dincludes=org.duckdb:duckdb_jdbc \ - -DdepVersion=${DUCKDB_JDBC_VERSION} \ - -DforceVersion=true && \ - mvn -q clean package -DskipTests + mvn clean package -DskipTests COPY entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh + CMD ["/root/entrypoint.sh"] + From 2320ed885bc4da7e801bcee02784610f9c4c1604 Mon Sep 17 00:00:00 2001 From: CelestaLiu Date: Wed, 3 Sep 2025 08:19:36 +0000 Subject: [PATCH 6/6] modify config --- duckdb/config.json | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/duckdb/config.json b/duckdb/config.json index d44ec81..ef0d84b 100644 --- a/duckdb/config.json +++ b/duckdb/config.json @@ -1,12 +1,10 @@ { - "embedded": "yes", - "dbms": "duckdb", - "version": "1.2.2.0", - "username": "N/A", - "password": "N/A", - "container_name": "localhost", - "oracle": "NoREC", - "num_threads": 4, - "timeout_seconds": 60 - } - \ No newline at end of file + "embedded": "yes", + "dbms": "duckdb", + "username": "N/A", + "password": "N/A", + "container_name": "localhost", + "oracle": "NoREC", + "num_threads": 4, + "timeout_seconds": 60 +} \ No newline at end of file