download and build any sqlite version for testing

.github/CONTRIBUTING.md

@@ -26,3 +26,17 @@ When making changes to stanchion, there are a few commands that are useful for t
 * [zig build ext]: build the runtime loadable extension as a dynamic library which can be found in the `zig-out/lib` folder
 
 Once your changes are ready, open a pull requests against the stanchion GitHub repository.
+
+### Test across SQLite versions
+
+By default, tests use the system SQLite library. However, stanchion's build can optionally download and compile a specific version of SQLite and use that version when running tests. Pass `-Dsqlite-test-version=$SQLITE_VERSION` to the build for unit and integration tests. For example:
+
+```
+zig build test -Dsqlite-test-version=3.38.5
+```
+
+It is also possible to launch a SQLite shell for any version of SQLite (a convenience feature for debugging):
+
+```
+zig build sqlite-shell -Dsqlite-test-version=3.43.2
+```

.github/workflows/ci.yaml

@@ -19,21 +19,14 @@ jobs:
         os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
         sqlite:
           - version: 3.26.0
-            year: 2018
           - version: 3.30.1
-            year: 2019
           - version: 3.34.1
-            year: 2021
           - version: 3.38.5
-            year: 2022
           - version: 3.42.0
-            year: 2023
-          - version: 3.45.0
-            year: 2024
+          - version: 3.45.1
     runs-on: ${{ matrix.os }}
     env:
       SQLITE_VERSION: ${{ matrix.sqlite.version }}
-      SQLITE_YEAR: ${{ matrix.sqlite.year }}
     steps:
       - uses: actions/checkout@v4
 

README.md

@@ -61,6 +61,20 @@ zig build ext -Doptimize=ReleaseFast
 
 The SQLite extension is the dynamic library named `libstanchion` in the `zig-out` directory.
 
+### Running tests
+
+By default, tests use the system SQLite library. However, stanchion's build can optionally download and compile a specific version of SQLite and use that version when running tests. Pass `-Dsqlite-test-version=$SQLITE_VERSION` to the build for unit and integration tests. For example:
+
+```
+zig build test -Dsqlite-test-version=3.38.5
+```
+
+It is also possible to launch a SQLite shell for any version of SQLite (a convenience feature for debugging):
+
+```
+zig build sqlite-shell -Dsqlite-test-version=3.43.2
+```
+
 ## Usage
 
 ### Load Stanchion

build.zig

@@ -3,39 +3,23 @@ const Build = std.Build;
 const Compile = Build.Step.Compile;
 const OptimizeMode = std.builtin.OptimizeMode;
 
-pub fn buildLoadableExt(b: *Build, target: Build.ResolvedTarget, optimize: OptimizeMode) *Compile {
-    const loadable_ext = b.addSharedLibrary(.{
-        .name = "stanchion",
-        .root_source_file = .{ .path = "src/main.zig" },
-        .target = target,
-        .optimize = optimize,
-    });
-    // TODO remove when issue is resolved:
-    //      https://github.com/ziglang/zig/issues/7935
-    if (target.result.cpu.arch == .x86) {
-        loadable_ext.link_z_notext = true;
-    }
-    // TODO should this be added back somehow?
-    //loadable_ext.force_pic = true;
-    // TODO remove when issue is resolved:
-    //      https://github.com/ziglang/zig/issues/15893
-    loadable_ext.addCSourceFile(.{
-        .file = .{ .path = "src/sqlite3/c/result-transient-ext.c" },
-        .flags = &[_][]const u8{"-std=c99"},
-    });
-    loadable_ext.addIncludePath(.{ .path = "src/sqlite3/c" });
-
-    const loadable_ext_options = b.addOptions();
-    loadable_ext.root_module.addOptions("build_options", loadable_ext_options);
-    loadable_ext_options.addOption(bool, "loadable_extension", true);
-
-    return loadable_ext;
-}
+const InstallSqliteSrc = struct {
+    include_dir: Build.LazyPath,
+    src_file: Build.LazyPath,
+    shell_src_file: Build.LazyPath,
+};
 
 // Although this function looks imperative, note that its job is to
 // declaratively construct a build graph that will be executed by an external
 // runner.
 pub fn build(b: *Build) void {
+    // If not specified, will attempt to use system sqlite
+    const sqlite_version = b.option(
+        []const u8,
+        "sqlite-test-version",
+        "Version of sqlite to use for testing",
+    );
+
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
     // means any target is allowed, and the default is native. Other options
@@ -55,6 +39,70 @@ pub fn build(b: *Build) void {
     const loadable_ext_build = b.step("ext", "Build the 'stanchion' sqlite loadable extension");
     loadable_ext_build.dependOn(&install_loadable_ext.step);
 
+    var install_sqlite_src: ?InstallSqliteSrc = null;
+    var build_sqlite_shell: ?*Build.Step.Compile = null;
+    if (sqlite_version) |version| {
+        // Install SQLite src
+
+        const download_sqlite_src = b.addExecutable(.{
+            .name = "download_sqlite_src",
+            .root_source_file = .{ .path = "tools/download_sqlite_src.zig" },
+            .target = b.host,
+        });
+        const run_dl_sqlite_src = b.addRunArtifact(download_sqlite_src);
+        run_dl_sqlite_src.addArg(version);
+        const sqlite_src_zip_name = b.fmt("sqlite-{s}.zip", .{version});
+        const sqlite_src_zip = run_dl_sqlite_src.addOutputFileArg(sqlite_src_zip_name);
+
+        const unzip_sqlite_src = b.addSystemCommand(&.{ "unzip", "-oj" });
+        unzip_sqlite_src.addFileArg(sqlite_src_zip);
+        unzip_sqlite_src.addArg("-d");
+        const sqlite_src_dir_name = b.fmt("sqlite-{s}", .{version});
+        const sqlite_src_dir = unzip_sqlite_src.addOutputFileArg(sqlite_src_dir_name);
+
+        const install_sqlite_step = b.step("install-sqlite", "Install SQLite");
+        install_sqlite_step.dependOn(&unzip_sqlite_src.step);
+
+        const inst_sqlite_src = b.addExecutable(.{
+            .name = "install_sqlite_src",
+            .root_source_file = .{ .path = "tools/install_sqlite_src.zig" },
+            .target = b.host,
+        });
+        const run_install_sqlite_src = b.addRunArtifact(inst_sqlite_src);
+        run_install_sqlite_src.addDirectoryArg(sqlite_src_dir);
+        const sqlite_src_file = run_install_sqlite_src.addOutputFileArg("sqlite3.c");
+        const shell_src_file = run_install_sqlite_src.addOutputFileArg("shell.c");
+
+        install_sqlite_src = .{
+            .include_dir = sqlite_src_dir,
+            .src_file = sqlite_src_file,
+            .shell_src_file = shell_src_file,
+        };
+
+        // SQLite shell
+
+        const sqlite_shell = b.addExecutable(.{
+            .name = "sqlite_shell",
+            .target = target,
+            .optimize = .ReleaseFast,
+        });
+        sqlite_shell.linkLibC();
+        sqlite_shell.addCSourceFile(.{
+            .file = install_sqlite_src.?.src_file,
+            .flags = &[_][]const u8{"-std=c99"},
+        });
+        sqlite_shell.addCSourceFile(.{
+            .file = install_sqlite_src.?.shell_src_file,
+        });
+        sqlite_shell.addIncludePath(install_sqlite_src.?.include_dir);
+
+        build_sqlite_shell = sqlite_shell;
+
+        const run_sqlite_shell = b.addRunArtifact(sqlite_shell);
+        const sqlite_shell_step = b.step("sqlite-shell", "Run sqlite shell");
+        sqlite_shell_step.dependOn(&run_sqlite_shell.step);
+    }
+
     // Unit tests
 
     const main_tests = b.addTest(.{
@@ -63,10 +111,19 @@ pub fn build(b: *Build) void {
         .optimize = optimize,
     });
     main_tests.linkLibC();
-    main_tests.addCSourceFile(.{
-        .file = .{ .path = "src/sqlite3/c/sqlite3.c" },
-        .flags = &[_][]const u8{"-std=c99"},
-    });
+    if (install_sqlite_src) |install_src| {
+        main_tests.addCSourceFile(.{
+            .file = install_src.src_file,
+            .flags = &[_][]const u8{"-std=c99"},
+        });
+        main_tests.addIncludePath(install_src.include_dir);
+    } else {
+        main_tests.linkSystemLibrary("sqlite3");
+        main_tests.addCSourceFile(.{
+            .file = .{ .path = "src/sqlite3/c/result-transient.c" },
+            .flags = &[_][]const u8{"-std=c99"},
+        });
+    }
     main_tests.addIncludePath(.{ .path = "src/sqlite3/c" });
     const main_tests_options = b.addOptions();
     main_tests.root_module.addOptions("build_options", main_tests_options);
@@ -78,14 +135,16 @@ pub fn build(b: *Build) void {
 
     // Integration tests
 
-    const run_integration_tests = b.addSystemCommand(&[_][]const u8{
-        "bash", "test/runtest.sh",
-    });
-    // Build the loadable extension and then run the integration test script
+    // Build the loadable extension in debug mode
     const loadable_ext_itest = buildLoadableExt(b, target, .Debug);
-    const install_loadable_ext_itest = b.addInstallArtifact(loadable_ext_itest, .{});
+    const run_integration_tests = b.addSystemCommand(&[_][]const u8{ "bash", "test/runtest.sh" });
+    if (build_sqlite_shell) |bss| {
+        run_integration_tests.addFileArg(bss.getEmittedBin());
+    } else {
+        run_integration_tests.addArg("sqlite3");
+    }
+    run_integration_tests.addFileArg(loadable_ext_itest.getEmittedBin());
     const integration_test_step = b.step("itest", "Run integration tests");
-    run_integration_tests.step.dependOn(&install_loadable_ext_itest.step);
     integration_test_step.dependOn(&run_integration_tests.step);
 
     // Benchmarks
@@ -97,10 +156,19 @@ pub fn build(b: *Build) void {
         .optimize = .ReleaseFast,
     });
     benches.linkLibC();
-    benches.addCSourceFile(.{
-        .file = .{ .path = "src/sqlite3/c/sqlite3.c" },
-        .flags = &[_][]const u8{"-std=c99"},
-    });
+    if (install_sqlite_src) |install_src| {
+        benches.addCSourceFile(.{
+            .file = install_src.src_file,
+            .flags = &[_][]const u8{"-std=c99"},
+        });
+        benches.addIncludePath(install_src.include_dir);
+    } else {
+        benches.linkSystemLibrary("sqlite3");
+        benches.addCSourceFile(.{
+            .file = .{ .path = "src/sqlite3/c/result-transient.c" },
+            .flags = &[_][]const u8{"-std=c99"},
+        });
+    }
     benches.addIncludePath(.{ .path = "src/sqlite3/c" });
     const benches_options = b.addOptions();
     benches.root_module.addOptions("build_options", benches_options);
@@ -110,3 +178,32 @@ pub fn build(b: *Build) void {
     const bench_step = b.step("bench", "Run benchmarks");
     bench_step.dependOn(&run_benches.step);
 }
+
+pub fn buildLoadableExt(b: *Build, target: Build.ResolvedTarget, optimize: OptimizeMode) *Compile {
+    const loadable_ext = b.addSharedLibrary(.{
+        .name = "stanchion",
+        .root_source_file = .{ .path = "src/main.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+    // TODO remove when issue is resolved:
+    //      https://github.com/ziglang/zig/issues/7935
+    if (target.result.cpu.arch == .x86) {
+        loadable_ext.link_z_notext = true;
+    }
+    // TODO should this be added back somehow?
+    //loadable_ext.force_pic = true;
+    // TODO remove when issue is resolved:
+    //      https://github.com/ziglang/zig/issues/15893
+    loadable_ext.addCSourceFile(.{
+        .file = .{ .path = "src/sqlite3/c/result-transient-ext.c" },
+        .flags = &[_][]const u8{"-std=c99"},
+    });
+    loadable_ext.addIncludePath(.{ .path = "src/sqlite3/c" });
+
+    const loadable_ext_options = b.addOptions();
+    loadable_ext.root_module.addOptions("build_options", loadable_ext_options);
+    loadable_ext_options.addOption(bool, "loadable_extension", true);
+
+    return loadable_ext;
+}

ci/linux-x86_64-test.sh

@@ -48,47 +48,6 @@ rm ~/deps/zig-version-index.json
 PATH="$HOME/deps/zig:$PATH"
 zig version
 
-## Sqlite
-sqlite32bit=0
-if [ ! -f ~/deps/SQLITE32BIT ]; then
-    echo ">> Downloading and installing SQLite (${SQLITE_VERSION})"
-
-    rm -rf ~/deps/sqlite
-    mkdir ~/deps/sqlite
-
-    IFS='.' read -r maj min pat <<EOF
-$SQLITE_VERSION
-EOF
-    if [ ${#min} = 1 ]; then
-        min="0${min}"
-    fi
-    if [ ${#pat} = 1 ]; then
-        pat="0${pat}"
-    fi
-    versionstring="${maj}${min}${pat}00"
-
-    sqliteurl="https://www.sqlite.org/$SQLITE_YEAR/sqlite-tools-linux-x64-$versionstring.zip"
-    # See if it comes in 64 bit
-    wget -q --spider --tries 1 "$sqliteurl" || sqlite32bit=$?
-    if [ $sqlite32bit -ne 0 ]; then
-        # Use the 32 bit version
-        sqliteurl="https://www.sqlite.org/$SQLITE_YEAR/sqlite-tools-linux-x86-$versionstring.zip"
-    fi
-
-    wget -qO ~/deps/sqlite.zip "$sqliteurl"
-    unzip -j -d ~/deps/sqlite ~/deps/sqlite.zip
-
-    echo "$sqlite32bit" > ~/deps/SQLITE32BIT
-
-    rm ~/deps/sqlite.zip
-else
-    echo ">> Using cached SQLite"
-    sqlite32bit=$(cat ~/deps/SQLITE32BIT)
-fi
-
-PATH="$HOME/deps/sqlite:$PATH"
-sqlite3 --version
-
 ####
 ## Checks
 ####
@@ -97,15 +56,9 @@ sqlite3 --version
 zig fmt --check .
 
 ## Unit tests
-zig build test --summary all
+zig build test -Dsqlite-test-version=${SQLITE_VERSION} --summary all
 
 ## Integration tests
-if [ $sqlite32bit -ne 0 ]; then
-    echo ">> Running with x86 sqlite"
-    zig build itest --summary all -Dtarget=x86-linux
-else
-    echo ">> Running with x86_64 sqlite"
-    zig build itest --summary all
-fi
+zig build itest -Dsqlite-test-version=${SQLITE_VERSION} --summary all
 
 exit 0