Cache native library build

.github/workflows/deploy.yml

@@ -3,6 +3,9 @@ name: Release
 on: workflow_dispatch
 
 jobs:
+  fetch_prebuilts:
+    uses: ./.github/workflows/prebuild_assets.yml
+
   draft_release:
     permissions:
       contents: write
@@ -31,6 +34,8 @@ jobs:
 
   maven_publish:
     runs-on: macos-latest
+    needs:
+      - fetch_prebuilts
     steps:
       - uses: actions/checkout@v5
       - name: Validate Gradle Wrapper
@@ -46,10 +51,16 @@ jobs:
           distribution: 'temurin'
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4
+      - name: Download prebuilts
+        uses: actions/download-artifact@v5
+        with:
+          artifact-ids: ${{ needs.fetch_prebuilts.outputs.artifact_id }}
+          path: internal/prebuild-binaries/build/output/
       - name: Gradle publish
         run: |
           ./gradlew \
             --no-configuration-cache \
+            -PhasPrebuiltAssets=true
             -PGITHUB_PUBLISH_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
             -PsigningInMemoryKey="${{ secrets.SIGNING_KEY }}" \
             -PsigningInMemoryKeyId="${{ secrets.SIGNING_KEY_ID }}" \

.github/workflows/prebuild_assets.yml

@@ -0,0 +1,65 @@
+name: Prebuild SQLite assets
+
+on:
+  workflow_call:
+    outputs:
+      artifact_url:
+        description: "URL of the artifact containing precompiled SQLite binaries"
+        value: ${{ jobs.compile.outputs.artifact_url }}
+      artifact_id:
+        description: "id of the artifact containing precompiled SQLite binaries"
+        value: ${{ jobs.compile.outputs.artifact_id }}
+
+jobs:
+  compile:
+    name: Compile SQLite
+    timeout-minutes: 30
+    runs-on: macos-latest
+    outputs:
+      artifact_url: ${{ steps.upload.outputs.artifact-url }}
+      artifact_id: ${{ steps.upload.outputs.artifact-id }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: actions/cache@v4
+        id: cache_prebuild
+        with:
+          path: internal/prebuild-binaries/build/output
+          key: sqlite-build-${{ hashFiles('internal/prebuild-binaries', 'plugins/build-plugin') }}
+
+      - name: Validate Gradle Wrapper
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        uses: gradle/actions/wrapper-validation@v4
+      - uses: actions/cache@v4
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        with:
+          path: ~/.konan
+          key: ${{ runner.os }}-${{ hashFiles('**/.lock') }}
+      - name: Set up JDK 25
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        uses: actions/setup-java@v4
+        with:
+          java-version: '25'
+          distribution: 'temurin'
+      - name: Set up Gradle
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
+      - name: Set up XCode
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        uses: maxim-lobanov/setup-xcode@v1
+      - name: Compile SQLite with Gradle
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        run: |
+          ./gradlew --scan internal:prebuild-binaries:compileNative
+        shell: bash
+      - uses: actions/upload-artifact@v5
+        id: upload
+        with:
+          path: internal/prebuild-binaries/build/output/*
+          name: kotlin-sqlite
+          retention-days: 1
+      - run: |
+          echo "${{ steps.upload.outputs.artifact-url }}"

.github/workflows/test.yml

@@ -11,7 +11,12 @@ permissions:
   contents: read
 
 jobs:
+  fetch_prebuilts:
+    uses: ./.github/workflows/prebuild_assets.yml
+
   build:
+    needs:
+      - fetch_prebuilts
     strategy:
       matrix:
         include:
@@ -61,10 +66,15 @@ jobs:
       with:
         # TODO: Update to latest-stable once GH installs iOS 26 simulators
         xcode-version: '^16.4.0'
+    - name: Download prebuilts
+      uses: actions/download-artifact@v5
+      with:
+        artifact-ids: ${{ needs.fetch_prebuilts.outputs.artifact_id }}
+        path: internal/prebuild-binaries/build/output/
 
     - name: Build and run tests with Gradle
       run: |
-        ./gradlew --scan \
+        ./gradlew --scan -PhasPrebuiltAssets=true \
           ${{ matrix.targets }}
       shell: bash
 

internal/prebuild-binaries/README.md

@@ -0,0 +1,15 @@
+## Prebuild SQLite binaries
+
+The purpose of this internal project is to build variants of SQLite and SQLite3MultipleCiphers used in the PowerSync
+Kotlin SDK.
+
+Specifically, this builds:
+
+1. SQLite as a static library for iOS/macOS/watchOS/tvOS (+ simulators).
+2. SQLite3MultipleCiphers as a static library for iOS/macOS/watchOS/tvOS (+ simulators).
+
+We don't want to build these assets on every build since they're included in a `cinterops` definition file, meaning that
+they would have to be built during Gradle sync, which slows down that process.
+
+Instead, we use a cache for GitHub actions to only recompile these when necessary. During the main build, we then use
+a custom property to download assets instead of recompiling.

internal/prebuild-binaries/build.gradle.kts

@@ -0,0 +1,127 @@
+import com.powersync.compile.ClangCompile
+import com.powersync.compile.UnzipSqlite
+import de.undercouch.gradle.tasks.download.Download
+import org.gradle.kotlin.dsl.register
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import com.powersync.compile.CreateStaticLibrary
+
+plugins {
+    alias(libs.plugins.downloadPlugin)
+}
+
+val sqlite3McVersion = "2.2.6"
+val sqlite3BaseVersion = "3.51.1"
+val sqlite3ReleaseYear = "2025"
+val sqlite3ExpandedVersion = "3510100"
+
+val downloadSQLiteSources by tasks.registering(Download::class) {
+    val zipFileName = "sqlite-amalgamation-$sqlite3ExpandedVersion.zip"
+    src("https://www.sqlite.org/$sqlite3ReleaseYear/$zipFileName")
+    dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) })
+    onlyIfNewer(true)
+    overwrite(false)
+}
+
+val downloadSqlite3MultipleCipherSources by tasks.registering(Download::class) {
+    val zipFileName = "sqlite3mc-$sqlite3McVersion.zip"
+    src("https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v$sqlite3McVersion/sqlite3mc-$sqlite3McVersion-sqlite-$sqlite3BaseVersion-amalgamation.zip")
+    dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) })
+    onlyIfNewer(true)
+    overwrite(false)
+}
+
+val unzipSQLiteSources by tasks.registering(UnzipSqlite::class) {
+    val zip = downloadSQLiteSources.map { it.outputs.files.singleFile }
+    inputs.file(zip)
+
+    unzipSqlite(
+        src = zipTree(zip),
+        dir = layout.buildDirectory.dir("downloads/sqlite3")
+    )
+}
+
+val unzipSqlite3MultipleCipherSources by tasks.registering(UnzipSqlite::class) {
+    val zip = downloadSqlite3MultipleCipherSources.map { it.outputs.files.singleFile }
+    inputs.file(zip)
+
+    unzipSqlite(
+        src = zipTree(zip),
+        dir = layout.buildDirectory.dir("downloads/sqlite3mc"),
+        filter = null,
+    )
+}
+
+fun compileSqliteForKotlinNativeOnApple(library: String, abi: String): TaskProvider<CreateStaticLibrary> {
+    val name = "$library$abi"
+    val outputDir = layout.buildDirectory.dir("c/$abi")
+
+    val sqlite3Obj = outputDir.map { it.file("$library.o") }
+    val archive = outputDir.map { it.file("lib$library.a") }
+
+    val compileSqlite = tasks.register("${name}CompileSqlite", ClangCompile::class) {
+        val (sourceTask, filename) = if (library == "sqlite3") {
+            unzipSQLiteSources to "sqlite3.c"
+        } else {
+            unzipSqlite3MultipleCipherSources to "sqlite3mc_amalgamation.c"
+        }
+
+        inputs.dir(sourceTask.map { it.destination })
+        include.set(unzipSQLiteSources.flatMap { it.destination })
+        inputFile.set(sourceTask.flatMap { it.destination.file(filename) })
+
+        konanTarget.set(abi)
+        objectFile.set(sqlite3Obj)
+    }
+
+    val createStaticLibrary = tasks.register("${name}ArchiveSqlite", CreateStaticLibrary::class) {
+        inputs.file(compileSqlite.map { it.objectFile })
+        objects.from(sqlite3Obj)
+        staticLibrary.set(archive)
+    }
+
+    return createStaticLibrary
+}
+
+data class CompiledAsset(
+    val output: Provider<RegularFileProperty>,
+    val fullName: String,
+)
+
+val compileTasks = buildList {
+    val targets = KonanTarget.predefinedTargets.values.filter { it.family.isAppleFamily }.map { it.name }.toList()
+    for (library in listOf("sqlite3", "sqlite3mc")) {
+        for (abi in targets) {
+            val task = compileSqliteForKotlinNativeOnApple(library, abi)
+            val output = task.map { it.staticLibrary }
+            val fullName = "$abi$library.a"
+
+            add(CompiledAsset(output, fullName))
+        }
+    }
+}
+
+val compileNative by tasks.registering(Copy::class) {
+    into(project.layout.buildDirectory.dir("output"))
+
+    for (task in compileTasks) {
+        from(task.output) {
+            rename { task.fullName }
+        }
+    }
+}
+
+val hasPrebuiltAssets = providers.gradleProperty("hasPrebuiltAssets").map { it.toBooleanStrict() }
+
+val nativeSqliteConfiguration by configurations.creating {
+    isCanBeResolved = false
+}
+
+artifacts {
+    if (hasPrebuiltAssets.getOrElse(false)) {
+        // In CI builds, we set hasPrebuiltAssets=true. In that case, contents of build/output have been downloaded from
+        // cache and don't need to be rebuilt.
+        add(nativeSqliteConfiguration.name, layout.buildDirectory.dir("output"))
+    } else {
+        add(nativeSqliteConfiguration.name, compileNative)
+    }
+}

plugins/build-plugin/src/main/kotlin/com/powersync/compile/UnzipSqlite.kt

@@ -16,10 +16,10 @@ abstract class UnzipSqlite: Copy() {
     @get:OutputDirectory
     abstract val destination: DirectoryProperty
 
-    fun unzipSqlite(src: FileTree, dir: Provider<Directory>) {
+    fun unzipSqlite(src: FileTree, dir: Provider<Directory>, filter: String? = "*/sqlite3.*") {
         from(
             src.matching {
-                include("*/sqlite3.*")
+                filter?.let { include(it) }
                 exclude {
                     it.isDirectory
                 }

settings.gradle.kts

@@ -30,6 +30,7 @@ rootProject.name = "powersync-root"
 include(":internal:download-core-extension")
 include(":internal:PowerSyncKotlin")
 include(":internal:testutils")
+include(":internal:prebuild-binaries")
 
 include(":common")
 include(":core")