diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bd48b9b5..4a5f4470 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,6 +3,9 @@ name: Release on: workflow_dispatch jobs: + fetch_prebuilts: + uses: ./.github/workflows/prebuild_assets.yml + draft_release: permissions: contents: write @@ -31,6 +34,8 @@ jobs: maven_publish: runs-on: macos-latest + needs: + - fetch_prebuilts steps: - uses: actions/checkout@v5 - name: Validate Gradle Wrapper @@ -46,10 +51,16 @@ jobs: distribution: 'temurin' - name: Set up Gradle uses: gradle/actions/setup-gradle@v4 + - name: Download prebuilts + uses: actions/download-artifact@v5 + with: + artifact-ids: ${{ needs.fetch_prebuilts.outputs.artifact_id }} + path: internal/prebuild-binaries/build/output/ - name: Gradle publish run: | ./gradlew \ --no-configuration-cache \ + -PhasPrebuiltAssets=true -PGITHUB_PUBLISH_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ -PsigningInMemoryKey="${{ secrets.SIGNING_KEY }}" \ -PsigningInMemoryKeyId="${{ secrets.SIGNING_KEY_ID }}" \ diff --git a/.github/workflows/prebuild_assets.yml b/.github/workflows/prebuild_assets.yml new file mode 100644 index 00000000..2dd05f07 --- /dev/null +++ b/.github/workflows/prebuild_assets.yml @@ -0,0 +1,65 @@ +name: Prebuild SQLite assets + +on: + workflow_call: + outputs: + artifact_url: + description: "URL of the artifact containing precompiled SQLite binaries" + value: ${{ jobs.compile.outputs.artifact_url }} + artifact_id: + description: "id of the artifact containing precompiled SQLite binaries" + value: ${{ jobs.compile.outputs.artifact_id }} + +jobs: + compile: + name: Compile SQLite + timeout-minutes: 30 + runs-on: macos-latest + outputs: + artifact_url: ${{ steps.upload.outputs.artifact-url }} + artifact_id: ${{ steps.upload.outputs.artifact-id }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/cache@v4 + id: cache_prebuild + with: + path: internal/prebuild-binaries/build/output + key: sqlite-build-${{ hashFiles('internal/prebuild-binaries', 'plugins/build-plugin') }} + + - name: Validate Gradle Wrapper + if: steps.cache_prebuild.outputs.cache-hit != 'true' + uses: gradle/actions/wrapper-validation@v4 + - uses: actions/cache@v4 + if: steps.cache_prebuild.outputs.cache-hit != 'true' + with: + path: ~/.konan + key: ${{ runner.os }}-${{ hashFiles('**/.lock') }} + - name: Set up JDK 25 + if: steps.cache_prebuild.outputs.cache-hit != 'true' + uses: actions/setup-java@v4 + with: + java-version: '25' + distribution: 'temurin' + - name: Set up Gradle + if: steps.cache_prebuild.outputs.cache-hit != 'true' + uses: gradle/actions/setup-gradle@v4 + with: + cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + - name: Set up XCode + if: steps.cache_prebuild.outputs.cache-hit != 'true' + uses: maxim-lobanov/setup-xcode@v1 + - name: Compile SQLite with Gradle + if: steps.cache_prebuild.outputs.cache-hit != 'true' + run: | + ./gradlew --scan internal:prebuild-binaries:compileNative + shell: bash + - uses: actions/upload-artifact@v5 + id: upload + with: + path: internal/prebuild-binaries/build/output/* + name: kotlin-sqlite + retention-days: 1 + - run: | + echo "${{ steps.upload.outputs.artifact-url }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9fb01025..57625181 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,12 @@ permissions: contents: read jobs: + fetch_prebuilts: + uses: ./.github/workflows/prebuild_assets.yml + build: + needs: + - fetch_prebuilts strategy: matrix: include: @@ -61,10 +66,15 @@ jobs: with: # TODO: Update to latest-stable once GH installs iOS 26 simulators xcode-version: '^16.4.0' + - name: Download prebuilts + uses: actions/download-artifact@v5 + with: + artifact-ids: ${{ needs.fetch_prebuilts.outputs.artifact_id }} + path: internal/prebuild-binaries/build/output/ - name: Build and run tests with Gradle run: | - ./gradlew --scan \ + ./gradlew --scan -PhasPrebuiltAssets=true \ ${{ matrix.targets }} shell: bash diff --git a/internal/prebuild-binaries/README.md b/internal/prebuild-binaries/README.md new file mode 100644 index 00000000..f2e8d28f --- /dev/null +++ b/internal/prebuild-binaries/README.md @@ -0,0 +1,15 @@ +## Prebuild SQLite binaries + +The purpose of this internal project is to build variants of SQLite and SQLite3MultipleCiphers used in the PowerSync +Kotlin SDK. + +Specifically, this builds: + +1. SQLite as a static library for iOS/macOS/watchOS/tvOS (+ simulators). +2. SQLite3MultipleCiphers as a static library for iOS/macOS/watchOS/tvOS (+ simulators). + +We don't want to build these assets on every build since they're included in a `cinterops` definition file, meaning that +they would have to be built during Gradle sync, which slows down that process. + +Instead, we use a cache for GitHub actions to only recompile these when necessary. During the main build, we then use +a custom property to download assets instead of recompiling. diff --git a/internal/prebuild-binaries/build.gradle.kts b/internal/prebuild-binaries/build.gradle.kts new file mode 100644 index 00000000..2db4e0f0 --- /dev/null +++ b/internal/prebuild-binaries/build.gradle.kts @@ -0,0 +1,127 @@ +import com.powersync.compile.ClangCompile +import com.powersync.compile.UnzipSqlite +import de.undercouch.gradle.tasks.download.Download +import org.gradle.kotlin.dsl.register +import org.jetbrains.kotlin.konan.target.KonanTarget +import com.powersync.compile.CreateStaticLibrary + +plugins { + alias(libs.plugins.downloadPlugin) +} + +val sqlite3McVersion = "2.2.6" +val sqlite3BaseVersion = "3.51.1" +val sqlite3ReleaseYear = "2025" +val sqlite3ExpandedVersion = "3510100" + +val downloadSQLiteSources by tasks.registering(Download::class) { + val zipFileName = "sqlite-amalgamation-$sqlite3ExpandedVersion.zip" + src("https://www.sqlite.org/$sqlite3ReleaseYear/$zipFileName") + dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) }) + onlyIfNewer(true) + overwrite(false) +} + +val downloadSqlite3MultipleCipherSources by tasks.registering(Download::class) { + val zipFileName = "sqlite3mc-$sqlite3McVersion.zip" + src("https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v$sqlite3McVersion/sqlite3mc-$sqlite3McVersion-sqlite-$sqlite3BaseVersion-amalgamation.zip") + dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) }) + onlyIfNewer(true) + overwrite(false) +} + +val unzipSQLiteSources by tasks.registering(UnzipSqlite::class) { + val zip = downloadSQLiteSources.map { it.outputs.files.singleFile } + inputs.file(zip) + + unzipSqlite( + src = zipTree(zip), + dir = layout.buildDirectory.dir("downloads/sqlite3") + ) +} + +val unzipSqlite3MultipleCipherSources by tasks.registering(UnzipSqlite::class) { + val zip = downloadSqlite3MultipleCipherSources.map { it.outputs.files.singleFile } + inputs.file(zip) + + unzipSqlite( + src = zipTree(zip), + dir = layout.buildDirectory.dir("downloads/sqlite3mc"), + filter = null, + ) +} + +fun compileSqliteForKotlinNativeOnApple(library: String, abi: String): TaskProvider { + val name = "$library$abi" + val outputDir = layout.buildDirectory.dir("c/$abi") + + val sqlite3Obj = outputDir.map { it.file("$library.o") } + val archive = outputDir.map { it.file("lib$library.a") } + + val compileSqlite = tasks.register("${name}CompileSqlite", ClangCompile::class) { + val (sourceTask, filename) = if (library == "sqlite3") { + unzipSQLiteSources to "sqlite3.c" + } else { + unzipSqlite3MultipleCipherSources to "sqlite3mc_amalgamation.c" + } + + inputs.dir(sourceTask.map { it.destination }) + include.set(unzipSQLiteSources.flatMap { it.destination }) + inputFile.set(sourceTask.flatMap { it.destination.file(filename) }) + + konanTarget.set(abi) + objectFile.set(sqlite3Obj) + } + + val createStaticLibrary = tasks.register("${name}ArchiveSqlite", CreateStaticLibrary::class) { + inputs.file(compileSqlite.map { it.objectFile }) + objects.from(sqlite3Obj) + staticLibrary.set(archive) + } + + return createStaticLibrary +} + +data class CompiledAsset( + val output: Provider, + val fullName: String, +) + +val compileTasks = buildList { + val targets = KonanTarget.predefinedTargets.values.filter { it.family.isAppleFamily }.map { it.name }.toList() + for (library in listOf("sqlite3", "sqlite3mc")) { + for (abi in targets) { + val task = compileSqliteForKotlinNativeOnApple(library, abi) + val output = task.map { it.staticLibrary } + val fullName = "$abi$library.a" + + add(CompiledAsset(output, fullName)) + } + } +} + +val compileNative by tasks.registering(Copy::class) { + into(project.layout.buildDirectory.dir("output")) + + for (task in compileTasks) { + from(task.output) { + rename { task.fullName } + } + } +} + +val hasPrebuiltAssets = providers.gradleProperty("hasPrebuiltAssets").map { it.toBooleanStrict() } + +val nativeSqliteConfiguration by configurations.creating { + isCanBeResolved = false +} + +artifacts { + if (hasPrebuiltAssets.getOrElse(false)) { + // In CI builds, we set hasPrebuiltAssets=true. In that case, contents of build/output have been downloaded from + // cache and don't need to be rebuilt. + add(nativeSqliteConfiguration.name, layout.buildDirectory.dir("output")) + } else { + add(nativeSqliteConfiguration.name, compileNative) + } +} diff --git a/plugins/build-plugin/src/main/kotlin/com/powersync/compile/UnzipSqlite.kt b/plugins/build-plugin/src/main/kotlin/com/powersync/compile/UnzipSqlite.kt index 65aa1157..af58169e 100644 --- a/plugins/build-plugin/src/main/kotlin/com/powersync/compile/UnzipSqlite.kt +++ b/plugins/build-plugin/src/main/kotlin/com/powersync/compile/UnzipSqlite.kt @@ -16,10 +16,10 @@ abstract class UnzipSqlite: Copy() { @get:OutputDirectory abstract val destination: DirectoryProperty - fun unzipSqlite(src: FileTree, dir: Provider) { + fun unzipSqlite(src: FileTree, dir: Provider, filter: String? = "*/sqlite3.*") { from( src.matching { - include("*/sqlite3.*") + filter?.let { include(it) } exclude { it.isDirectory } diff --git a/settings.gradle.kts b/settings.gradle.kts index 064dd7e2..e793b1c7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -30,6 +30,7 @@ rootProject.name = "powersync-root" include(":internal:download-core-extension") include(":internal:PowerSyncKotlin") include(":internal:testutils") +include(":internal:prebuild-binaries") include(":common") include(":core") diff --git a/static-sqlite-driver/build.gradle.kts b/static-sqlite-driver/build.gradle.kts index f6c2cbc1..e4fe686d 100644 --- a/static-sqlite-driver/build.gradle.kts +++ b/static-sqlite-driver/build.gradle.kts @@ -1,39 +1,34 @@ -import com.powersync.compile.ClangCompile import com.powersync.compile.CreateSqliteCInterop -import com.powersync.compile.CreateStaticLibrary -import com.powersync.compile.UnzipSqlite -import java.io.File import com.powersync.plugins.utils.powersyncTargets -import de.undercouch.gradle.tasks.download.Download +import org.gradle.kotlin.dsl.register import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget import org.jetbrains.kotlin.konan.target.HostManager plugins { alias(libs.plugins.kotlinMultiplatform) - alias(libs.plugins.downloadPlugin) alias(libs.plugins.kotlinter) id("com.powersync.plugins.sonatype") } -val sqliteVersion = "3500300" -val sqliteReleaseYear = "2025" +val nativeSqliteConfiguration: Configuration by configurations.creating { + isCanBeConsumed = false +} -val downloadSQLiteSources by tasks.registering(Download::class) { - val zipFileName = "sqlite-amalgamation-$sqliteVersion.zip" - src("https://www.sqlite.org/$sqliteReleaseYear/$zipFileName") - dest(layout.buildDirectory.dir("downloads").map { it.file(zipFileName) }) - onlyIfNewer(true) - overwrite(false) +dependencies { + nativeSqliteConfiguration(project(path=":internal:prebuild-binaries", configuration="nativeSqliteConfiguration")) } -val unzipSQLiteSources by tasks.registering(UnzipSqlite::class) { - val zip = downloadSQLiteSources.map { it.outputs.files.singleFile } - inputs.file(zip) +fun linkSqliteCInterop(target: KotlinNativeTarget): TaskProvider { + val buildCInteropDef = tasks.register("${target.name}CinteropSqlite", CreateSqliteCInterop::class) { + val precompiledSqlite: FileCollection = nativeSqliteConfiguration + inputs.files(precompiledSqlite) + + val staticLibrary = precompiledSqlite.singleFile.resolve("${target.konanTarget.name}sqlite3.a") + archiveFile.set(staticLibrary) + definitionFile.value(layout.buildDirectory.map { it.file("interopDefs/${target.name}/sqlite3.def") }) + } - unzipSqlite( - src = zipTree(zip), - dir = layout.buildDirectory.dir("downloads/sqlite3") - ) + return buildCInteropDef } // Obtain host and platform manager from Kotlin multiplatform plugin. They're supposed to be @@ -41,38 +36,6 @@ val unzipSQLiteSources by tasks.registering(UnzipSqlite::class) { // use to compile SQLite for the platforms we need. val hostManager = HostManager() -fun compileSqlite(target: KotlinNativeTarget): TaskProvider { - val name = target.targetName - val outputDir = layout.buildDirectory.dir("c/$name") - - val sqlite3Obj = outputDir.map { it.file("sqlite3.o") } - val archive = outputDir.map { it.file("libsqlite3.a") } - - val compileSqlite = tasks.register("${name}CompileSqlite", ClangCompile::class) { - inputs.dir(unzipSQLiteSources.map { it.destination }) - - inputFile.set(unzipSQLiteSources.flatMap { it.destination.file("sqlite3.c") }) - konanTarget.set(target.konanTarget.name) - include.set(unzipSQLiteSources.flatMap { it.destination }) - objectFile.set(sqlite3Obj) - } - - val createStaticLibrary = tasks.register("${name}ArchiveSqlite", CreateStaticLibrary::class) { - inputs.file(compileSqlite.map { it.objectFile }) - objects.from(sqlite3Obj) - staticLibrary.set(archive) - } - - val buildCInteropDef = tasks.register("${name}CinteropSqlite", CreateSqliteCInterop::class) { - inputs.file(createStaticLibrary.map { it.staticLibrary }) - - archiveFile.set(archive) - definitionFile.fileProvider(archive.map { File(it.asFile.parentFile, "sqlite3.def") }) - } - - return buildCInteropDef -} - kotlin { // We use sqlite3-jdbc on JVM platforms instead powersyncTargets(jvm=false) @@ -92,11 +55,11 @@ kotlin { targets.withType { if (hostManager.isEnabled(konanTarget)) { - val compileSqlite3 = compileSqlite(this) + val interopSource = linkSqliteCInterop(this) compilations.named("main") { cinterops.create("sqlite3") { - definitionFile.set(compileSqlite3.flatMap { it.definitionFile }) + definitionFile.set(interopSource.flatMap { it.definitionFile }) } } }