diff --git a/.Rbuildignore b/.Rbuildignore index c92e40bb..8712a853 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,4 +15,5 @@ ^\.task$ ^src/\.cargo$ ^src/rust/vendor$ +^src/Makevars$ ^tools/libprqlr\.a$ diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ea31ebe7..5f465700 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -31,6 +31,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} @@ -41,15 +45,16 @@ jobs: fail-fast: false matrix: config: - - {os: macos-latest, r: 'release'} - - {os: windows-latest, r: 'release'} - - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - - {os: ubuntu-latest, r: 'release'} - - {os: ubuntu-latest, r: 'oldrel-1'} + - { os: macos-latest, r: "release" } + - { os: windows-latest, r: "release" } + - { os: ubuntu-latest, r: "devel", http-user-agent: "release" } + - { os: ubuntu-latest, r: "release" } + - { os: ubuntu-latest, r: "oldrel-1" } env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes + LIBPRQLR_BUILD: "true" steps: - uses: actions/checkout@v4 @@ -72,3 +77,56 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true + + source-with-bin-check: + runs-on: ${{ matrix.os }} + + name: ${{ matrix.os }} with pre-built binary (${{ matrix.r }}) + + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + - ubuntu-latest + r: + - release + + env: + NOT_CRAN: "true" + LIB_SUMS_PATH: "tools/lib-sums.tsv" + + steps: + - uses: actions/checkout@v4 + + - name: Check for pre-built binary + run: | + if [[ -f "${LIB_SUMS_PATH}" ]]; then + echo "TEST_BIN_LIB=true" >>"${GITHUB_ENV}" + rm -f "$(rustup which cargo)" + else + echo "TEST_BIN_LIB=false" >>"${GITHUB_ENV}" + fi + + - uses: r-lib/actions/setup-pandoc@v2 + if: env.TEST_BIN_LIB == 'true' + + - uses: r-lib/actions/setup-r@v2 + if: env.TEST_BIN_LIB == 'true' + with: + r-version: ${{ matrix.r }} + use-public-rspm: true + Ncpus: "2" + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck, any::devtools + needs: check + + - name: Install with pre-built binary + if: env.TEST_BIN_LIB == 'true' + shell: Rscript {0} + run: | + remotes::install_local() + devtools::test() diff --git a/.github/workflows/release-lib.yml b/.github/workflows/release-lib.yml index c3a1e3aa..0b80533d 100644 --- a/.github/workflows/release-lib.yml +++ b/.github/workflows/release-lib.yml @@ -82,7 +82,7 @@ jobs: run: | LIB_PATH="$(pwd)/rust/target/${TARGET}/${PRQLR_PROFILE}/${LIB_NAME}.a" ARTIFACT_NAME="${LIB_NAME}-${LIB_VERSION}-${TARGET}.tar.gz" - make -f Makevars${{ runner.os == 'Windows' && '.win' || '' }} "${LIB_PATH}" + make -f Makevars${{ runner.os == 'Windows' && '.win' || '.in' }} "${LIB_PATH}" tar -czf "../${ARTIFACT_NAME}" -C "rust/target/${TARGET}/${PRQLR_PROFILE}" "${LIB_NAME}.a" echo "ARTIFACT_NAME=${ARTIFACT_NAME}" >>"$GITHUB_ENV" diff --git a/.gitignore b/.gitignore index 973668af..c56bc3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ docs src/rust/vendor src/rust/vendor.tar.xz +src/Makevars tools/libprqlr.a diff --git a/DESCRIPTION b/DESCRIPTION index 62ca176a..36252555 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -31,7 +31,7 @@ Language: en-US Encoding: UTF-8 Roxygen: list(markdown = TRUE) RoxygenNote: 7.2.3 -SystemRequirements: GNU make, Cargo (Rust's package manager), rustc +SystemRequirements: Cargo (Rust's package manager), rustc VignetteBuilder: knitr Config/testthat/edition: 3 Config/rextendr/version: 0.3.1 @@ -45,3 +45,4 @@ Config/Needs/dev: Config/Needs/website: pkgdown, rextendr +Config/prqlr/LibVersion: 0.9.0 diff --git a/Taskfile.yml b/Taskfile.yml index 4ab0b2d1..4b4277ed 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,7 +1,8 @@ version: "3" env: - NOT_CRAN: true + NOT_CRAN: "true" + LIBPRQLR_BUILD: "true" vars: MANIFEST: src/rust/Cargo.toml @@ -53,9 +54,22 @@ tasks: cmds: - Rscript dev/vendoring.R + build-lib-sums: + desc: Build lib-sums.tsv. + sources: + - dev/generate-lib-sums.R + - DESCRIPTION + - "{{.CARGO_LOCK}}" + generates: + - tools/lib-sums.tsv + - tools/prep-lib.R + cmds: + - Rscript dev/generate-lib-sums.R + build-all: desc: Build the R package, generate documents, run all tests, and update files. deps: + - build-lib-sums - build-documents cmds: - task: test-all @@ -118,7 +132,7 @@ tasks: - Rscript -e 'devtools::load_all(); list.files("vignettes/", pattern = r"(\.Rmd$)", recursive = TRUE, full.names = TRUE) |> - purrr::walk(\(x) rmarkdown::render(x, output_dir = tempdir()))' + purrr::walk(\(x) rmarkdown::render(x, output_dir = tempdir()))' build-documents: desc: Build the R package and generate documents. diff --git a/cleanup b/cleanup new file mode 100755 index 00000000..ab72eb48 --- /dev/null +++ b/cleanup @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +rm -f src/Makevars diff --git a/configure b/configure index f0e0d5d0..8e59305e 100755 --- a/configure +++ b/configure @@ -1,10 +1,17 @@ #!/usr/bin/env sh +NOT_CRAN=${NOT_CRAN:-"false"} +LIBPRQLR_BUILD=${LIBPRQLR_BUILD:-""} + +LIBNAME="libprqlr.a" +LIBPRQLR_PATH="tools/${LIBNAME}" + export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { if [ ! "$(command -v cargo)" ]; then - echo "----------------------- [RUST NOT FOUND]---------------------------" + echo "" + echo "------------------------- [RUST NOT FOUND] -------------------------" echo "The 'cargo' command was not found on the PATH. Please install rustc" echo "from: https://www.rust-lang.org/tools/install" echo "" @@ -12,12 +19,51 @@ check_cargo() { echo " - Debian/Ubuntu: apt-get install cargo" echo " - Fedora/CentOS: dnf install cargo" echo " - macOS: brew install rustc" - echo "-------------------------------------------------------------------" + echo "--------------------------------------------------------------------" echo "" exit 1 + else + echo "" + echo "--------------------------- [RUST FOUND] ---------------------------" + "$(cargo -V)" + echo "" + "$(rustc -vV)" + echo "--------------------------------------------------------------------" + echo "" fi } +check_bin_lib() { + if [ "${NOT_CRAN}" = "true" ] && [ -z "${LIBPRQLR_BUILD}" ]; then + LIBPRQLR_BUILD="false" + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "${LIBPRQLR_PATH}" ] ; then + echo "Try to download pre-built binary..." + LIBPRQLR_PATH="tools/${LIBNAME}" + Rscript "tools/prep-lib.R" || echo "Failed to download pre-built binary..." + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then + echo "" + echo "------------------------- [LIBRARY FOUND] -------------------------" + echo "The library was found at <${LIBPRQLR_PATH}>. No need to build it." + echo "-------------------------------------------------------------------" + echo "" + sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars + exit 0 + elif [ "${LIBPRQLR_BUILD}" = "false" ]; then + echo "" + echo "----------------------- [LIBRARY NOT FOUND] -----------------------" + echo "The library was not found at <${LIBPRQLR_PATH}>." + echo "-------------------------------------------------------------------" + echo "" + fi +} + +check_bin_lib check_cargo +sed -e "s|@RUST_TARGET@|$(rustc -vV | grep host | cut -d' ' -f2)|" src/Makevars.in >src/Makevars + exit 0 diff --git a/configure.win b/configure.win index 5fda9c32..9ec3c13f 100755 --- a/configure.win +++ b/configure.win @@ -1,18 +1,62 @@ #!/bin/sh +NOT_CRAN=${NOT_CRAN:-"false"} +LIBPRQLR_BUILD=${LIBPRQLR_BUILD:-""} + +LIBNAME="libprqlr.a" +LIBPRQLR_PATH="tools/${LIBNAME}" + export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { if [ ! "$(command -v cargo)" ]; then - echo "----------------------- [RUST NOT FOUND]---------------------------" + echo "" + echo "------------------------- [RUST NOT FOUND] -------------------------" echo "The 'cargo' command was not found on the PATH. Please install rustc" echo "from: https://www.rust-lang.org/tools/install" - echo "-------------------------------------------------------------------" + echo "--------------------------------------------------------------------" echo "" exit 1 + else + echo "" + echo "--------------------------- [RUST FOUND] ---------------------------" + "$(cargo -V)" + echo "" + "$(rustc -vV)" + echo "--------------------------------------------------------------------" + echo "" + fi +} + +check_bin_lib() { + if [ "${NOT_CRAN}" = "true" ] && [ -z "${LIBPRQLR_BUILD}" ]; then + LIBPRQLR_BUILD="false" + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "${LIBPRQLR_PATH}" ] ; then + echo "Try to download pre-built binary..." + LIBPRQLR_PATH="tools/${LIBNAME}" + Rscript "tools/prep-lib.R" || echo "Failed to download pre-built binary..." + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then + echo "" + echo "------------------------- [LIBRARY FOUND] -------------------------" + echo "The library was found at <${LIBPRQLR_PATH}>. No need to build it." + echo "-------------------------------------------------------------------" + echo "" + sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars + exit 0 + elif [ "${LIBPRQLR_BUILD}" = "false" ]; then + echo "" + echo "----------------------- [LIBRARY NOT FOUND] -----------------------" + echo "The library was not found at <${LIBPRQLR_PATH}>." + echo "-------------------------------------------------------------------" + echo "" fi } +check_bin_lib check_cargo exit 0 diff --git a/dev/generate-lib-sums.R b/dev/generate-lib-sums.R new file mode 100644 index 00000000..fc711ded --- /dev/null +++ b/dev/generate-lib-sums.R @@ -0,0 +1,40 @@ +base_url <- "https://github.com/eitsupi/prqlr/releases/download/" + +tag_prefix <- "lib-v" + +lib_data_file_path <- file.path("tools", "lib-sums.tsv") + +package_name <- desc::desc_get("Package") +current_lib_version <- RcppTOML::parseTOML("src/rust/Cargo.toml")$package$version + +latest_released_lib_version <- gert::git_remote_ls(remote = "https://github.com/eitsupi/prqlr/") |> + dplyr::pull(ref) |> + stringr::str_subset(stringr::str_c(r"(^refs/tags/)", tag_prefix)) |> + stringr::str_remove(stringr::str_c(".*", tag_prefix)) |> + sort(decreasing = TRUE) |> + _[1] + +write_bin_lib_data <- function(path, sums_url, libs_base_url) { + df <- readr::read_table(sums_url, col_names = FALSE, show_col_types = FALSE) |> + dplyr::mutate( + url = glue::glue("{libs_base_url}{X2}"), + sha256sum = X1, + .keep = "none" + ) + + readr::write_tsv(df, path) +} + +desc::desc_set(paste0("Config/", package_name, "/LibVersion"), current_lib_version) + +if (identical(current_lib_version, latest_released_lib_version)) { + message("Current lib version is available via the binary release.") + write_bin_lib_data( + lib_data_file_path, + glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/sha256sums.txt"), + glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/") + ) +} else { + message("Current lib version is not available via binary releases.") + if (fs::file_exists(lib_data_file_path)) fs::file_delete(lib_data_file_path) +} diff --git a/src/Makevars b/src/Makevars.in similarity index 88% rename from src/Makevars rename to src/Makevars.in index 2dad261c..b36060ed 100644 --- a/src/Makevars +++ b/src/Makevars.in @@ -1,4 +1,4 @@ -TARGET ?= $(shell export PATH="$(PATH):$(HOME)/.cargo/bin" && rustc -vV | grep host | cut -d" " -f2) +TARGET ?= @RUST_TARGET@ PRQLR_PROFILE ?= release PRQLR_FEATURES ?= @@ -43,4 +43,4 @@ C_clean: rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" clean: - rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(CURDIR)/rust/target" + rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(TARGET_DIR)" diff --git a/src/Makevars.win b/src/Makevars.win index de891737..ae5dd797 100644 --- a/src/Makevars.win +++ b/src/Makevars.win @@ -57,4 +57,4 @@ C_clean: rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" clean: - rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(CURDIR)/rust/target" + rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(TARGET_DIR)" diff --git a/tools/lib-sums.tsv b/tools/lib-sums.tsv new file mode 100644 index 00000000..a0fe3836 --- /dev/null +++ b/tools/lib-sums.tsv @@ -0,0 +1,6 @@ +url sha256sum +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-aarch64-apple-darwin.tar.gz 882ffcc187e225d44d17c1c3bdcac3c27e7fa3b9577a26a4ab2f74677e87f0e8 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-aarch64-unknown-linux-musl.tar.gz e91a0e3db7a60e8b6b61f55dd061ec1d0429e82e749e0ce16a8af8830fa55eec +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-apple-darwin.tar.gz de46cb2e1a68dfca1a6a40116d176cdb850b0c63de0d8495b4453a1099dfcde1 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-pc-windows-gnu.tar.gz 4170bd99c98bc1b39610bdadeccc60a69e5aeaf00f348d2e760da19cb188dbe8 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-unknown-linux-musl.tar.gz 3a9fcdae0b4fdd42bc2e12476b44471c1704a6702770d0dd4e85dee7df38972f diff --git a/tools/prep-lib.R b/tools/prep-lib.R index e69de29b..e0450807 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -0,0 +1,87 @@ +check_sha256 <- function(file, sum, os = c("linux", "macos", "windows")) { + message("Checking SHA256 for <", file, ">...") + + if (match.arg(os) == "linux") { + out <- system2("sha256sum", args = file, stdout = TRUE) |> + gsub(r"(\s.*)", "", x = _) + } else if (match.arg(os) == "macos") { + out <- system2("shasum", args = c("-a", "256", file), stdout = TRUE) |> + gsub(r"(\s.*)", "", x = _) + } else if (match.arg(os) == "windows") { + out <- system2("certutil", args = c("-hashfile", file, "SHA256"), stdout = TRUE)[2] + } else { + stop("Unsupported OS: ", os) + } + + if (out != sum) { + stop("SHA256 mismatch for <", file, ">. Expected: <", sum, ">. Got: <", out, ">") + } + + message("SHA256 matches for <", file, ">.") + + invisible() +} + +which_os <- function() { + if (identical(.Platform$OS.type, "windows")) { + "windows" + } else if (Sys.info()["sysname"] == "Darwin") { + "macos" + } else if (Sys.info()["sysname"] == "Linux") { + "linux" + } else { + stop("Pre built binaries are not available for OS: ", R.version$os) + } +} + +current_os <- which_os() + +if (identical(current_os, "windows")) { + vendor_sys_abi <- "pc-windows-gnu" +} else if (identical(current_os, "macos")) { + vendor_sys_abi <- "apple-darwin" +} else if (identical(current_os, "linux")) { + vendor_sys_abi <- "unknown-linux-musl" +} + +if ((Sys.info()[["machine"]] %in% c("amd64", "x86_64", "x86-64"))) { + vendor_cpu_abi <- "x86_64" +} else if (Sys.info()[["machine"]] %in% c("arm64", "aarch64")) { + vendor_cpu_abi <- "aarch64" +} else { + stop("Pre built binaries are not available for Arch: ", Sys.info()[["machine"]]) +} + +target_triple <- paste0(vendor_cpu_abi, "-", vendor_sys_abi) + +lib_data <- utils::read.table("tools/lib-sums.tsv", header = TRUE, stringsAsFactors = FALSE) + +package_name <- read.dcf("DESCRIPTION", fields = "Package", all = TRUE) +lib_version <- read.dcf("DESCRIPTION", fields = sprintf("Config/%s/LibVersion", package_name), all = TRUE) +lib_tag_prefix <- "lib-v" + +target_url <- sprintf( + "https://github.com/eitsupi/prqlr/releases/download/%s%s/libprqlr-%s-%s.tar.gz", + lib_tag_prefix, + lib_version, + lib_version, + target_triple +) + +lib_sum <- lib_data |> + subset(url == target_url) |> + (\(x) x$sha256sum)() + +if (!length(lib_sum)) stop("No pre built binary found at <", target_url, ">") + +message("Found pre built binary at <", target_url, ">.\nDownloading...") + +destfile <- tempfile(fileext = ".tar.gz") +on.exit(unlink(destfile)) + +utils::download.file(target_url, destfile, quiet = TRUE, mode = "wb") +check_sha256(destfile, lib_sum, os = current_os) + +utils::untar(destfile, exdir = "tools") + +message("Extracted pre built binary to directory.")