From d49dc2c16a4ffddab652275ed6692dc26c379522 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Thu, 28 Sep 2023 15:26:30 +0000 Subject: [PATCH 01/19] WIP feat(installation): download lib binaries --- .Rbuildignore | 1 + .gitignore | 1 + DESCRIPTION | 3 ++- cleanup | 3 +++ configure | 27 +++++++++++++++++++++++ dev/config-lib.R | 40 +++++++++++++++++++++++++++++++++++ src/{Makevars => Makevars.in} | 2 +- tools/lib-sums.tsv | 6 ++++++ tools/prep-lib.R | 21 ++++++++++++++++++ 9 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 cleanup create mode 100644 dev/config-lib.R rename src/{Makevars => Makevars.in} (92%) create mode 100644 tools/lib-sums.tsv diff --git a/.Rbuildignore b/.Rbuildignore index c92e40bb..8712a853 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,4 +15,5 @@ ^\.task$ ^src/\.cargo$ ^src/rust/vendor$ +^src/Makevars$ ^tools/libprqlr\.a$ diff --git a/.gitignore b/.gitignore index 973668af..c56bc3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ docs src/rust/vendor src/rust/vendor.tar.xz +src/Makevars tools/libprqlr.a diff --git a/DESCRIPTION b/DESCRIPTION index 62ca176a..d75554c1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -31,7 +31,7 @@ Language: en-US Encoding: UTF-8 Roxygen: list(markdown = TRUE) RoxygenNote: 7.2.3 -SystemRequirements: GNU make, Cargo (Rust's package manager), rustc +SystemRequirements: Cargo (Rust's package manager), rustc VignetteBuilder: knitr Config/testthat/edition: 3 Config/rextendr/version: 0.3.1 @@ -45,3 +45,4 @@ Config/Needs/dev: Config/Needs/website: pkgdown, rextendr +Config/prqlr/LibVersion: 0.1.0 diff --git a/cleanup b/cleanup new file mode 100644 index 00000000..ab72eb48 --- /dev/null +++ b/cleanup @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +rm -f src/Makevars diff --git a/configure b/configure index f0e0d5d0..5c2fc3c1 100755 --- a/configure +++ b/configure @@ -1,5 +1,9 @@ #!/usr/bin/env sh +NOT_CRAN=${NOT_CRAN:-"false"} +LIBPRQLR_BUILD=${LIBPRQLR_BUILD:-""} +LIBNAME="libprqlr.a" + export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { @@ -18,6 +22,29 @@ check_cargo() { fi } +check_bin_lib() { + if [ "${NOT_CRAN}" = "true" ] && [ -z "${LIBPRQLR_BUILD}" ]; then + LIBPRQLR_BUILD="false" + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "tools/${LIBNAME}" ] ; then + echo "Try to download pre-built binary..." + Rscript "tools/prep-lib.R" || echo "Failed to download pre-built binary..." + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/${LIBNAME}" ]; then + echo "----------------------- [LIBRARY FOUND]---------------------------" + echo "The library was found in the tools directory. No need to build it." + echo "-------------------------------------------------------------------" + echo "" + sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars + exit 0 + fi +} + +check_bin_lib check_cargo +sed -e "s|@RUST_TARGET@|$(rustc -vV | grep host | cut -d' ' -f2)|" src/Makevars.in >src/Makevars + exit 0 diff --git a/dev/config-lib.R b/dev/config-lib.R new file mode 100644 index 00000000..648addfe --- /dev/null +++ b/dev/config-lib.R @@ -0,0 +1,40 @@ +base_url <- "https://github.com/eitsupi/prqlr/releases/download/" + +tag_prefix <- "lib-" + +lib_data_file_path <- file.path("tools", "lib-sums.tsv") + +package_name <- desc::desc_get("Package") +current_lib_version <- RcppTOML::parseTOML("src/rust/Cargo.toml")$package$version + +latest_released_lib_version <- gert::git_remote_ls(remote = "https://github.com/eitsupi/prqlr/") |> + dplyr::pull(ref) |> + stringr::str_subset(stringr::str_c(r"(^refs/tags/)", tag_prefix)) |> + stringr::str_remove(stringr::str_c(".*", tag_prefix)) |> + sort(decreasing = TRUE) |> + _[1] + +write_bin_lib_data <- function(path, sums_url, libs_base_url) { + df <- readr::read_table(sums_url, col_names = FALSE, show_col_types = FALSE) |> + dplyr::mutate( + url = glue::glue("{libs_base_url}{X2}"), + sha256sum = X1, + .keep = "none" + ) + + readr::write_tsv(df, path) +} + +desc::desc_set(paste0("Config/", package_name, "/LibVersion"), current_lib_version) + +if (identical(current_lib_version, latest_released_lib_version)) { + message("Current lib version is available via the binary release.") + write_bin_lib_data( + lib_data_file_path, + glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/sha256sums.txt"), + glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/") + ) +} else { + message("Current lib version is not available via binary releases.") + fs::file_delete(lib_data_file_path) +} diff --git a/src/Makevars b/src/Makevars.in similarity index 92% rename from src/Makevars rename to src/Makevars.in index 2dad261c..dd7495af 100644 --- a/src/Makevars +++ b/src/Makevars.in @@ -1,4 +1,4 @@ -TARGET ?= $(shell export PATH="$(PATH):$(HOME)/.cargo/bin" && rustc -vV | grep host | cut -d" " -f2) +TARGET ?= @RUST_TARGET@ PRQLR_PROFILE ?= release PRQLR_FEATURES ?= diff --git a/tools/lib-sums.tsv b/tools/lib-sums.tsv new file mode 100644 index 00000000..787127d1 --- /dev/null +++ b/tools/lib-sums.tsv @@ -0,0 +1,6 @@ +url sha256sum +https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-aarch64-apple-darwin.tar.gz 2b4a7d38fab3dc56d4502aa54adf5bfe9480f14bf2b9a2bc6d952d5c5f15285b +https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-aarch64-unknown-linux-musl.tar.gz a81a8643eb6dce90c249dbd704759759d54a295b38cc23525e8b1aa1da9bd570 +https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-apple-darwin.tar.gz e49045b9418083a2a7a1cb0fc5f301e6d630618a26e048948ad6cc039043c4f6 +https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-pc-windows-gnu.tar.gz c0e97ac0424b1700088e52878ca599c5c241344646526755335590867725bd3e +https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-unknown-linux-musl.tar.gz 6a25a5c7ea446dce24683fbc64845c1ffb51badff76b05e1b02bd28221c35bfb diff --git a/tools/prep-lib.R b/tools/prep-lib.R index e69de29b..04be176b 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -0,0 +1,21 @@ +if (identical(.Platform$OS.type, "windows")) { + vendor_sys_abi <- "pc-windows-gnu" +} else if (grepl("^darwin", R.version$os)) { + vendor_sys_abi <- "apple-darwin" +} else if (identical(R.version$os, "linux-gnu")) { + vendor_sys_abi <- "unknown-linux-musl" +} else { + stop("Pre built binaries are not available for OS: ", R.version$os) +} + +if ((R.version$arch %in% c("amd64", "x86_64"))) { + vendor_cpu_abi <- "x86_64" +} else if (R.version$arch %in% c("arm64", "aarch64")) { + vendor_cpu_abi <- "aarch64" +} else { + stop("Pre built binaries are not available for Arch: ", R.version$arch) +} + +target_triple <- paste0(vendor_cpu_abi, "-", vendor_sys_abi) + +lib_data <- utils::read.table("tools/lib-sums.tsv", header = TRUE, stringsAsFactors = FALSE) From d5c94e3a6c14dfde2fa97bb2781c9f21c42f2f53 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Thu, 28 Sep 2023 15:37:24 +0000 Subject: [PATCH 02/19] fix: make executable --- cleanup | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cleanup diff --git a/cleanup b/cleanup old mode 100644 new mode 100755 From 2a3c2e2789cda17d48e6c11c07e6c0f75d87b494 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Thu, 28 Sep 2023 22:44:24 +0000 Subject: [PATCH 03/19] chore: revert not to use Makevars.in --- .Rbuildignore | 1 - .gitignore | 1 - cleanup | 3 --- configure | 3 --- src/{Makevars.in => Makevars} | 2 +- 5 files changed, 1 insertion(+), 9 deletions(-) delete mode 100755 cleanup rename src/{Makevars.in => Makevars} (98%) diff --git a/.Rbuildignore b/.Rbuildignore index 8712a853..c92e40bb 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,5 +15,4 @@ ^\.task$ ^src/\.cargo$ ^src/rust/vendor$ -^src/Makevars$ ^tools/libprqlr\.a$ diff --git a/.gitignore b/.gitignore index c56bc3fa..973668af 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,4 @@ docs src/rust/vendor src/rust/vendor.tar.xz -src/Makevars tools/libprqlr.a diff --git a/cleanup b/cleanup deleted file mode 100755 index ab72eb48..00000000 --- a/cleanup +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -rm -f src/Makevars diff --git a/configure b/configure index 5c2fc3c1..83491018 100755 --- a/configure +++ b/configure @@ -37,7 +37,6 @@ check_bin_lib() { echo "The library was found in the tools directory. No need to build it." echo "-------------------------------------------------------------------" echo "" - sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars exit 0 fi } @@ -45,6 +44,4 @@ check_bin_lib() { check_bin_lib check_cargo -sed -e "s|@RUST_TARGET@|$(rustc -vV | grep host | cut -d' ' -f2)|" src/Makevars.in >src/Makevars - exit 0 diff --git a/src/Makevars.in b/src/Makevars similarity index 98% rename from src/Makevars.in rename to src/Makevars index dd7495af..cf4dbd5b 100644 --- a/src/Makevars.in +++ b/src/Makevars @@ -1,4 +1,4 @@ -TARGET ?= @RUST_TARGET@ +TARGET ?= PRQLR_PROFILE ?= release PRQLR_FEATURES ?= From 5613266f5c4189ab5a4104a730ddf4fd97ede6b5 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Thu, 28 Sep 2023 22:57:37 +0000 Subject: [PATCH 04/19] Revert "chore: revert not to use Makevars.in" This reverts commit 2a3c2e2789cda17d48e6c11c07e6c0f75d87b494. --- .Rbuildignore | 1 + .gitignore | 1 + cleanup | 3 +++ configure | 3 +++ src/{Makevars => Makevars.in} | 2 +- 5 files changed, 9 insertions(+), 1 deletion(-) create mode 100755 cleanup rename src/{Makevars => Makevars.in} (98%) diff --git a/.Rbuildignore b/.Rbuildignore index c92e40bb..8712a853 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,4 +15,5 @@ ^\.task$ ^src/\.cargo$ ^src/rust/vendor$ +^src/Makevars$ ^tools/libprqlr\.a$ diff --git a/.gitignore b/.gitignore index 973668af..c56bc3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ docs src/rust/vendor src/rust/vendor.tar.xz +src/Makevars tools/libprqlr.a diff --git a/cleanup b/cleanup new file mode 100755 index 00000000..ab72eb48 --- /dev/null +++ b/cleanup @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +rm -f src/Makevars diff --git a/configure b/configure index 83491018..5c2fc3c1 100755 --- a/configure +++ b/configure @@ -37,6 +37,7 @@ check_bin_lib() { echo "The library was found in the tools directory. No need to build it." echo "-------------------------------------------------------------------" echo "" + sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars exit 0 fi } @@ -44,4 +45,6 @@ check_bin_lib() { check_bin_lib check_cargo +sed -e "s|@RUST_TARGET@|$(rustc -vV | grep host | cut -d' ' -f2)|" src/Makevars.in >src/Makevars + exit 0 diff --git a/src/Makevars b/src/Makevars.in similarity index 98% rename from src/Makevars rename to src/Makevars.in index cf4dbd5b..dd7495af 100644 --- a/src/Makevars +++ b/src/Makevars.in @@ -1,4 +1,4 @@ -TARGET ?= +TARGET ?= @RUST_TARGET@ PRQLR_PROFILE ?= release PRQLR_FEATURES ?= From 156f1eefde8fa8689ef2e4bc0c08060f69ca0710 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Fri, 29 Sep 2023 10:25:19 +0000 Subject: [PATCH 05/19] chore: refactor --- src/Makevars.in | 2 +- src/Makevars.win | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makevars.in b/src/Makevars.in index dd7495af..b36060ed 100644 --- a/src/Makevars.in +++ b/src/Makevars.in @@ -43,4 +43,4 @@ C_clean: rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" clean: - rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(CURDIR)/rust/target" + rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(TARGET_DIR)" diff --git a/src/Makevars.win b/src/Makevars.win index de891737..ae5dd797 100644 --- a/src/Makevars.win +++ b/src/Makevars.win @@ -57,4 +57,4 @@ C_clean: rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" clean: - rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(CURDIR)/rust/target" + rm -Rf "$(SHLIB)" "$(STATLIB)" "$(OBJECTS)" "$(TARGET_DIR)" From 2d2382c0cad07a13090884b766816b666c5ffdb9 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Fri, 29 Sep 2023 10:52:35 +0000 Subject: [PATCH 06/19] feat: download pre built binary --- tools/prep-lib.R | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/prep-lib.R b/tools/prep-lib.R index 04be176b..0b769ce8 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -19,3 +19,30 @@ if ((R.version$arch %in% c("amd64", "x86_64"))) { target_triple <- paste0(vendor_cpu_abi, "-", vendor_sys_abi) lib_data <- utils::read.table("tools/lib-sums.tsv", header = TRUE, stringsAsFactors = FALSE) + +package_name <- read.dcf("DESCRIPTION", fields = "Package", all = TRUE) +lib_version <- read.dcf("DESCRIPTION", fields = sprintf("Config/%s/LibVersion", package_name), all = TRUE) +lib_tag_prefix <- "lib-" + +target_url <- sprintf( + "https://github.com/eitsupi/prqlr/releases/download/%s%s/libprqlr-%s-%s.tar.gz", + lib_tag_prefix, + lib_version, + lib_version, + target_triple +) + +lib_sum <- lib_data |> + subset(url == target_url) |> + _$sha256sum + +if (!length(lib_sum)) stop("No pre built binary found at <", target_url, ">") + +message("Found pre built binary at <", target_url, ">. Downloading...") + +destfile <- tempfile(fileext = ".tar.gz") + +utils::download.file(target_url, destfile, quiet = TRUE, mode = "wb") +outfile <- utils::untar(destfile, exdir = "tools", list = TRUE) + +message("Extracted pre built binary to <", file.path("tools", outfile), ">") From 4f2c2203e8f888a3cc9ad1bc25eef762c9aa377b Mon Sep 17 00:00:00 2001 From: eitsupi Date: Fri, 29 Sep 2023 11:27:24 +0000 Subject: [PATCH 07/19] feat: checksum --- tools/prep-lib.R | 56 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/tools/prep-lib.R b/tools/prep-lib.R index 0b769ce8..f5a2b22e 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -1,11 +1,48 @@ -if (identical(.Platform$OS.type, "windows")) { +check_sha256 <- function(file, sum, os = c("linux", "macos", "windows")) { + message("Checking SHA256 hash to <", sum, "> for <", file, ">...") + + if (match.arg(os) == "linux") { + out <- system2("sha256sum", args = file, stdout = TRUE) |> + gsub(r"(\s.*)", "", x = _) + } else if (match.arg(os) == "macos") { + out <- system2("shasum", args = c("-a", "256", file), stdout = TRUE) |> + gsub(r"(\s.*)", "", x = _) + } else if (match.arg(os) == "windows") { + out <- system2("certutil", args = c("-hashfile", file, "SHA256"), stdout = TRUE) |> + _[2] + } else { + stop("Unsupported OS: ", os) + } + + if (out != sum) { + stop("SHA256 mismatch for <", file, ">. Expected: <", sum, ">. Got: <", out, ">") + } + + message("SHA256 matches for <", file, ">.") + + invisible() +} + +which_os <- function() { + if (identical(.Platform$OS.type, "windows")) { + "windows" + } else if (grepl("^darwin", R.version$os)) { + "macos" + } else if (identical(R.version$os, "linux-gnu")) { + "linux" + } else { + stop("Pre built binaries are not available for OS: ", R.version$os) + } +} + +current_os <- which_os() + +if (identical(current_os, "windows")) { vendor_sys_abi <- "pc-windows-gnu" -} else if (grepl("^darwin", R.version$os)) { +} else if (identical(current_os, "macos")) { vendor_sys_abi <- "apple-darwin" -} else if (identical(R.version$os, "linux-gnu")) { +} else if (identical(current_os, "linux")) { vendor_sys_abi <- "unknown-linux-musl" -} else { - stop("Pre built binaries are not available for OS: ", R.version$os) } if ((R.version$arch %in% c("amd64", "x86_64"))) { @@ -38,11 +75,14 @@ lib_sum <- lib_data |> if (!length(lib_sum)) stop("No pre built binary found at <", target_url, ">") -message("Found pre built binary at <", target_url, ">. Downloading...") +message("Found pre built binary at <", target_url, ">.\nDownloading...") destfile <- tempfile(fileext = ".tar.gz") +on.exit(unlink(destfile)) utils::download.file(target_url, destfile, quiet = TRUE, mode = "wb") -outfile <- utils::untar(destfile, exdir = "tools", list = TRUE) +check_sha256(destfile, lib_sum, os = current_os) + +utils::untar(destfile, exdir = "tools") -message("Extracted pre built binary to <", file.path("tools", outfile), ">") +message("Extracted pre built binary to directory.") From 3c99bf09f68070f08f86eed05e18283a3bf4c897 Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sat, 30 Sep 2023 00:42:12 +0000 Subject: [PATCH 08/19] chore: refactoring --- tools/prep-lib.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/prep-lib.R b/tools/prep-lib.R index f5a2b22e..0a7eac9d 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -26,9 +26,9 @@ check_sha256 <- function(file, sum, os = c("linux", "macos", "windows")) { which_os <- function() { if (identical(.Platform$OS.type, "windows")) { "windows" - } else if (grepl("^darwin", R.version$os)) { + } else if (Sys.info()["sysname"] == "Darwin") { "macos" - } else if (identical(R.version$os, "linux-gnu")) { + } else if (Sys.info()["sysname"] == "Linux") { "linux" } else { stop("Pre built binaries are not available for OS: ", R.version$os) @@ -45,12 +45,12 @@ if (identical(current_os, "windows")) { vendor_sys_abi <- "unknown-linux-musl" } -if ((R.version$arch %in% c("amd64", "x86_64"))) { +if ((Sys.info()[["machine"]] %in% c("amd64", "x86_64"))) { vendor_cpu_abi <- "x86_64" -} else if (R.version$arch %in% c("arm64", "aarch64")) { +} else if (Sys.info()[["machine"]] %in% c("arm64", "aarch64")) { vendor_cpu_abi <- "aarch64" } else { - stop("Pre built binaries are not available for Arch: ", R.version$arch) + stop("Pre built binaries are not available for Arch: ", Sys.info()[["machine"]]) } target_triple <- paste0(vendor_cpu_abi, "-", vendor_sys_abi) From 25bcb1cca78a22400b529e3b84ea828469291997 Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sat, 30 Sep 2023 00:44:55 +0000 Subject: [PATCH 09/19] fix: support R 4.2 syntax --- tools/prep-lib.R | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/prep-lib.R b/tools/prep-lib.R index 0a7eac9d..b5ee37e1 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -8,8 +8,7 @@ check_sha256 <- function(file, sum, os = c("linux", "macos", "windows")) { out <- system2("shasum", args = c("-a", "256", file), stdout = TRUE) |> gsub(r"(\s.*)", "", x = _) } else if (match.arg(os) == "windows") { - out <- system2("certutil", args = c("-hashfile", file, "SHA256"), stdout = TRUE) |> - _[2] + out <- system2("certutil", args = c("-hashfile", file, "SHA256"), stdout = TRUE)[2] } else { stop("Unsupported OS: ", os) } @@ -71,7 +70,7 @@ target_url <- sprintf( lib_sum <- lib_data |> subset(url == target_url) |> - _$sha256sum + (\(x) x$sha256sum)() if (!length(lib_sum)) stop("No pre built binary found at <", target_url, ">") From b6cac30c266e1171bae1c031413a859e21686eed Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sat, 30 Sep 2023 00:50:17 +0000 Subject: [PATCH 10/19] chore: tweak messages --- configure | 12 ++++++++---- tools/prep-lib.R | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 5c2fc3c1..da2d3cd5 100755 --- a/configure +++ b/configure @@ -2,7 +2,9 @@ NOT_CRAN=${NOT_CRAN:-"false"} LIBPRQLR_BUILD=${LIBPRQLR_BUILD:-""} + LIBNAME="libprqlr.a" +LIBPRQLR_PATH="tools/${LIBNAME}" export PATH="$PATH:$HOME/.cargo/bin" @@ -27,14 +29,16 @@ check_bin_lib() { LIBPRQLR_BUILD="false" fi - if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "tools/${LIBNAME}" ] ; then + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "${LIBPRQLR_PATH}" ] ; then echo "Try to download pre-built binary..." + LIBPRQLR_PATH="tools/${LIBNAME}" Rscript "tools/prep-lib.R" || echo "Failed to download pre-built binary..." fi - if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/${LIBNAME}" ]; then - echo "----------------------- [LIBRARY FOUND]---------------------------" - echo "The library was found in the tools directory. No need to build it." + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then + echo "" + echo "----------------------- [LIBRARY FOUND]----------------------------" + echo "The library was found at ${LIBPRQLR_PATH}. No need to build it." echo "-------------------------------------------------------------------" echo "" sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars diff --git a/tools/prep-lib.R b/tools/prep-lib.R index b5ee37e1..ed7c763b 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -1,5 +1,5 @@ check_sha256 <- function(file, sum, os = c("linux", "macos", "windows")) { - message("Checking SHA256 hash to <", sum, "> for <", file, ">...") + message("Checking SHA256 for <", file, ">...") if (match.arg(os) == "linux") { out <- system2("sha256sum", args = file, stdout = TRUE) |> From f1b63568eaf491680bd3fb8a1c3133b4ae7cb9b5 Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sat, 30 Sep 2023 00:53:19 +0000 Subject: [PATCH 11/19] chore: tweak messages and update for windows --- configure | 6 ++++++ configure.win | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/configure b/configure index da2d3cd5..e5f359e4 100755 --- a/configure +++ b/configure @@ -43,6 +43,12 @@ check_bin_lib() { echo "" sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars exit 0 + else + echo "" + echo "----------------------- [LIBRARY NOT FOUND]------------------------" + echo "The library was not found at ${LIBPRQLR_PATH}." + echo "-------------------------------------------------------------------" + echo "" fi } diff --git a/configure.win b/configure.win index 5fda9c32..e8b7b31c 100755 --- a/configure.win +++ b/configure.win @@ -1,5 +1,11 @@ #!/bin/sh +NOT_CRAN=${NOT_CRAN:-"false"} +LIBPRQLR_BUILD=${LIBPRQLR_BUILD:-""} + +LIBNAME="libprqlr.a" +LIBPRQLR_PATH="tools/${LIBNAME}" + export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { @@ -13,6 +19,35 @@ check_cargo() { fi } +check_bin_lib() { + if [ "${NOT_CRAN}" = "true" ] && [ -z "${LIBPRQLR_BUILD}" ]; then + LIBPRQLR_BUILD="false" + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "${LIBPRQLR_PATH}" ] ; then + echo "Try to download pre-built binary..." + LIBPRQLR_PATH="tools/${LIBNAME}" + Rscript "tools/prep-lib.R" || echo "Failed to download pre-built binary..." + fi + + if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then + echo "" + echo "----------------------- [LIBRARY FOUND]----------------------------" + echo "The library was found at ${LIBPRQLR_PATH}. No need to build it." + echo "-------------------------------------------------------------------" + echo "" + sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars + exit 0 + else + echo "" + echo "----------------------- [LIBRARY NOT FOUND]------------------------" + echo "The library was not found at ${LIBPRQLR_PATH}." + echo "-------------------------------------------------------------------" + echo "" + fi +} + +check_bin_lib check_cargo exit 0 From 01bc636fc9e871a4e049af48d87373c2ac4a7eb9 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 12:49:01 +0000 Subject: [PATCH 12/19] fix: update tag name pattern --- dev/config-lib.R | 2 +- tools/prep-lib.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/config-lib.R b/dev/config-lib.R index 648addfe..677bddcc 100644 --- a/dev/config-lib.R +++ b/dev/config-lib.R @@ -1,6 +1,6 @@ base_url <- "https://github.com/eitsupi/prqlr/releases/download/" -tag_prefix <- "lib-" +tag_prefix <- "lib-v" lib_data_file_path <- file.path("tools", "lib-sums.tsv") diff --git a/tools/prep-lib.R b/tools/prep-lib.R index ed7c763b..643c4dcc 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -58,7 +58,7 @@ lib_data <- utils::read.table("tools/lib-sums.tsv", header = TRUE, stringsAsFact package_name <- read.dcf("DESCRIPTION", fields = "Package", all = TRUE) lib_version <- read.dcf("DESCRIPTION", fields = sprintf("Config/%s/LibVersion", package_name), all = TRUE) -lib_tag_prefix <- "lib-" +lib_tag_prefix <- "lib-v" target_url <- sprintf( "https://github.com/eitsupi/prqlr/releases/download/%s%s/libprqlr-%s-%s.tar.gz", From 768d181fe396c25e35cee23948b91941455417fb Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 13:49:11 +0000 Subject: [PATCH 13/19] ci: add tests for installing with bin lib --- .github/workflows/check.yml | 65 ++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ea31ebe7..ea73baba 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -31,6 +31,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} @@ -41,15 +45,16 @@ jobs: fail-fast: false matrix: config: - - {os: macos-latest, r: 'release'} - - {os: windows-latest, r: 'release'} - - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - - {os: ubuntu-latest, r: 'release'} - - {os: ubuntu-latest, r: 'oldrel-1'} + - { os: macos-latest, r: "release" } + - { os: windows-latest, r: "release" } + - { os: ubuntu-latest, r: "devel", http-user-agent: "release" } + - { os: ubuntu-latest, r: "release" } + - { os: ubuntu-latest, r: "oldrel-1" } env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes + LIBPRQLR_BUILD: "true" steps: - uses: actions/checkout@v4 @@ -72,3 +77,53 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true + + source-with-bin-check: + runs-on: ${{ matrix.os }} + + name: ${{ matrix.os }} with pre-built binary (${{ matrix.r }}) + + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + - ubuntu-latest + r: + - release + + env: + NOT_CRAN: "true" + LIB_SUMS_PATH: "tools/lib-sums.tsv" + + steps: + - uses: actions/checkout@v4 + + - name: Check for pre-built binary + run: | + if [[ -f "${LIB_SUMS_PATH}" ]]; then + echo "TEST_BIN_LIB=true" >>"${GITHUB_ENV}" + rm -f "$(rustup which cargo)" + else + echo "TEST_BIN_LIB=false" >>"${GITHUB_ENV}" + fi + + - uses: r-lib/actions/setup-pandoc@v2 + if: env.TEST_BIN_LIB == 'true' + + - uses: r-lib/actions/setup-r@v2 + if: env.TEST_BIN_LIB == 'true' + with: + r-version: ${{ matrix.r }} + use-public-rspm: true + Ncpus: "2" + + - name: Install with pre-built binary + if: env.TEST_BIN_LIB == 'true' + shell: Rscript {0} + run: | + install.packages("pak") + pak::pak(c("devtools", "remotes")) + remotes::install_local() + devtools::test() From ce37290ec43b2cbed8e2fbd43355bfef4fb5e115 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 13:59:18 +0000 Subject: [PATCH 14/19] fix: fix to detect amd64 --- tools/prep-lib.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/prep-lib.R b/tools/prep-lib.R index 643c4dcc..e0450807 100644 --- a/tools/prep-lib.R +++ b/tools/prep-lib.R @@ -44,7 +44,7 @@ if (identical(current_os, "windows")) { vendor_sys_abi <- "unknown-linux-musl" } -if ((Sys.info()[["machine"]] %in% c("amd64", "x86_64"))) { +if ((Sys.info()[["machine"]] %in% c("amd64", "x86_64", "x86-64"))) { vendor_cpu_abi <- "x86_64" } else if (Sys.info()[["machine"]] %in% c("arm64", "aarch64")) { vendor_cpu_abi <- "aarch64" From 3811fcedb7f550ebc6b9f278aa48b4efe78b3305 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 14:35:48 +0000 Subject: [PATCH 15/19] build: add the generate-lib-sums task to the Taskfile --- Taskfile.yml | 18 ++++++++++++++++-- dev/{config-lib.R => generate-lib-sums.R} | 2 +- tools/lib-sums.tsv | 6 ------ 3 files changed, 17 insertions(+), 9 deletions(-) rename dev/{config-lib.R => generate-lib-sums.R} (94%) delete mode 100644 tools/lib-sums.tsv diff --git a/Taskfile.yml b/Taskfile.yml index 4ab0b2d1..4b4277ed 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,7 +1,8 @@ version: "3" env: - NOT_CRAN: true + NOT_CRAN: "true" + LIBPRQLR_BUILD: "true" vars: MANIFEST: src/rust/Cargo.toml @@ -53,9 +54,22 @@ tasks: cmds: - Rscript dev/vendoring.R + build-lib-sums: + desc: Build lib-sums.tsv. + sources: + - dev/generate-lib-sums.R + - DESCRIPTION + - "{{.CARGO_LOCK}}" + generates: + - tools/lib-sums.tsv + - tools/prep-lib.R + cmds: + - Rscript dev/generate-lib-sums.R + build-all: desc: Build the R package, generate documents, run all tests, and update files. deps: + - build-lib-sums - build-documents cmds: - task: test-all @@ -118,7 +132,7 @@ tasks: - Rscript -e 'devtools::load_all(); list.files("vignettes/", pattern = r"(\.Rmd$)", recursive = TRUE, full.names = TRUE) |> - purrr::walk(\(x) rmarkdown::render(x, output_dir = tempdir()))' + purrr::walk(\(x) rmarkdown::render(x, output_dir = tempdir()))' build-documents: desc: Build the R package and generate documents. diff --git a/dev/config-lib.R b/dev/generate-lib-sums.R similarity index 94% rename from dev/config-lib.R rename to dev/generate-lib-sums.R index 677bddcc..fc711ded 100644 --- a/dev/config-lib.R +++ b/dev/generate-lib-sums.R @@ -36,5 +36,5 @@ if (identical(current_lib_version, latest_released_lib_version)) { ) } else { message("Current lib version is not available via binary releases.") - fs::file_delete(lib_data_file_path) + if (fs::file_exists(lib_data_file_path)) fs::file_delete(lib_data_file_path) } diff --git a/tools/lib-sums.tsv b/tools/lib-sums.tsv deleted file mode 100644 index 787127d1..00000000 --- a/tools/lib-sums.tsv +++ /dev/null @@ -1,6 +0,0 @@ -url sha256sum -https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-aarch64-apple-darwin.tar.gz 2b4a7d38fab3dc56d4502aa54adf5bfe9480f14bf2b9a2bc6d952d5c5f15285b -https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-aarch64-unknown-linux-musl.tar.gz a81a8643eb6dce90c249dbd704759759d54a295b38cc23525e8b1aa1da9bd570 -https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-apple-darwin.tar.gz e49045b9418083a2a7a1cb0fc5f301e6d630618a26e048948ad6cc039043c4f6 -https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-pc-windows-gnu.tar.gz c0e97ac0424b1700088e52878ca599c5c241344646526755335590867725bd3e -https://github.com/eitsupi/prqlr/releases/download/lib-0.1.0/libprqlr-0.1.0-x86_64-unknown-linux-musl.tar.gz 6a25a5c7ea446dce24683fbc64845c1ffb51badff76b05e1b02bd28221c35bfb From 31f342822aaf7b1dd5acaf31802227468f439295 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 14:47:58 +0000 Subject: [PATCH 16/19] ci: update the Makevars path to Makevars.in --- .github/workflows/release-lib.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-lib.yml b/.github/workflows/release-lib.yml index c3a1e3aa..0b80533d 100644 --- a/.github/workflows/release-lib.yml +++ b/.github/workflows/release-lib.yml @@ -82,7 +82,7 @@ jobs: run: | LIB_PATH="$(pwd)/rust/target/${TARGET}/${PRQLR_PROFILE}/${LIB_NAME}.a" ARTIFACT_NAME="${LIB_NAME}-${LIB_VERSION}-${TARGET}.tar.gz" - make -f Makevars${{ runner.os == 'Windows' && '.win' || '' }} "${LIB_PATH}" + make -f Makevars${{ runner.os == 'Windows' && '.win' || '.in' }} "${LIB_PATH}" tar -czf "../${ARTIFACT_NAME}" -C "rust/target/${TARGET}/${PRQLR_PROFILE}" "${LIB_NAME}.a" echo "ARTIFACT_NAME=${ARTIFACT_NAME}" >>"$GITHUB_ENV" From ab55b30b00e85dbd09382b5eb4fe89bbaabf6173 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 15:33:12 +0000 Subject: [PATCH 17/19] chore: update lib-sums --- DESCRIPTION | 2 +- tools/lib-sums.tsv | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tools/lib-sums.tsv diff --git a/DESCRIPTION b/DESCRIPTION index d75554c1..36252555 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -45,4 +45,4 @@ Config/Needs/dev: Config/Needs/website: pkgdown, rextendr -Config/prqlr/LibVersion: 0.1.0 +Config/prqlr/LibVersion: 0.9.0 diff --git a/tools/lib-sums.tsv b/tools/lib-sums.tsv new file mode 100644 index 00000000..a0fe3836 --- /dev/null +++ b/tools/lib-sums.tsv @@ -0,0 +1,6 @@ +url sha256sum +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-aarch64-apple-darwin.tar.gz 882ffcc187e225d44d17c1c3bdcac3c27e7fa3b9577a26a4ab2f74677e87f0e8 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-aarch64-unknown-linux-musl.tar.gz e91a0e3db7a60e8b6b61f55dd061ec1d0429e82e749e0ce16a8af8830fa55eec +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-apple-darwin.tar.gz de46cb2e1a68dfca1a6a40116d176cdb850b0c63de0d8495b4453a1099dfcde1 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-pc-windows-gnu.tar.gz 4170bd99c98bc1b39610bdadeccc60a69e5aeaf00f348d2e760da19cb188dbe8 +https://github.com/eitsupi/prqlr/releases/download/lib-v0.9.0/libprqlr-0.9.0-x86_64-unknown-linux-musl.tar.gz 3a9fcdae0b4fdd42bc2e12476b44471c1704a6702770d0dd4e85dee7df38972f From c9dc20118b5432b740eac62948dd4d562e081bb9 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 15:43:39 +0000 Subject: [PATCH 18/19] ci: fix test step --- .github/workflows/check.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ea73baba..5f465700 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -119,11 +119,14 @@ jobs: use-public-rspm: true Ncpus: "2" + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck, any::devtools + needs: check + - name: Install with pre-built binary if: env.TEST_BIN_LIB == 'true' shell: Rscript {0} run: | - install.packages("pak") - pak::pak(c("devtools", "remotes")) remotes::install_local() devtools::test() From 03a9fec800de613dcfa0faa3ed2c870dda6840a8 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 30 Sep 2023 16:07:52 +0000 Subject: [PATCH 19/19] chore: tweak messages --- configure | 23 ++++++++++++++++------- configure.win | 23 ++++++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/configure b/configure index e5f359e4..8e59305e 100755 --- a/configure +++ b/configure @@ -10,7 +10,8 @@ export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { if [ ! "$(command -v cargo)" ]; then - echo "----------------------- [RUST NOT FOUND]---------------------------" + echo "" + echo "------------------------- [RUST NOT FOUND] -------------------------" echo "The 'cargo' command was not found on the PATH. Please install rustc" echo "from: https://www.rust-lang.org/tools/install" echo "" @@ -18,9 +19,17 @@ check_cargo() { echo " - Debian/Ubuntu: apt-get install cargo" echo " - Fedora/CentOS: dnf install cargo" echo " - macOS: brew install rustc" - echo "-------------------------------------------------------------------" + echo "--------------------------------------------------------------------" echo "" exit 1 + else + echo "" + echo "--------------------------- [RUST FOUND] ---------------------------" + "$(cargo -V)" + echo "" + "$(rustc -vV)" + echo "--------------------------------------------------------------------" + echo "" fi } @@ -37,16 +46,16 @@ check_bin_lib() { if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then echo "" - echo "----------------------- [LIBRARY FOUND]----------------------------" - echo "The library was found at ${LIBPRQLR_PATH}. No need to build it." + echo "------------------------- [LIBRARY FOUND] -------------------------" + echo "The library was found at <${LIBPRQLR_PATH}>. No need to build it." echo "-------------------------------------------------------------------" echo "" sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars exit 0 - else + elif [ "${LIBPRQLR_BUILD}" = "false" ]; then echo "" - echo "----------------------- [LIBRARY NOT FOUND]------------------------" - echo "The library was not found at ${LIBPRQLR_PATH}." + echo "----------------------- [LIBRARY NOT FOUND] -----------------------" + echo "The library was not found at <${LIBPRQLR_PATH}>." echo "-------------------------------------------------------------------" echo "" fi diff --git a/configure.win b/configure.win index e8b7b31c..9ec3c13f 100755 --- a/configure.win +++ b/configure.win @@ -10,12 +10,21 @@ export PATH="$PATH:$HOME/.cargo/bin" check_cargo() { if [ ! "$(command -v cargo)" ]; then - echo "----------------------- [RUST NOT FOUND]---------------------------" + echo "" + echo "------------------------- [RUST NOT FOUND] -------------------------" echo "The 'cargo' command was not found on the PATH. Please install rustc" echo "from: https://www.rust-lang.org/tools/install" - echo "-------------------------------------------------------------------" + echo "--------------------------------------------------------------------" echo "" exit 1 + else + echo "" + echo "--------------------------- [RUST FOUND] ---------------------------" + "$(cargo -V)" + echo "" + "$(rustc -vV)" + echo "--------------------------------------------------------------------" + echo "" fi } @@ -32,16 +41,16 @@ check_bin_lib() { if [ "${LIBPRQLR_BUILD}" = "false" ] && [ -f "${LIBPRQLR_PATH}" ]; then echo "" - echo "----------------------- [LIBRARY FOUND]----------------------------" - echo "The library was found at ${LIBPRQLR_PATH}. No need to build it." + echo "------------------------- [LIBRARY FOUND] -------------------------" + echo "The library was found at <${LIBPRQLR_PATH}>. No need to build it." echo "-------------------------------------------------------------------" echo "" sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars exit 0 - else + elif [ "${LIBPRQLR_BUILD}" = "false" ]; then echo "" - echo "----------------------- [LIBRARY NOT FOUND]------------------------" - echo "The library was not found at ${LIBPRQLR_PATH}." + echo "----------------------- [LIBRARY NOT FOUND] -----------------------" + echo "The library was not found at <${LIBPRQLR_PATH}>." echo "-------------------------------------------------------------------" echo "" fi