Background execution and R process pool

.github/actions/setup/action.yaml

@@ -28,7 +28,7 @@ runs:
             libssl-dev
 
       - name: Set up Rust nightly toolchain
-        if: inputs.rust-nightly == 'true' || env.RPOLARS_ALL_FEATURES == 'true'
+        if: inputs.rust-nightly == 'true' || env.RPOLARS_FULL_FEATURES == 'true'
         shell: bash
         run: |
           make requirements-rs

.github/workflows/check.yaml

@@ -18,7 +18,7 @@ env:
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}
-    name: ${{ matrix.config.os }} (${{ matrix.config.r }}) ${{ matrix.config.all-features && 'all-features' || '' }}
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }}) ${{ matrix.config.full-features && 'full-features' || '' }}
     strategy:
       fail-fast: false
       matrix:
@@ -30,9 +30,9 @@ jobs:
           - { os: ubuntu-latest, r: "release" }
           - { os: ubuntu-latest, r: "oldrel-1" }
         include:
-          - config: { os: macos-latest, r: "release", all-features: true }
-          - config: { os: windows-latest, r: "release", all-features: true }
-          - config: { os: ubuntu-latest, r: "release", all-features: true }
+          - config: { os: macos-latest, r: "release", full-features: true }
+          - config: { os: windows-latest, r: "release", full-features: true }
+          - config: { os: ubuntu-latest, r: "release", full-features: true }
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
@@ -46,7 +46,7 @@ jobs:
 
       - uses: ./.github/actions/setup
         with:
-          rust-nightly: "${{ matrix.config.all-features }}"
+          rust-nightly: "${{ matrix.config.full-features }}"
 
       - name: print files
         run: print(list.files("..",recursive = TRUE,full.names=TRUE))
@@ -77,10 +77,10 @@ jobs:
           echo "RPOLARS_RUST_SOURCE=${PWD}/src/rust" >> $GITHUB_ENV
 
       - name: Set env vars for build option
-        if: matrix.config.all-features
+        if: matrix.config.full-features
         shell: bash
         run: |
-          echo "RPOLARS_ALL_FEATURES=true" >>$GITHUB_ENV
+          echo "RPOLARS_FULL_FEATURES=true" >>$GITHUB_ENV
           echo "RPOLARS_PROFILE=release-optimized" >>$GITHUB_ENV
 
       - uses: r-lib/actions/check-r-package@v2

.github/workflows/docs.yaml

@@ -21,7 +21,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RPOLARS_ALL_FEATURES: "true"
+  RPOLARS_FULL_FEATURES: "true"
   RPOLARS_CARGO_CLEAN_DEPS: "true"
   RPOLARS_PROFILE: release-optimized
 

.github/workflows/release.yaml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RPOLARS_ALL_FEATURES: "true"
+  RPOLARS_FULL_FEATURES: "true"
   RPOLARS_CARGO_CLEAN_DEPS: "true"
   RPOLARS_PROFILE: release-optimized
 

DESCRIPTION

@@ -83,14 +83,14 @@ Collate:
     'groupby.R'
     'info.R'
     'ipc.R'
-    'lazyframe__background.R'
     'lazyframe__groupby.R'
     'lazyframe__lazy.R'
     'namespace.R'
     'options.R'
     'parquet.R'
     'pkg-knitr.R'
     'pkg-nanoarrow.R'
+    'rbackground.R'
     'rlang.R'
     'rust_result.R'
     's3_methods.R'

Makefile

@@ -46,15 +46,19 @@ requirements-rs:
 
 .PHONY: build
 build: ## Compile polars R package with all features and generate Rd files
-	export RPOLARS_ALL_FEATURES=true \
-	&& export RPOLARS_PROFILE=release-optimized \
+	export RPOLARS_FULL_FEATURES=true \
 	&& Rscript -e 'if (!(require(arrow)&&require(nanoarrow))) warning("could not load arrow/nanoarrow, igonore changes to nanoarrow.Rd"); rextendr::document()'
 
+.PHONY: install
+install:
+	export RPOLARS_FULL_FEATURES=true \
+	&& R CMD INSTALL --no-multiarch --with-keep.source .
+
 .PHONY: all
 all: fmt build test README.md LICENSE.note ## build -> test -> Update README.md, LICENSE.note
 
 .PHONY: docs
-docs: build README.md docs/docs/reference_home.md ## Generate docs
+docs: build install README.md docs/docs/reference_home.md ## Generate docs
 	cp docs/mkdocs.orig.yml docs/mkdocs.yml
 	Rscript -e 'altdoc::update_docs(custom_reference = "docs/make-docs.R")'
 	cd docs && ../$(VENV_BIN)/python3 -m mkdocs build
@@ -76,10 +80,6 @@ LICENSE.note: src/rust/Cargo.lock ## Update LICENSE.note
 test: build ## Run fast unittests
 	Rscript -e 'devtools::load_all(); devtools::test()'
 
-.PHONY: install
-install: ## Install this R package locally
-	Rscript -e 'devtools::install(pkg = ".", dependencies = TRUE)'
-
 .PHONY: fmt
 fmt: fmt-rs fmt-r ## Format files
 

NAMESPACE

@@ -18,12 +18,12 @@ S3method("$",FeatureInfo)
 S3method("$",GroupBy)
 S3method("$",LazyFrame)
 S3method("$",LazyGroupBy)
-S3method("$",PolarsBackgroundHandle)
 S3method("$",ProtoExprArray)
 S3method("$",RField)
 S3method("$",RNullValues)
 S3method("$",RPolarsDataType)
 S3method("$",RPolarsErr)
+S3method("$",RThreadHandle)
 S3method("$",Series)
 S3method("$",VecDataFrame)
 S3method("$",When)
@@ -63,12 +63,12 @@ S3method("[[",FeatureInfo)
 S3method("[[",GroupBy)
 S3method("[[",LazyFrame)
 S3method("[[",LazyGroupBy)
-S3method("[[",PolarsBackgroundHandle)
 S3method("[[",ProtoExprArray)
 S3method("[[",RField)
 S3method("[[",RNullValues)
 S3method("[[",RPolarsDataType)
 S3method("[[",RPolarsErr)
+S3method("[[",RThreadHandle)
 S3method("[[",Series)
 S3method("[[",VecDataFrame)
 S3method("[[",When)
@@ -80,9 +80,9 @@ S3method(.DollarNames,DataFrame)
 S3method(.DollarNames,Expr)
 S3method(.DollarNames,GroupBy)
 S3method(.DollarNames,LazyFrame)
-S3method(.DollarNames,PolarsBackgroundHandle)
 S3method(.DollarNames,RField)
 S3method(.DollarNames,RPolarsErr)
+S3method(.DollarNames,RThreadHandle)
 S3method(.DollarNames,Series)
 S3method(.DollarNames,VecDataFrame)
 S3method(.DollarNames,When)
@@ -91,6 +91,7 @@ S3method(.DollarNames,WhenThenThen)
 S3method(.DollarNames,method_environment)
 S3method(.DollarNames,polars_option_list)
 S3method(as.character,RPolarsErr)
+S3method(as.character,RThreadHandle)
 S3method(as.character,Series)
 S3method(as.data.frame,DataFrame)
 S3method(as.data.frame,LazyFrame)
@@ -128,10 +129,10 @@ S3method(print,GroupBy)
 S3method(print,LazyFrame)
 S3method(print,LazyGroupBy)
 S3method(print,PTime)
-S3method(print,PolarsBackgroundHandle)
 S3method(print,RField)
 S3method(print,RPolarsDataType)
 S3method(print,RPolarsErr)
+S3method(print,RThreadHandle)
 S3method(print,Series)
 S3method(print,When)
 S3method(print,WhenThen)

NEWS.md

@@ -2,20 +2,30 @@
 
 ## BREAKING CHANGES
 - `$rpow()` is removed. It should never have been translated. Use `^` and `$pow()` instead (#346).
+- `<LazyFrame>$collect_background()` renamed `<LazyFrame>$collect_in_background()` and reworked. 
+ Likewise `PolarsBackgroundHandle` reworked and renamed to `RThreadHandle` (#311).
 
 ## What's changed
-
 - New method `$explode()` for `DataFrame` and `LazyFrame` (#314).
 - New method `$clone()` for `LazyFrame` (#347).
 - `$with_column()` is now deprecated (following upstream `polars`). It will be
   removed in 0.9.0. It should be replaced with `$with_columns()` (#313).
 - New lazy function translated: `concat_str()` to concatenate several columns
   into one (#349).
 - New stat functions `pl$cov()`, `pl$rolling_cov()` `pl$corr()`, `pl$rolling_corr()` (#351).
+- Add functions `pl$set_global_rpool_cap()`, `pl$get_global_rpool_cap()`, class `RThreadHandle` and
+  `in_background = FALSE` param to `<Expr>$map()` and `$apply()`. It is now possible to run R code
+  with `<LazyFrame>collect_in_background()` and/or let polars parallize R code in an R processes
+  pool. See `RThreadHandle-class` in reference docs for more info. (#311)
+- Internal IPC/shared-mem channel to serialize and send R objects / polars DataFrame across 
+  R processes. (#311)
+- Compile environment flag RPOLARS_ALL_FEATURES changes name to RPOLARS_FULL_FEATURES. If 'true'
+  will trigger something like `Cargo build --features "full_features"` which is not exactly the same
+  as `Cargo build --all-features`. Some dev features are not included in "full_features" (#311).
 - Fix bug to allow using polars without library(polars) (#355).
 - New methods `<LazyFrame>$optimization_toggle()` + `$profile()` and enable rust-polars feature
   CSE: "Activate common subplan elimination optimization" (#323)
-  
+
 # polars 0.7.0
 
 ## BREAKING CHANGES

R/after-wrappers.R

@@ -82,7 +82,6 @@ extendr_method_to_pure_functions = function(env, class_name = NULL) {
 .pr$GroupBy = NULL # derived from DataFrame in R, has no rust calls
 .pr$LazyFrame = extendr_method_to_pure_functions(LazyFrame)
 .pr$LazyGroupBy = extendr_method_to_pure_functions(LazyGroupBy)
-.pr$PolarsBackgroundHandle = extendr_method_to_pure_functions(PolarsBackgroundHandle)
 .pr$DataType = extendr_method_to_pure_functions(RPolarsDataType)
 .pr$DataTypeVector = extendr_method_to_pure_functions(DataTypeVector)
 .pr$RField = extendr_method_to_pure_functions(RField)
@@ -94,6 +93,7 @@ extendr_method_to_pure_functions = function(env, class_name = NULL) {
 .pr$VecDataFrame = extendr_method_to_pure_functions(VecDataFrame)
 .pr$RNullValues = extendr_method_to_pure_functions(RNullValues)
 .pr$RPolarsErr = extendr_method_to_pure_functions(RPolarsErr)
+.pr$RThreadHandle = extendr_method_to_pure_functions(RThreadHandle)
 
 
 
@@ -223,9 +223,8 @@ pl$show_all_public_functions = function() {
 #' @examples
 #' pl$show_all_public_methods()
 pl$show_all_public_methods = function(class_names = NULL) {
-
-  #subset classes to show
-  show_this_env = if(!is.null(class_names)) {
+  # subset classes to show
+  show_this_env = if (!is.null(class_names)) {
     as.environment(mget(class_names, envir = pl_pub_class_env))
   } else {
     pl_pub_class_env
@@ -265,7 +264,7 @@ DataType = clone_env_one_level_deep(RPolarsDataType)
 # used for printing public environment
 pl_class_names = sort(
   c(
-    "LazyFrame", "Series", "LazyGroupBy", "DataType", "Expr", "DataFrame", "PolarsBackgroundHandle",
+    "LazyFrame", "Series", "LazyGroupBy", "DataType", "Expr", "DataFrame",
     "When", "WhenThen", "WhenThenThen"
   )
 ) # TODO discover all public class automatically