Bump rust-polars to 0.41

NEWS.md

@@ -2,21 +2,60 @@
 
 ## Polars R Package (development version)
 
+Updated rust-polars to 0.41.2 (#1147).
+
 ### Breaking changes
 
 - In `$n_chunks()`, the default value of `strategy` now is `"first"` (#1137).
--`$sample()` for Expr and DataFrame (#1136):
+- `$sample()` for Expr and DataFrame (#1136):
   - the argument `frac` is renamed `fraction`;
   - all the arguments except `n` must be named;
   - for the Expr method only, the first argument is now `n` (it was already the
     case for the DataFrame method);
   - for the Expr method only, the default value for `with_replacement` is now
     `FALSE` (it was already the case for the DataFrame method).
+- `$melt()` had several changes (#1147):
+  - `melt()` is renamed `$unpivot()`.
+  - Some arguments were renamed: `id_vars` is now `index`, `value_vars` is now
+    `on`.
+  - The order of arguments has changed: `on` is now first, then `index`. The
+    order of the other arguments hasn't changed. Note that `on` can be unnamed
+    but all the other arguments must be named.
+- `pivot()` had several changes (#1147):
+  - The argument `columns` is renamed `on`.
+  - The order of arguments has changed: `on` is now first, then `index` and
+    `values`. The order of the other arguments hasn't changed. Note that `on`
+    can be unnamed but all the other arguments must be named.
+- In `$write_parquet()` and `$sink_parquet()`, the default value of argument
+  `statistics` is now `TRUE` and can take other values than `TRUE/FALSE` (#1147).
+- In `$dt$truncate()` and `$dt$round()`, the argument `offset` has been removed.
+  Use `$dt$offset_by()` after those functions instead (#1147).
+- In `$top_k()` and `$bottom_k()` for `Expr`, the arguments `nulls_last`,
+  `maintain_order` and `multithreaded` have been removed. If any `null` values
+  are in the top/bottom `k` values, they will always be positioned last (#1147).
+- `$replace()` has been split in two functions depending on the desired
+  behaviour (#1147):
+  - `$replace()` recodes some values in the column, leaving all other values
+    unchanged. Compared to the previous version, it doesn't use the arguments
+    `default` and `return_dtype` anymore.
+  - `$replace_strict()` replaces all values by different values. If a value
+    doesn't have a specific mapping, it is replaced by the `default` value.
+- `$str$concat()` is deprecated, use `$str$join()` (with the same arguments)
+  instead (#1147).
+- In `pl$date_range()` and `pl$date_ranges()`, the arguments `time_unit` and
+  `time_zone` have been removed. They were deprecated in previous versions
+  (#1147).
+
 
 ### New features
 
 - New method `$has_nulls()` (#1133).
 - New method `$list$explode()` (#1139).
+- `$over()` gains a new argument `order_by` to specify the order of values
+  within each group. This is useful when the operation depends on the order of
+  values, such as `$shift()` (#1147).
+- `$value_counts()` gains an argument `normalize` to give relative frequencies
+  of unique values instead of their count (#1147).
 
 ## Polars R Package 0.17.0
 

R/dataframe__frame.R

@@ -1490,7 +1490,7 @@ DataFrame_join_asof = function(
 
 
 
-#' @inherit LazyFrame_melt
+#' @inherit LazyFrame_unpivot
 #' @keywords DataFrame
 #'
 #' @return A new `DataFrame`
@@ -1502,25 +1502,26 @@ DataFrame_join_asof = function(
 #'   c = c(2, 4, 6),
 #'   d = c(7, 8, 9)
 #' )
-#' df$melt(id_vars = "a", value_vars = c("b", "c", "d"))
-DataFrame_melt = function(
-    id_vars = NULL,
-    value_vars = NULL,
+#' df$unpivot(index = "a", on = c("b", "c", "d"))
+DataFrame_unpivot = function(
+    on = NULL,
+    ...,
+    index = NULL,
     variable_name = NULL,
     value_name = NULL) {
-  .pr$DataFrame$melt(
-    self, id_vars %||% character(), value_vars %||% character(),
+  .pr$DataFrame$unpivot(
+    self, on %||% character(), index %||% character(),
     value_name, variable_name
-  ) |> unwrap("in $melt( ): ")
+  ) |> unwrap("in $unpivot( ): ")
 }
 
 
 
 #' Pivot data from long to wide
 #' @param values Column values to aggregate. Can be multiple columns if the
-#' `columns` arguments contains multiple columns as well.
+#' `on` arguments contains multiple columns as well.
 #' @param index  One or multiple keys to group by.
-#' @param columns  Name of the column(s) whose values will be used as the header
+#' @param on  Name of the column(s) whose values will be used as the header
 #' of the output DataFrame.
 #' @param ... Not used.
 #' @param aggregate_function One of:
@@ -1544,7 +1545,7 @@ DataFrame_melt = function(
 #' df
 #'
 #' df$pivot(
-#'   values = "baz", index = "foo", columns = "bar"
+#'   values = "baz", index = "foo", on = "bar"
 #' )
 #'
 #' # Run an expression as aggregation function
@@ -1557,15 +1558,15 @@ DataFrame_melt = function(
 #'
 #' df$pivot(
 #'   index = "col1",
-#'   columns = "col2",
+#'   on = "col2",
 #'   values = "col3",
 #'   aggregate_function = pl$element()$tanh()$mean()
 #' )
 DataFrame_pivot = function(
-    values,
-    index,
-    columns,
+    on,
     ...,
+    index,
+    values,
     aggregate_function = NULL,
     maintain_order = TRUE,
     sort_columns = FALSE,
@@ -1586,7 +1587,7 @@ DataFrame_pivot = function(
     )) |>
     # run pivot when valid aggregate_expr
     and_then(\(aggregate_expr) .pr$DataFrame$pivot_expr(
-      self, index, columns, values, maintain_order, sort_columns, aggregate_expr, separator
+      self, on, index, values, maintain_order, sort_columns, aggregate_expr, separator
     )) |>
     # unwrap and add method context name
     unwrap("in $pivot():")
@@ -1736,7 +1737,7 @@ DataFrame_describe = function(percentiles = c(.25, .75), interpolation = "neares
     )$
       unnest("fields")$
       drop("column")$
-      pivot(index = "statistic", columns = "variable", values = "column_0")$
+      pivot(index = "statistic", on = "variable", values = "column_0")$
       with_columns(statistic = pl$lit(metrics))
   }) |>
     uw()
@@ -2031,9 +2032,11 @@ DataFrame_write_parquet = function(
     ...,
     compression = "zstd",
     compression_level = 3,
-    statistics = FALSE,
+    statistics = TRUE,
     row_group_size = NULL,
     data_pagesize_limit = NULL) {
+  statistics = translate_statistics(statistics) |>
+    unwrap("in $write_parquet():")
   .pr$DataFrame$write_parquet(
     self,
     file,

R/expr__datetime.R

@@ -2,8 +2,8 @@
 #' @description  Divide the date/datetime range into buckets.
 #' Each date/datetime is mapped to the start of its bucket.
 #'
-#' @param every string encoding duration see details.
-#' @param offset optional string encoding duration see details.
+#' @param every Either an Expr or a string indicating a column name or a
+#' duration (see Details).
 #'
 #' @details The ``every`` and ``offset`` argument are created with the
 #' the following string language:
@@ -20,22 +20,18 @@
 #' These strings can be combined:
 #'   - 3d12h4m25s # 3 days, 12 hours, 4 minutes, and 25 seconds
 #' @return   Date/Datetime expr
-#' @keywords ExprDT
-#' @aliases (Expr)$dt$truncate
 #' @examples
 #' t1 = as.POSIXct("3040-01-01", tz = "GMT")
 #' t2 = t1 + as.difftime(25, units = "secs")
-#' s = pl$date_range(t1, t2, interval = "2s", time_unit = "ms")
+#' s = pl$datetime_range(t1, t2, interval = "2s", time_unit = "ms")
 #'
-#' # use a dt namespace function
 #' df = pl$DataFrame(datetime = s)$with_columns(
-#'   pl$col("datetime")$dt$truncate("4s")$alias("truncated_4s"),
-#'   pl$col("datetime")$dt$truncate("4s", offset("3s"))$alias("truncated_4s_offset_2s")
+#'   pl$col("datetime")$dt$truncate("4s")$alias("truncated_4s")
 #' )
 #' df
-ExprDT_truncate = function(every, offset = NULL) {
-  offset = parse_as_polars_duration_string(offset, default = "0ns")
-  .pr$Expr$dt_truncate(self, every, offset) |>
+ExprDT_truncate = function(every) {
+  every = parse_as_polars_duration_string(every, default = "0ns")
+  .pr$Expr$dt_truncate(self, every) |>
     unwrap("in $dt$truncate()")
 }
 
@@ -46,46 +42,20 @@ ExprDT_truncate = function(every, offset = NULL) {
 #' Each date/datetime in the second half of the interval
 #' is mapped to the end of its bucket.
 #'
+#' @inherit ExprDT_truncate params details return
 #'
-#' @param every string encoding duration see details.
-#' @param offset optional string encoding duration see details.
-#'
-#' @details The ``every`` and ``offset`` arguments are created with the
-#' following string language:
-#' - 1ns # 1 nanosecond
-#' - 1us # 1 microsecond
-#' - 1ms # 1 millisecond
-#' - 1s  # 1 second
-#' - 1m  # 1 minute
-#' - 1h  # 1 hour
-#' - 1d  # 1 day
-#' - 1w  # 1 calendar week
-#' - 1mo # 1 calendar month
-#' - 1y  # 1 calendar year
-#' These strings can be combined:
-#'   - 3d12h4m25s # 3 days, 12 hours, 4 minutes, and 25 seconds
-#'
-#' This functionality is currently experimental and may
-#' change without it being considered a breaking change.
-#'
-#' @return   Date/Datetime expr
-#' @keywords ExprDT
-#' @aliases (Expr)$dt$round
 #' @examples
 #' t1 = as.POSIXct("3040-01-01", tz = "GMT")
 #' t2 = t1 + as.difftime(25, units = "secs")
-#' s = pl$date_range(t1, t2, interval = "2s", time_unit = "ms")
+#' s = pl$datetime_range(t1, t2, interval = "2s", time_unit = "ms")
 #'
-#' # use a dt namespace function
 #' df = pl$DataFrame(datetime = s)$with_columns(
-#'   pl$col("datetime")$dt$truncate("4s")$alias("truncated_4s"),
-#'   pl$col("datetime")$dt$truncate("4s", offset("3s"))$alias("truncated_4s_offset_2s")
+#'   pl$col("datetime")$dt$round("4s")$alias("rounded_4s")
 #' )
 #' df
-ExprDT_round = function(every, offset = NULL) {
+ExprDT_round = function(every) {
   every = parse_as_polars_duration_string(every, default = "0ns")
-  offset = parse_as_polars_duration_string(offset, default = "0ns")
-  .pr$Expr$dt_round(self, every, offset) |>
+  .pr$Expr$dt_round(self, every) |>
     unwrap("in $dt$round()")
 }
 
@@ -370,7 +340,7 @@ ExprDT_ordinal_day = function() {
 #' @aliases (Expr)$dt$hour
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     as.Date("2020-12-25"),
 #'     as.Date("2021-1-05"),
 #'     interval = "1d2h",
@@ -395,7 +365,7 @@ ExprDT_hour = function() {
 #' @aliases (Expr)$dt$minute
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     as.Date("2020-12-25"),
 #'     as.Date("2021-1-05"),
 #'     interval = "1d5s",
@@ -556,7 +526,7 @@ ExprDT_epoch = function(tu = c("us", "ns", "ms", "s", "d")) {
 #' @aliases (Expr)$dt$timestamp
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     start = as.Date("2001-1-1"),
 #'     end = as.Date("2001-1-3"),
 #'     interval = "1d1s"
@@ -585,7 +555,7 @@ ExprDT_timestamp = function(tu = c("ns", "us", "ms")) {
 #' @aliases (Expr)$dt$with_time_unit
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     start = as.Date("2001-1-1"),
 #'     end = as.Date("2001-1-3"),
 #'     interval = "1d1s"
@@ -615,7 +585,7 @@ ExprDT_with_time_unit = function(tu = c("ns", "us", "ms")) {
 #' @aliases (Expr)$dt$cast_time_unit
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     start = as.Date("2001-1-1"),
 #'     end = as.Date("2001-1-3"),
 #'     interval = "1d1s"
@@ -641,10 +611,10 @@ ExprDT_cast_time_unit = function(tu = c("ns", "us", "ms")) {
 #' @return Expr of i64
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     as.POSIXct("2020-03-01", tz = "UTC"),
 #'     as.POSIXct("2020-05-01", tz = "UTC"),
-#'     "1mo"
+#'     "1mo1s"
 #'   )
 #' )
 #'
@@ -681,10 +651,10 @@ ExprDT_convert_time_zone = function(time_zone) {
 #' @aliases (Expr)$dt$replace_time_zone
 #' @examples
 #' df1 = pl$DataFrame(
-#'   london_timezone = pl$date_range(
+#'   london_timezone = pl$datetime_range(
 #'     as.POSIXct("2020-03-01", tz = "UTC"),
 #'     as.POSIXct("2020-07-01", tz = "UTC"),
-#'     "1mo"
+#'     "1mo1s"
 #'   )$dt$convert_time_zone("Europe/London")
 #' )
 #'
@@ -729,10 +699,10 @@ ExprDT_replace_time_zone = function(
 #' @return Expr of i64
 #' @examples
 #' df = pl$DataFrame(
-#'   date = pl$date_range(
+#'   date = pl$datetime_range(
 #'     start = as.Date("2020-3-1"),
 #'     end = as.Date("2020-5-1"),
-#'     interval = "1mo"
+#'     interval = "1mo1s"
 #'   )
 #' )
 #' df$select(
@@ -791,7 +761,7 @@ ExprDT_total_minutes = function() {
 #'
 #' @return Expr of i64
 #' @examples
-#' df = pl$DataFrame(date = pl$date_range(
+#' df = pl$DataFrame(date = pl$datetime_range(
 #'   start = as.POSIXct("2020-1-1", tz = "GMT"),
 #'   end = as.POSIXct("2020-1-1 00:04:00", tz = "GMT"),
 #'   interval = "1m"
@@ -810,7 +780,7 @@ ExprDT_total_seconds = function() {
 #'
 #' @return Expr of i64
 #' @examples
-#' df = pl$DataFrame(date = pl$date_range(
+#' df = pl$DataFrame(date = pl$datetime_range(
 #'   start = as.POSIXct("2020-1-1", tz = "GMT"),
 #'   end = as.POSIXct("2020-1-1 00:00:01", tz = "GMT"),
 #'   interval = "1ms"
@@ -829,7 +799,7 @@ ExprDT_total_milliseconds = function() {
 #'
 #' @return Expr of i64
 #' @examples
-#' df = pl$DataFrame(date = pl$date_range(
+#' df = pl$DataFrame(date = pl$datetime_range(
 #'   start = as.POSIXct("2020-1-1", tz = "GMT"),
 #'   end = as.POSIXct("2020-1-1 00:00:01", tz = "GMT"),
 #'   interval = "1ms"
@@ -848,7 +818,7 @@ ExprDT_total_microseconds = function() {
 #'
 #' @return Expr of i64
 #' @examples
-#' df = pl$DataFrame(date = pl$date_range(
+#' df = pl$DataFrame(date = pl$datetime_range(
 #'   start = as.POSIXct("2020-1-1", tz = "GMT"),
 #'   end = as.POSIXct("2020-1-1 00:00:01", tz = "GMT"),
 #'   interval = "1ms"
@@ -907,7 +877,7 @@ ExprDT_total_nanoseconds = function() {
 #'
 #' # the "by" argument also accepts expressions
 #' df = pl$DataFrame(
-#'   dates = pl$date_range(
+#'   dates = pl$datetime_range(
 #'     as.POSIXct("2022-01-01", tz = "GMT"),
 #'     as.POSIXct("2022-01-02", tz = "GMT"),
 #'     interval = "6h", time_unit = "ms", time_zone = "GMT"
@@ -932,7 +902,7 @@ ExprDT_offset_by = function(by) {
 #'
 #'
 #' @examples
-#' df = pl$DataFrame(dates = pl$date_range(
+#' df = pl$DataFrame(dates = pl$datetime_range(
 #'   as.Date("2000-1-1"),
 #'   as.Date("2000-1-2"),
 #'   "1h"