Nodejs updates (#2895)

* chore(nodejs): fix compile issues

* chore(nodejs): sync node polars with main

* docs(nodejs): fix  #2893

nodejs-polars/Cargo.toml

@@ -42,7 +42,6 @@ features = [
   "random",
   "object",
   "csv-file",
-  "pretty_fmt",
   "performant",
   "dtype-full",
   "rows",

nodejs-polars/polars/lazy/expr.ts

@@ -459,8 +459,8 @@ export interface Expr extends
   skew(bias?: boolean): Expr
   skew({bias}: {bias: boolean}): Expr
   /** Slice the Series. */
-  slice(offset: number, length: number): Expr
-  slice({offset, length}: {offset: number, length: number}): Expr
+  slice(offset: number | Expr, length: number | Expr): Expr
+  slice({offset, length}: {offset: number | Expr, length: number | Expr}): Expr
   /**
    * Sort this column. In projection/ selection context the whole column is sorted.
    * @param reverse
@@ -508,7 +508,6 @@ export interface Expr extends
 const _Expr = (_expr: any): Expr => {
 
   const wrap = (method, args?): Expr => {
-
     return Expr(pli.expr[method]({_expr, ...args }));
   };
 
@@ -744,7 +743,13 @@ const _Expr = (_expr: any): Expr => {
       }
     },
     skew: wrapUnaryWithDefault("skew", "bias", true),
-    slice: wrapBinary("slice", "offset", "length"),
+    slice(arg, len?) {
+      if(typeof arg === "number") {
+        return wrap("slice", {offset: lit(arg)._expr, length: lit(len)._expr});
+      }
+
+      return wrap("slice", {offset: lit(arg.offset)._expr, length: lit(arg.length)._expr});
+    },
     sort(reverse: any = false, nullsLast=false) {
       if(typeof reverse === "boolean") {
         return wrap("sortWith", {reverse, nullsLast});

nodejs-polars/src/conversion/from.rs

@@ -388,7 +388,7 @@ impl FromJsUnknown for Schema {
                     })
                     .collect();
 
-                Ok(Schema::new(fields))
+                Ok(Schema::from(fields))
             }
             dt => {
                 return Err(JsPolarsEr::Other(format!(

nodejs-polars/src/dataframe/frame.rs

@@ -182,9 +182,9 @@ pub fn schema(cx: CallContext) -> JsResult<JsObject> {
     let df = params.get_external::<DataFrame>(&cx, "_df")?;
     let mut obj = cx.env.create_object()?;
 
-    for field in df.schema().fields() {
-        let field_name = format!("{}", field.name()).try_into_js(&cx)?;
-        let dtype: JsDataType = field.data_type().clone().into();
+    for (name, dtype) in df.schema().iter() {
+        let field_name = format!("{}", name).try_into_js(&cx)?;
+        let dtype: JsDataType = dtype.clone().into();
         let js_string = dtype.to_string().try_into_js(&cx)?;
         obj.set_property(field_name, js_string).unwrap();
     }
@@ -421,7 +421,7 @@ pub fn sort_in_place(cx: CallContext) -> JsResult<JsUndefined> {
     let df = params.get_external_mut::<DataFrame>(&cx, "_df")?;
     let by_column = params.get_as::<&str>("by")?;
     let reverse = params.get_as::<bool>("reverse")?;
-    
+
     df.sort_in_place([by_column], reverse)
         .map_err(JsPolarsEr::from)?;
 

nodejs-polars/src/dataframe/io.rs

@@ -3,7 +3,7 @@ use crate::error::JsPolarsEr;
 use crate::file::JsWriteStream;
 use crate::prelude::JsResult;
 use napi::{CallContext, JsExternal, JsObject, JsString, JsUndefined, JsUnknown, ValueType};
-use polars::frame::row::{rows_to_schema, Row, infer_schema};
+use polars::frame::row::{infer_schema, rows_to_schema, Row};
 use polars::io::RowCount;
 use polars::prelude::*;
 use std::borrow::Borrow;
@@ -276,7 +276,7 @@ pub(crate) fn read_parquet_buffer(cx: CallContext) -> JsResult<JsExternal> {
 pub(crate) fn write_parquet_path(cx: CallContext) -> JsResult<JsUndefined> {
     let params = get_params(&cx)?;
     let compression = params.get_as::<String>("compression")?;
-    let df = params.get_external::<DataFrame>(&cx, "_df")?;
+    let df = params.get_external_mut::<DataFrame>(&cx, "_df")?;
     let path = params.get_as::<String>("path")?;
     let compression = match compression.as_str() {
         "uncompressed" => ParquetCompression::Uncompressed,
@@ -301,7 +301,7 @@ pub(crate) fn write_parquet_path(cx: CallContext) -> JsResult<JsUndefined> {
 #[js_function(1)]
 pub(crate) fn write_parquet_stream(cx: CallContext) -> JsResult<JsUndefined> {
     let params = get_params(&cx)?;
-    let df = params.get_external::<DataFrame>(&cx, "_df")?;
+    let df = params.get_external_mut::<DataFrame>(&cx, "_df")?;
     let stream = params.get::<JsObject>("writeStream")?;
     let writeable = JsWriteStream {
         inner: stream,
@@ -439,6 +439,7 @@ pub(crate) fn read_json_buffer(cx: CallContext) -> JsResult<JsExternal> {
     JsonReader::new(reader)
         .infer_schema_len(infer_schema_length)
         .with_batch_size(batch_size)
+        .with_json_format(JsonFormat::JsonLines)
         .finish()
         .map_err(JsPolarsEr::from)?
         .try_into_js(&cx)
@@ -456,6 +457,7 @@ pub(crate) fn read_json_path(cx: CallContext) -> JsResult<JsExternal> {
     JsonReader::new(reader)
         .infer_schema_len(infer_schema_length)
         .with_batch_size(batch_size)
+        .with_json_format(JsonFormat::JsonLines)
         .finish()
         .map_err(JsPolarsEr::from)?
         .try_into_js(&cx)
@@ -605,7 +607,7 @@ pub(crate) fn to_row_object(cx: CallContext) -> JsResult<JsObject> {
 fn coerce_js_anyvalue<'a>(
     cx: &'a CallContext,
     val: JsUnknown,
-    dtype: &'a DataType,
+    dtype: DataType,
 ) -> JsResult<AnyValue<'a>> {
     use DataType::*;
     let vtype = val.get_type().unwrap();
@@ -720,10 +722,9 @@ pub(crate) fn read_rows(cx: CallContext) -> JsResult<JsExternal> {
         .map(|idx| {
             let js_row: JsObject = rows.get_element_unchecked(idx).unwrap();
             Row(schema
-                .fields()
-                .iter()
+                .iter_fields()
                 .map(|fld| {
-                    let dtype = fld.data_type();
+                    let dtype = fld.data_type().clone();
                     let key = fld.name();
                     match js_row.get_named_property::<JsUnknown>(key) {
                         Ok(value) => {
@@ -775,15 +776,14 @@ fn resolve_homedir(path: &Path) -> PathBuf {
 
 fn finish_from_rows(rows: Vec<Row>) -> JsResult<DataFrame> {
     let schema = rows_to_schema(&rows);
-    let fields = schema
-        .fields()
-        .iter()
+    let fields: Vec<Field> = schema
+        .iter_fields()
         .map(|fld| match fld.data_type() {
             DataType::Null => Field::new(fld.name(), DataType::Boolean),
             _ => fld.clone(),
         })
         .collect();
-    let schema = Schema::new(fields);
+    let schema = Schema::from(fields);
 
     DataFrame::from_rows_and_schema(&rows, &schema).map_err(|err| JsPolarsEr::from(err).into())
 }

nodejs-polars/src/dataframe/mod.rs

@@ -114,7 +114,6 @@ impl JsDataFrame {
             napi::Property::new(env, "readIPCBuffer")?.with_method(io::read_ipc_buffer),
             napi::Property::new(env, "write_ipc_path")?.with_method(io::write_ipc_path),
             napi::Property::new(env, "write_ipc_stream")?.with_method(io::write_ipc_stream),
-
         ])?;
         Ok(df_obj)
     }

nodejs-polars/src/datatypes.rs

@@ -156,7 +156,6 @@ impl From<napi::ValueType> for Wrap<DataType> {
     }
 }
 
-
 impl From<DataType> for JsDataType {
     fn from(dt: DataType) -> Self {
         (&dt).into()

nodejs-polars/src/lazy/dsl.rs

@@ -216,8 +216,8 @@ pub fn take_every(cx: CallContext) -> JsResult<JsExternal> {
 pub fn slice(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let expr = params.get_external::<Expr>(&cx, "_expr")?;
-    let offset = params.get_as::<i64>("offset")?;
-    let length = params.get_as::<usize>("length")?;
+    let offset = params.get_external::<Expr>(&cx, "offset")?.clone();
+    let length = params.get_external::<Expr>(&cx, "length")?.clone();
 
     expr.clone().slice(offset, length).try_into_js(&cx)
 }
@@ -522,7 +522,10 @@ pub fn timestamp(cx: CallContext) -> JsResult<JsExternal> {
     let expr = params.get_external::<Expr>(&cx, "_expr")?;
     expr.clone()
         .map(
-            |s| s.timestamp().map(|ca| ca.into_series()),
+            |s| {
+                s.timestamp(TimeUnit::Milliseconds)
+                    .map(|ca| ca.into_series())
+            },
             GetOutput::from_type(DataType::Int64),
         )
         .try_into_js(&cx)

nodejs-polars/src/lazy/dsl_object.rs

@@ -187,7 +187,6 @@ impl dsl::JsExpr {
             napi::Property::new(env, "sortWith")?.with_method(dsl::sort_with),
             napi::Property::new(env, "std")?.with_method(dsl::std),
             napi::Property::new(env, "suffix")?.with_method(dsl::suffix),
-
             napi::Property::new(env, "sum")?.with_method(dsl::sum),
             napi::Property::new(env, "tail")?.with_method(dsl::tail),
             napi::Property::new(env, "takeEvery")?.with_method(dsl::take_every),

nodejs-polars/src/lazy/functions.rs

@@ -1,7 +1,7 @@
 use crate::conversion::prelude::*;
 use crate::prelude::JsResult;
 use napi::*;
-use polars::lazy::functions;
+use polars::lazy::dsl;
 use polars::prelude::*;
 
 #[js_function(1)]
@@ -10,7 +10,7 @@ pub fn arange(cx: CallContext) -> JsResult<JsExternal> {
     let low = params.get_external::<Expr>(&cx, "low")?.clone();
     let high = params.get_external::<Expr>(&cx, "high")?.clone();
     let step: usize = params.get_or("step", 1)?;
-    let arange = functions::arange(low, high, step);
+    let arange = dsl::arange(low, high, step);
     arange.try_into_js(&cx)
 }
 
@@ -19,14 +19,14 @@ pub fn argsort_by(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let by = params.get_external_vec::<Expr>(&cx, "by")?;
     let reverse = params.get_as::<Vec<bool>>("reverse")?;
-    functions::argsort_by(by, &reverse).try_into_js(&cx)
+    dsl::argsort_by(by, &reverse).try_into_js(&cx)
 }
 
 #[js_function(1)]
 pub fn concat_lst(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let items = params.get_external_vec::<Expr>(&cx, "items")?;
-    functions::concat_lst(items).try_into_js(&cx)
+    dsl::concat_lst(items).try_into_js(&cx)
 }
 
 #[js_function(1)]
@@ -35,29 +35,29 @@ pub fn concat_str(cx: CallContext) -> JsResult<JsExternal> {
     let items = params.get_external_vec::<Expr>(&cx, "items")?;
     let sep: &str = params.get_or("sep", ",")?;
 
-    functions::concat_str(items, sep).try_into_js(&cx)
+    dsl::concat_str(items, sep).try_into_js(&cx)
 }
 
 #[js_function(1)]
 pub fn cov(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let a = params.get_external::<Expr>(&cx, "a")?.clone();
     let b = params.get_external::<Expr>(&cx, "b")?.clone();
-    functions::cov(a, b).try_into_js(&cx)
+    dsl::cov(a, b).try_into_js(&cx)
 }
 
 #[js_function(1)]
 pub fn pearson_corr(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let a = params.get_external::<Expr>(&cx, "a")?.clone();
     let b = params.get_external::<Expr>(&cx, "b")?.clone();
-    functions::pearson_corr(a, b).try_into_js(&cx)
+    dsl::pearson_corr(a, b).try_into_js(&cx)
 }
 
 #[js_function(1)]
 pub fn spearman_rank_corr(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let a = params.get_external::<Expr>(&cx, "a")?.clone();
     let b = params.get_external::<Expr>(&cx, "b")?.clone();
-    functions::spearman_rank_corr(a, b).try_into_js(&cx)
+    dsl::spearman_rank_corr(a, b).try_into_js(&cx)
 }

nodejs-polars/src/lazy/lazyframe.rs

@@ -1,11 +1,11 @@
 use crate::conversion::prelude::*;
 use crate::prelude::{JsPolarsEr, JsResult};
 use napi::{CallContext, JsExternal, JsObject, JsString};
+use polars::io::RowCount;
 use polars::lazy::frame::{LazyCsvReader, LazyFrame, LazyGroupBy};
 use polars::lazy::prelude::col;
 use polars::prelude::NullValues;
 use polars::prelude::*;
-use polars::io::RowCount;
 
 impl IntoJs<JsExternal> for LazyFrame {
     fn try_into_js(self, cx: &CallContext) -> JsResult<JsExternal> {
@@ -92,7 +92,7 @@ pub fn scan_ipc(cx: CallContext) -> JsResult<JsExternal> {
     let cache: bool = params.get_or("cache", true)?;
     let rechunk: bool = params.get_or("rechunk", true)?;
     let row_count = params.get_as::<Option<RowCount>>("rowCount")?;
-    
+
     let args = ScanArgsIpc {
         n_rows,
         cache,
@@ -157,7 +157,12 @@ pub fn sort(cx: CallContext) -> JsResult<JsExternal> {
     let ldf = params.get_external::<LazyFrame>(&cx, "_ldf")?;
     let by_column = params.get_as::<String>("by")?;
     let reverse = params.get_or("reverse", false)?;
-    ldf.clone().sort(&by_column, reverse).try_into_js(&cx)
+    let sort_opts = SortOptions {
+        descending: reverse,
+        nulls_last: true,
+    };
+
+    ldf.clone().sort(&by_column, sort_opts).try_into_js(&cx)
 }
 
 #[js_function(1)]
@@ -492,15 +497,9 @@ pub fn drop_columns(cx: CallContext) -> JsResult<JsExternal> {
 pub fn columns(cx: CallContext) -> JsResult<JsObject> {
     let params = get_params(&cx)?;
     let ldf = params.get_external::<LazyFrame>(&cx, "_ldf")?.clone();
-    let columns: Vec<String> = ldf
-        .schema()
-        .fields()
-        .iter()
-        .map(|fld| fld.name().to_string())
-        .collect();
-    let mut arr = cx.env.create_array_with_length(columns.len())?;
-    for (idx, item) in columns.into_iter().enumerate() {
-        arr.set_element(idx as u32, cx.env.create_string_from_std(item)?)?;
+    let mut arr = cx.env.create_array()?;
+    for (idx, item) in ldf.schema().iter_names().enumerate() {
+        arr.set_element(idx as u32, cx.env.create_string_from_std(item.clone())?)?;
     }
     Ok(arr)
 }

nodejs-polars/src/series.rs

@@ -806,8 +806,11 @@ pub fn argsort(cx: CallContext) -> JsResult<JsExternal> {
     let params = get_params(&cx)?;
     let series = params.get_external::<Series>(&cx, "_series")?;
     let val = params.get_as::<bool>("reverse")?;
-
-    series.argsort(val).into_series().try_into_js(&cx)
+    let sort_opts = SortOptions {
+        descending: val,
+        nulls_last: true,
+    };
+    series.argsort(sort_opts).into_series().try_into_js(&cx)
 }
 #[js_function(1)]
 pub fn n_chunks(cx: CallContext) -> JsResult<JsNumber> {
@@ -1048,8 +1051,18 @@ impl_from_chunked_with_err!(minute);
 impl_from_chunked_with_err!(second);
 impl_from_chunked_with_err!(nanosecond);
 impl_from_chunked_with_err!(is_first);
-impl_from_chunked_with_err!(timestamp);
 
+#[js_function(1)]
+pub fn timestamp(cx: CallContext) -> JsResult<JsExternal> {
+    let params = get_params(&cx)?;
+    let series = params.get_external::<Series>(&cx, "_series")?;
+
+    series
+        .timestamp(TimeUnit::Milliseconds)
+        .map_err(JsPolarsEr::from)?
+        .into_series()
+        .try_into_js(&cx)
+}
 macro_rules! impl_method {
     ($name:ident) => {
         #[js_function(1)]

nodejs-polars/tsconfig.json

@@ -33,7 +33,7 @@
   "typedocOptions": {
     "excludePrivate": true,
     "entryPoints": [
-      "polars/series.ts",
+      "polars/series/series.ts",
       "polars/dataframe.ts",
       "polars/groupby.ts",
       "polars/io.ts",