Updating to use rs-0.30.0 (#78)

* Upgrading to rs-0.30.0 * Fixing tests --------- Co-authored-by: Darek <dchrostowski@medallia.com>
pola-rs · Jun 21, 2023 · 0875ae0 · 0875ae0
1 parent f485fea
commit 0875ae0
Show file tree

Hide file tree

Showing 24 changed files with 331 additions and 247 deletions.
diff --git a/.yarn/install-state.gz b/.yarn/install-state.gz
diff --git a/Cargo.toml b/Cargo.toml
@@ -15,11 +15,12 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 ahash = "0.8.3"
 bincode = "1.3.3"
-napi = {version = "2.13.1", default-features = false, features = ["napi8", "serde-json", "experimental"]}
+napi = {version = "2.13.2", default-features = false, features = ["napi8", "serde-json", "experimental"]}
 napi-derive = {version = "2.13.0", default-features = false}
-polars-core = {git = "https://github.com/pola-rs/polars.git", rev = "af2948a430846dae784d062a3c6f223e685af9ac", default-features = false}
-polars-ops = {git = "https://github.com/pola-rs/polars.git", rev = "af2948a430846dae784d062a3c6f223e685af9ac", default-features = false}
-polars-io = {git = "https://github.com/pola-rs/polars.git", rev = "af2948a430846dae784d062a3c6f223e685af9ac", default-features = false}
+polars-core = {git = "https://github.com/pola-rs/polars.git", rev = "ee2366b68f35c4b69dfe34cbf1eae107d4ebe97b", default-features = false}
+polars-ops = {git = "https://github.com/pola-rs/polars.git", rev = "ee2366b68f35c4b69dfe34cbf1eae107d4ebe97b", default-features = false}
+polars-io = {git = "https://github.com/pola-rs/polars.git", rev = "ee2366b68f35c4b69dfe34cbf1eae107d4ebe97b", default-features = false}
+polars-plan = {git = "https://github.com/pola-rs/polars.git", rev = "ee2366b68f35c4b69dfe34cbf1eae107d4ebe97b", default-features = false}
 thiserror = "1.0.40"
 smartstring = { version = "1" }
 serde_json = {version = "1"}
@@ -37,7 +38,6 @@ features = [
   "temporal",
   "random",
   "object",
-  "csv-file",
   "fmt",
   "performant",
   "dtype-full",
@@ -95,7 +95,7 @@ features = [
   "timezones",
 ]
 git = "https://github.com/pola-rs/polars.git"
-rev = "af2948a430846dae784d062a3c6f223e685af9ac"
+rev = "ee2366b68f35c4b69dfe34cbf1eae107d4ebe97b"
 
 [build-dependencies]
 napi-build = "2.0.1"

diff --git a/__tests__/dataframe.test.ts b/__tests__/dataframe.test.ts
@@ -1468,11 +1468,15 @@ describe("join", () => {
     expect(actual).toFrameEqual(expected);
   });
   test("asof_cross_join", () => {
-    const left = pl.DataFrame({ a: [-10, 5, 10], left_val: ["a", "b", "c"] });
-    const right = pl.DataFrame({
-      a: [1, 2, 3, 6, 7],
-      right_val: [1, 2, 3, 6, 7],
-    });
+    const left = pl
+      .DataFrame({ a: [-10, 5, 10], left_val: ["a", "b", "c"] })
+      .sort("a");
+    const right = pl
+      .DataFrame({
+        a: [1, 2, 3, 6, 7],
+        right_val: [1, 2, 3, 6, 7],
+      })
+      .sort("a");
 
     //  only test dispatch of asof join
     let out = left.joinAsof(right, { on: "a" });

diff --git a/__tests__/datelike.test.ts b/__tests__/datelike.test.ts
@@ -2,35 +2,48 @@ import pl from "@polars";
 describe("datelike", () => {
   test("asof join", () => {
     const fmt = "%F %T%.3f";
-    const quotes = pl.DataFrame({
-      dates: pl
-        .Series([
-          "2016-05-25 13:30:00.023",
-          "2016-05-25 13:30:00.023",
-          "2016-05-25 13:30:00.030",
-          "2016-05-25 13:30:00.041",
-          "2016-05-25 13:30:00.048",
-          "2016-05-25 13:30:00.049",
-          "2016-05-25 13:30:00.072",
-          "2016-05-25 13:30:00.075",
-        ])
-        .str.strptime(pl.Datetime("ms"), fmt),
-      ticker: ["GOOG", "MSFT", "MSFT", "MSFT", "GOOG", "AAPL", "GOOG", "MSFT"],
-      bid: [720.5, 51.95, 51.97, 51.99, 720.5, 97.99, 720.5, 52.01],
-    });
-    const trades = pl.DataFrame({
-      dates: pl
-        .Series([
-          "2016-05-25 13:30:00.023",
-          "2016-05-25 13:30:00.038",
-          "2016-05-25 13:30:00.048",
-          "2016-05-25 13:30:00.048",
-          "2016-05-25 13:30:00.048",
-        ])
-        .str.strptime(pl.Datetime("ms"), fmt),
-      ticker: ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
-      bid: [51.95, 51.95, 720.77, 720.92, 98.0],
-    });
+    const quotes = pl
+      .DataFrame({
+        dates: pl
+          .Series([
+            "2016-05-25 13:30:00.023",
+            "2016-05-25 13:30:00.023",
+            "2016-05-25 13:30:00.030",
+            "2016-05-25 13:30:00.041",
+            "2016-05-25 13:30:00.048",
+            "2016-05-25 13:30:00.049",
+            "2016-05-25 13:30:00.072",
+            "2016-05-25 13:30:00.075",
+          ])
+          .str.strptime(pl.Datetime("ms"), fmt),
+        ticker: [
+          "GOOG",
+          "MSFT",
+          "MSFT",
+          "MSFT",
+          "GOOG",
+          "AAPL",
+          "GOOG",
+          "MSFT",
+        ],
+        bid: [720.5, 51.95, 51.97, 51.99, 720.5, 97.99, 720.5, 52.01],
+      })
+      .sort("dates");
+    const trades = pl
+      .DataFrame({
+        dates: pl
+          .Series([
+            "2016-05-25 13:30:00.023",
+            "2016-05-25 13:30:00.038",
+            "2016-05-25 13:30:00.048",
+            "2016-05-25 13:30:00.048",
+            "2016-05-25 13:30:00.048",
+          ])
+          .str.strptime(pl.Datetime("ms"), fmt),
+        ticker: ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
+        bid: [51.95, 51.95, 720.77, 720.92, 98.0],
+      })
+      .sort("dates");
     let out: any = trades.joinAsof(quotes, { on: "dates" });
     expect(out.columns).toEqual([
       "dates",
@@ -41,8 +54,8 @@ describe("datelike", () => {
     ]);
     expect(out.getColumn("dates").cast(pl.Float64).div(1000).toArray()).toEqual(
       [
-        1464183000023, 1464183000038, 1464183000048, 1464183000048,
-        1464183000048,
+        1464183000.023, 1464183000.038, 1464183000.048, 1464183000.048,
+        1464183000.048,
       ],
     );
     out = trades

diff --git a/__tests__/expr.test.ts b/__tests__/expr.test.ts
@@ -453,8 +453,7 @@ describe("expr", () => {
     const actual = df
       .groupBy("a")
       .agg(col("b").list().keepName())
-      .sort({ by: "a" })
-      .explode("b");
+      .sort({ by: "a" });
     expect(actual).toFrameEqual(expected);
   });
   test("kurtosis", () => {
@@ -478,7 +477,7 @@ describe("expr", () => {
       a: ["a", "b", "c"],
     });
     const expected = pl.DataFrame({
-      list: [["a", "b", "c"]],
+      list: ["a", "b", "c"],
     });
     const actual = df.select(col("a").list().alias("list"));
     expect(actual).toFrameEqual(expected);

diff --git a/__tests__/groupby.test.ts b/__tests__/groupby.test.ts
@@ -162,7 +162,8 @@ describe("groupby ops", () => {
 
     const df = pl
       .DataFrame({ dt: dates, a: [3, 7, 5, 9, 2, 1] })
-      .withColumn(pl.col("dt").str.strptime(pl.Datetime("ms")));
+      .withColumn(pl.col("dt").str.strptime(pl.Datetime("ms")))
+      .sort("dt");
     const a = pl.col("a");
     const out = df
       .groupByRolling({ indexColumn: "dt", period: "2d" })
@@ -231,28 +232,31 @@ describe("groupby ops", () => {
   });
   test("default negative every offset dynamic groupby", () => {
     const dates = [
-      new Date("2020-01-01"),
-      new Date("2020-01-02"),
-      new Date("2020-02-01"),
-      new Date("2020-03-01"),
+      new Date("2023-01-01"),
+      new Date("2023-01-02"),
+      new Date("2023-02-01"),
+      new Date("2023-03-01"),
     ];
-    const df = pl.DataFrame({
-      dt: dates,
-      idx: Array.from({ length: dates.length }, (_v, k) => k),
-    });
+    const df = pl
+      .DataFrame({
+        dt: dates,
+        idx: Array.from({ length: dates.length }, (_v, k) => k),
+      })
+      .sort("dt");
     const actual = df
       .groupByDynamic({
         indexColumn: "dt",
         every: "1mo",
         closed: "right",
         start_by: "datapoint",
+        check_sorted: true,
       })
       .agg(pl.col("idx"));
     const expected = pl.DataFrame({
       dt: [
-        new Date("2019-12-30"),
-        new Date("2020-01-30"),
-        new Date("2020-02-29"),
+        new Date("2022-12-26"),
+        new Date("2023-01-26"),
+        new Date("2023-02-26"),
       ],
       idx: [[0, 1], [2], [3]],
     });

diff --git a/__tests__/io.test.ts b/__tests__/io.test.ts
@@ -330,12 +330,16 @@ describe("stream", () => {
     await expect(promise).rejects.toBeDefined();
   });
 
-  test.skip("readJSON", async () => {
+  test("readJSON", async () => {
     const readStream = new Stream.Readable({ read() {} });
-    readStream.push(`${JSON.stringify({ a: 1, b: 2 })} \n`);
-    readStream.push(`${JSON.stringify({ a: 2, b: 2 })} \n`);
-    readStream.push(`${JSON.stringify({ a: 3, b: 2 })} \n`);
-    readStream.push(`${JSON.stringify({ a: 4, b: 2 })} \n`);
+    readStream.push(
+      `${JSON.stringify([
+        { a: 1, b: 2 },
+        { a: 2, b: 2 },
+        { a: 3, b: 2 },
+        { a: 4, b: 2 },
+      ])}`,
+    );
     readStream.push(null);
 
     const expected = pl.DataFrame({

diff --git a/__tests__/lazy_functions.test.ts b/__tests__/lazy_functions.test.ts
@@ -153,9 +153,11 @@ describe("lazy functions", () => {
   });
   test("argSortBy", () => {
     const actual = _df()
-      .select(pl.argSortBy(["int_nulls", "floats"], [false, true]))
-      .getColumn("literal");
-    const expected = pl.Series("literal", [0]);
+      .select(
+        pl.argSortBy([pl.col("int_nulls"), pl.col("floats")], [false, true]),
+      )
+      .getColumn("int_nulls");
+    const expected = pl.Series("int_nulls", [1, 0, 2]);
     expect(actual).toSeriesEqual(expected);
   });
   test("avg", () => {
@@ -367,8 +369,7 @@ describe("lazy functions", () => {
     const actual = df
       .groupBy("a")
       .agg(pl.list("b").keepName())
-      .sort({ by: "a" })
-      .explode("b");
+      .sort({ by: "a" });
     expect(actual).toFrameEqual(expected);
   });
   test("mean:series", () => {

diff --git a/package.json b/package.json
@@ -58,14 +58,14 @@
     "@napi-rs/cli": "^2.16.1",
     "@types/chance": "^1.1.3",
     "@types/jest": "^29.5.2",
-    "@types/node": "^20.2.5",
+    "@types/node": "^20.3.1",
     "chance": "^1.1.11",
     "jest": "^29.5.0",
     "rome": "^12.1.3",
     "source-map-support": "^0.5.21",
     "ts-jest": "^29.1.0",
     "ts-node": "^10.9.1",
-    "typedoc": "^0.24.7",
+    "typedoc": "^0.24.8",
     "typescript": "5.1.3"
   },
   "packageManager": "yarn@3.3.1",

diff --git a/polars/dataframe.ts b/polars/dataframe.ts
@@ -1862,6 +1862,7 @@ export const _DataFrame = (_df: any): DataFrame => {
         opts.offset,
         opts.closed,
         opts.by,
+        opts.check_sorted,
       );
     },
     groupByDynamic({

diff --git a/polars/groupby.ts b/polars/groupby.ts
@@ -263,17 +263,25 @@ export interface RollingGroupBy {
 /** @ignore */
 export function RollingGroupBy(
   df: any,
-  indexColumn: string,
+  indexColumn: ColumnsOrExpr,
   period: string,
   offset?: string,
   closed?,
   by?: ColumnsOrExpr,
+  check_sorted?: boolean,
 ): RollingGroupBy {
   return {
     agg(column: ColumnsOrExpr, ...columns: ColumnsOrExpr[]) {
       return df
         .lazy()
-        .groupByRolling({ indexColumn, period, offset, closed, by } as any)
+        .groupByRolling({
+          indexColumn,
+          period,
+          offset,
+          closed,
+          by,
+          check_sorted,
+        } as any)
         .agg(column as any, ...columns)
         .collectSync();
     },

diff --git a/polars/lazy/dataframe.ts b/polars/lazy/dataframe.ts
@@ -616,11 +616,19 @@ export const _LazyDataFrame = (_ldf: any): LazyDataFrame => {
 
       return _LazyGroupBy(_ldf.groupby(by, maintainOrder));
     },
-    groupByRolling({ indexColumn, by, period, offset, closed }) {
+    groupByRolling({ indexColumn, by, period, offset, closed, check_sorted }) {
       offset = offset ?? `-${period}`;
       closed = closed ?? "right";
       by = prepareGroupbyInputs(by);
-      const lgb = _ldf.groupbyRolling(indexColumn, period, offset, closed, by);
+      check_sorted = check_sorted ?? false;
+      const lgb = _ldf.groupbyRolling(
+        pli.col(indexColumn),
+        period,
+        offset,
+        closed,
+        by,
+        check_sorted,
+      );
 
       return _LazyGroupBy(lgb);
     },
@@ -634,6 +642,7 @@ export const _LazyDataFrame = (_ldf: any): LazyDataFrame => {
       closed,
       by,
       start_by,
+      check_sorted,
     }) {
       period = period ?? every;
       offset = offset ?? `-${period}`;
@@ -642,9 +651,10 @@ export const _LazyDataFrame = (_ldf: any): LazyDataFrame => {
       truncate = truncate ?? true;
       includeBoundaries = includeBoundaries ?? false;
       start_by = start_by ?? "monday";
+      check_sorted = check_sorted ?? false;
 
       const lgb = _ldf.groupbyDynamic(
-        indexColumn,
+        pli.col(indexColumn),
         every,
         period,
         offset,
@@ -653,6 +663,7 @@ export const _LazyDataFrame = (_ldf: any): LazyDataFrame => {
         closed,
         by,
         start_by,
+        check_sorted,
       );
 
       return _LazyGroupBy(lgb);