decimate transforms

docs/transforms/decimate.md

@@ -0,0 +1,7 @@
+# Decimate <VersionBadge pr="1966" />
+
+TODO
+
+## decimateX(*options*) {#decimateX}
+
+## decimateY(*options*) {#decimateY}

src/index.d.ts

@@ -47,6 +47,7 @@ export * from "./symbol.js";
 export * from "./transforms/basic.js";
 export * from "./transforms/bin.js";
 export * from "./transforms/centroid.js";
+export * from "./transforms/decimate.js";
 export * from "./transforms/dodge.js";
 export * from "./transforms/group.js";
 export * from "./transforms/hexbin.js";

src/index.js

@@ -34,6 +34,7 @@ export {valueof, column, identity, indexOf} from "./options.js";
 export {filter, reverse, sort, shuffle, basic as transform, initializer} from "./transforms/basic.js";
 export {bin, binX, binY} from "./transforms/bin.js";
 export {centroid, geoCentroid} from "./transforms/centroid.js";
+export {decimateX, decimateY} from "./transforms/decimate.js";
 export {dodgeX, dodgeY} from "./transforms/dodge.js";
 export {find, group, groupX, groupY, groupZ} from "./transforms/group.js";
 export {hexbin} from "./transforms/hexbin.js";

src/marks/area.d.ts

@@ -3,6 +3,7 @@ import type {CurveOptions} from "../curve.js";
 import type {Data, MarkOptions, RenderableMark} from "../mark.js";
 import type {BinOptions, BinReducer} from "../transforms/bin.js";
 import type {StackOptions} from "../transforms/stack.js";
+import type {DecimateOptions} from "../transforms/decimate.js";
 
 /** Options for the area, areaX, and areaY marks. */
 export interface AreaOptions extends MarkOptions, StackOptions, CurveOptions {
@@ -45,7 +46,7 @@ export interface AreaOptions extends MarkOptions, StackOptions, CurveOptions {
 }
 
 /** Options for the areaX mark. */
-export interface AreaXOptions extends Omit<AreaOptions, "y1" | "y2">, BinOptions {
+export interface AreaXOptions extends Omit<AreaOptions, "y1" | "y2">, BinOptions, DecimateOptions {
   /**
    * The horizontal position (or length) channel, typically bound to the *x*
    * scale.
@@ -85,7 +86,7 @@ export interface AreaXOptions extends Omit<AreaOptions, "y1" | "y2">, BinOptions
 }
 
 /** Options for the areaY mark. */
-export interface AreaYOptions extends Omit<AreaOptions, "x1" | "x2">, BinOptions {
+export interface AreaYOptions extends Omit<AreaOptions, "x1" | "x2">, BinOptions, DecimateOptions {
   /**
    * The horizontal position channel, typically bound to the *x* scale; defaults
    * to the zero-based index of the data [0, 1, 2, …].

src/marks/area.js

@@ -13,6 +13,7 @@ import {
 import {maybeDenseIntervalX, maybeDenseIntervalY} from "../transforms/bin.js";
 import {maybeIdentityX, maybeIdentityY} from "../transforms/identity.js";
 import {maybeStackX, maybeStackY} from "../transforms/stack.js";
+import {decimateX, decimateY} from "../transforms/decimate.js";
 
 const defaults = {
   ariaLabel: "area",
@@ -78,10 +79,10 @@ export function area(data, options) {
 
 export function areaX(data, options) {
   const {y = indexOf, ...rest} = maybeDenseIntervalY(options);
-  return new Area(data, maybeStackX(maybeIdentityX({...rest, y1: y, y2: undefined})));
+  return new Area(data, decimateY(maybeStackX(maybeIdentityX({...rest, y1: y, y2: undefined}))));
 }
 
 export function areaY(data, options) {
   const {x = indexOf, ...rest} = maybeDenseIntervalX(options);
-  return new Area(data, maybeStackY(maybeIdentityY({...rest, x1: x, x2: undefined})));
+  return new Area(data, decimateX(maybeStackY(maybeIdentityY({...rest, x1: x, x2: undefined}))));
 }

src/marks/difference.js

@@ -5,6 +5,7 @@ import {inferScaleOrder} from "../scales.js";
 import {getClipId} from "../style.js";
 import {area} from "./area.js";
 import {line} from "./line.js";
+import {decimateX} from "../transforms/decimate.js";
 
 export function differenceY(
   data,
@@ -37,48 +38,57 @@ export function differenceY(
   return marks(
     !isNoneish(positiveFill)
       ? Object.assign(
-          area(data, {
-            x1,
-            x2,
-            y1,
-            y2,
-            z,
-            fill: positiveFill,
-            fillOpacity: positiveFillOpacity,
-            render: composeRender(render, clipDifferenceY(true)),
-            clip,
-            ...options
-          }),
+          area(
+            data,
+            decimateX({
+              x1,
+              x2,
+              y1,
+              y2,
+              z,
+              fill: positiveFill,
+              fillOpacity: positiveFillOpacity,
+              render: composeRender(render, clipDifferenceY(true)),
+              clip,
+              ...options
+            })
+          ),
           {ariaLabel: "positive difference"}
         )
       : null,
     !isNoneish(negativeFill)
       ? Object.assign(
-          area(data, {
-            x1,
-            x2,
-            y1,
-            y2,
-            z,
-            fill: negativeFill,
-            fillOpacity: negativeFillOpacity,
-            render: composeRender(render, clipDifferenceY(false)),
-            clip,
-            ...options
-          }),
+          area(
+            data,
+            decimateX({
+              x1,
+              x2,
+              y1,
+              y2,
+              z,
+              fill: negativeFill,
+              fillOpacity: negativeFillOpacity,
+              render: composeRender(render, clipDifferenceY(false)),
+              clip,
+              ...options
+            })
+          ),
           {ariaLabel: "negative difference"}
         )
       : null,
-    line(data, {
-      x: x2,
-      y: y2,
-      z,
-      stroke,
-      strokeOpacity,
-      tip,
-      clip: true,
-      ...options
-    })
+    line(
+      data,
+      decimateX({
+        x: x2,
+        y: y2,
+        z,
+        stroke,
+        strokeOpacity,
+        tip,
+        clip: true,
+        ...options
+      })
+    )
   );
 }
 

src/marks/line.d.ts

@@ -3,6 +3,7 @@ import type {CurveAutoOptions} from "../curve.js";
 import type {Data, MarkOptions, RenderableMark} from "../mark.js";
 import type {MarkerOptions} from "../marker.js";
 import type {BinOptions, BinReducer} from "../transforms/bin.js";
+import type {DecimateOptions} from "../transforms/decimate.js";
 
 /** Options for the line mark. */
 export interface LineOptions extends MarkOptions, MarkerOptions, CurveAutoOptions {
@@ -25,7 +26,7 @@ export interface LineOptions extends MarkOptions, MarkerOptions, CurveAutoOption
 }
 
 /** Options for the lineX mark. */
-export interface LineXOptions extends LineOptions, BinOptions {
+export interface LineXOptions extends LineOptions, BinOptions, DecimateOptions {
   /**
    * The vertical position channel, typically bound to the *y* scale; defaults
    * to the zero-based index of the data [0, 1, 2, …].
@@ -54,7 +55,7 @@ export interface LineXOptions extends LineOptions, BinOptions {
 }
 
 /** Options for the lineY mark. */
-export interface LineYOptions extends LineOptions, BinOptions {
+export interface LineYOptions extends LineOptions, BinOptions, DecimateOptions {
   /**
    * The horizontal position channel, typically bound to the *x* scale; defaults
    * to the zero-based index of the data [0, 1, 2, …].

src/marks/line.js

@@ -12,6 +12,7 @@ import {
   groupIndex
 } from "../style.js";
 import {maybeDenseIntervalX, maybeDenseIntervalY} from "../transforms/bin.js";
+import {decimateX, decimateY} from "../transforms/decimate.js";
 
 const defaults = {
   ariaLabel: "line",
@@ -105,9 +106,9 @@ export function line(data, {x, y, ...options} = {}) {
 }
 
 export function lineX(data, {x = identity, y = indexOf, ...options} = {}) {
-  return new Line(data, maybeDenseIntervalY({...options, x, y}));
+  return new Line(data, decimateY(maybeDenseIntervalY({...options, x, y})));
 }
 
 export function lineY(data, {x = indexOf, y = identity, ...options} = {}) {
-  return new Line(data, maybeDenseIntervalX({...options, x, y}));
+  return new Line(data, decimateX(maybeDenseIntervalX({...options, x, y})));
 }

src/transforms/decimate.d.ts

@@ -0,0 +1,56 @@
+import type {Initialized} from "./basic.js";
+
+/** Options for the decimate transform. */
+export interface DecimateOptions {
+  /**
+   * The size of the decimation pixel. Defaults to 0.5, taking into account
+   * high-density displays.
+   */
+  pixelSize?: number;
+}
+
+/**
+ * Decimates a series by grouping consecutive points that share the same
+ * horizontal position (quantized by **pixelSize**), then retaining in each
+ * group a subset that includes the first and last points, and any other point
+ * necessary to cover the minimum and the maximum scaled values of the **x** and
+ * **y** channels. Additionally, the second and penultimate points are retained
+ * when the options specify a **curve** that is not guaranteed to behave
+ * monotonically.
+ *
+ * Decimation simplifies grouped marks by filtering out most of the points that
+ * do not bring any visual change to the generated path. This enables the
+ * rendering of _e.g._ time series with potentially millions of points as a path
+ * with a moderate size.
+ *
+ * ```js
+ * Plot.lineY(d3.cumsum({ length: 1_000_000 }, d3.randomNormal()), Plot.decimateX())
+ * ```
+ *
+ * The decimateX transform can be applied to any mark that consumes **x** and
+ * **y**, and is applied by default to the areaY, differenceY and lineY marks.
+ */
+export function decimateX<T>(options?: T & DecimateOptions): Initialized<T>;
+
+/**
+ * Decimates a series by grouping consecutive points that share the same
+ * vertical position (quantized by **pixelSize**), then retaining in each group
+ * a subset that includes the first and last points, and any other point
+ * necessary to cover the minimum and the maximum scaled values of the **x** and
+ * **y** channels. Additionally, the second and penultimate points are retained
+ * when the options specify a **curve** that is not guaranteed to behave
+ * monotonically.
+ *
+ * Decimation simplifies grouped marks by filtering out most of the points that
+ * do not bring any visual change to the generated path. This enables the
+ * rendering of _e.g._ time series with potentially millions of points as a path
+ * with a moderate size.
+ *
+ * ```js
+ * Plot.lineX(d3.cumsum({ length: 1_000_000 }, d3.randomNormal()), Plot.decimateY())
+ * ```
+ *
+ * The decimateY transform can be applied to any mark that consumes **x** and
+ * **y**, and is applied by default to the areaX and lineX marks.
+ */
+export function decimateY<T>(options?: T & DecimateOptions): Initialized<T>;

src/transforms/decimate.js

@@ -0,0 +1,81 @@
+import {group} from "d3";
+import {initializer} from "./basic.js";
+import {valueof} from "../options.js";
+
+// Retain the indices that share the same pixel value and correspond to first,
+// second, min X, max X, min Y, max Y, next-to-last and last values. Some known
+// curves, including the default (linear), can skip the second and penultimate.
+function decimateIndex(index, [X, Y], Z, {pixelSize = 0.5, curve} = {}) {
+  if (typeof curve === "string" && curve.match(/^(bump|linear|monotone|step)/)) curve = false;
+  const J = [];
+  const pixel = [];
+  for (const I of Z ? group(index, (i) => Z[i]).values() : [index]) {
+    let x0;
+    for (const i of I) {
+      const x = Math.floor(X[i] / pixelSize);
+      if (x !== x0) pick(), (x0 = x);
+      pixel.push(i);
+    }
+    pick();
+  }
+  return J;
+
+  function pick() {
+    const n = pixel.length;
+    if (!n) return;
+    let x1 = Infinity;
+    let y1 = Infinity;
+    let x2 = -Infinity;
+    let y2 = -Infinity;
+    let ix1, ix2, iy1, iy2;
+    let j = 0;
+    for (; j < n; ++j) {
+      const x = X[pixel[j]];
+      const y = Y[pixel[j]];
+      if (x < x1) (ix1 = j), (x1 = x);
+      if (x > x2) (ix2 = j), (x2 = x);
+      if (y < y1) (iy1 = j), (y1 = y);
+      if (y > y2) (iy2 = j), (y2 = y);
+    }
+    for (j = 0; j < n; ++j) {
+      if (
+        j === 0 ||
+        j === n - 1 ||
+        j === ix1 ||
+        j === ix2 ||
+        j === iy1 ||
+        j === iy2 ||
+        (curve && (j === 1 || j === n - 2))
+      )
+        J.push(pixel[j]);
+    }
+    pixel.length = 0;
+  }
+}
+
+function decimateK(k, pixelSize, options) {
+  if (!pixelSize) return options;
+  return initializer(options, function (data, facets, values, scales) {
+    let X = values.x2 ?? values.x ?? values.x1;
+    let Y = values.y2 ?? values.y ?? values.y1;
+    if (!X) throw new Error("missing channel x");
+    if (!Y) throw new Error("missing channel y");
+    const XY = [
+      X.scale ? valueof(X.value, scales[X.scale], Float64Array) : X.value,
+      Y.scale ? valueof(Y.value, scales[Y.scale], Float64Array) : Y.value
+    ];
+    if (k === "y") XY.reverse();
+    return {
+      data,
+      facets: facets.map((index) => decimateIndex(index, XY, values.z?.value, {pixelSize, curve: options.curve}))
+    };
+  });
+}
+
+export function decimateX({pixelSize = 0.5, ...options} = {}) {
+  return decimateK("x", pixelSize, options);
+}
+
+export function decimateY({pixelSize = 0.5, ...options} = {}) {
+  return decimateK("y", pixelSize, options);
+}