(core) Options for plugin API functions which fetch data from the sel…

…ected table or record

Summary:
Adds a new interface `FetchSelectedOptions` with three keys (including the preexisting `keepEncoded`) and adds/updates an optional `options: FetchSelectedOptions` to six related functions which fetch data from the selected table or record. The `keepEncoded` and `format` options have different default values for different methods for backwards compatibility, but otherwise the different methods now have much more similar behaviour. The new `includeColumns` option allows fetching all columns which was previously only possible using `docApi.fetchTable` (which wasn't always a great alternative) but this requires full access to avoid exposing more data than before and violating user expectations.

Eventually, similar options should be added to `docApi.fetchTable` to make the API even more consistent.

Discussion: https://grist.slack.com/archives/C0234CPPXPA/p1696510548994899

Test Plan: Added a new nbrowser test with a corresponding fixture site and document, showing how the functions have different default option values but are all configurable now.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4077

app/client/components/CustomView.ts

@@ -239,7 +239,7 @@ export class CustomView extends Disposable {
           GristDocAPIImpl.defaultAccess);
         frame.exposeAPI(
           "GristView",
-          new GristViewImpl(view), new MinimumLevel(AccessLevel.read_table));
+          new GristViewImpl(view, access), new MinimumLevel(AccessLevel.read_table));
         frame.exposeAPI(
           "CustomSectionAPI",
           new CustomSectionAPIImpl(

app/client/components/WidgetFrame.ts

@@ -11,8 +11,10 @@ import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
 import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions';
 import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes';
 import {Theme} from 'app/common/ThemePrefs';
-import {AccessTokenOptions, CursorPos, CustomSectionAPI, GristDocAPI, GristView,
-        InteractionOptionsRequest, WidgetAPI, WidgetColumnMap} from 'app/plugin/grist-plugin-api';
+import {
+  AccessTokenOptions, CursorPos, CustomSectionAPI, FetchSelectedOptions, GristDocAPI, GristView,
+  InteractionOptionsRequest, WidgetAPI, WidgetColumnMap
+} from 'app/plugin/grist-plugin-api';
 import {MsgType, Rpc} from 'grain-rpc';
 import {Computed, Disposable, dom, Observable} from 'grainjs';
 import noop = require('lodash/noop');
@@ -374,13 +376,14 @@ export class GristDocAPIImpl implements GristDocAPI {
  * GristViewAPI implemented over BaseView.
  */
 export class GristViewImpl implements GristView {
-  constructor(private _baseView: BaseView) {}
+  constructor(private _baseView: BaseView, private _access: AccessLevel) {
+  }
 
-  public async fetchSelectedTable(): Promise<any> {
+  public async fetchSelectedTable(options: FetchSelectedOptions = {}): Promise<any> {
     // If widget has a custom columns mapping, we will ignore hidden columns section.
     // Hidden/Visible columns will eventually reflect what is available, but this operation
     // is not instant - and widget can receive rows with fields that are not in the mapping.
-    const columns: ColumnRec[] = this._visibleColumns();
+    const columns: ColumnRec[] = this._visibleColumns(options);
     const rowIds = this._baseView.sortedRows.getKoArray().peek().filter(id => id != 'new');
     const data: BulkColValues = {};
     for (const column of columns) {
@@ -394,13 +397,13 @@ export class GristViewImpl implements GristView {
     return data;
   }
 
-  public async fetchSelectedRecord(rowId: number): Promise<any> {
+  public async fetchSelectedRecord(rowId: number, options: FetchSelectedOptions = {}): Promise<any> {
     // Prepare an object containing the fields available to the view
     // for the specified row.  A RECORD()-generated rendering would be
     // more useful. but the data engine needs to know what information
     // the custom view depends on, so we shouldn't volunteer any untracked
     // information here.
-    const columns: ColumnRec[] = this._visibleColumns();
+    const columns: ColumnRec[] = this._visibleColumns(options);
     const data: RowRecord = {id: rowId};
     for (const column of columns) {
       const colId: string = column.displayColModel.peek().colId.peek();
@@ -434,16 +437,32 @@ export class GristViewImpl implements GristView {
     return Promise.resolve();
   }
 
-  private _visibleColumns() {
+  private _visibleColumns(options: FetchSelectedOptions): ColumnRec[] {
     const columns: ColumnRec[] = this._baseView.viewSection.columns.peek();
-    const hiddenCols = this._baseView.viewSection.hiddenColumns.peek().map(c => c.id.peek());
-    const mappings = this._baseView.viewSection.mappedColumns.peek();
-    const mappedColumns = new Set(flatMap(Object.values(mappings || {})));
-    const notHidden = (col: ColumnRec) => !hiddenCols.includes(col.id.peek());
-    const mapped = (col: ColumnRec) => mappings && mappedColumns.has(col.colId.peek());
     // If columns are mapped, return only those that are mapped.
-    // Otherwise return all not hidden columns;
-    return mappings ? columns.filter(mapped) : columns.filter(notHidden);
+    const mappings = this._baseView.viewSection.mappedColumns.peek();
+    if (mappings) {
+      const mappedColumns = new Set(flatMap(Object.values(mappings)));
+      const mapped = (col: ColumnRec) => mappedColumns.has(col.colId.peek());
+      return columns.filter(mapped);
+    } else if (options.includeColumns === 'shown' || !options.includeColumns) {
+      // Return columns that have been shown by the user, i.e. have a corresponding view field.
+      const hiddenCols = this._baseView.viewSection.hiddenColumns.peek().map(c => c.id.peek());
+      const notHidden = (col: ColumnRec) => !hiddenCols.includes(col.id.peek());
+      return columns.filter(notHidden);
+    }
+    // These options are newer and expose more data than the user may have intended,
+    // so they require full access.
+    if (this._access !== AccessLevel.full) {
+      throwError(this._access);
+    }
+    if (options.includeColumns === 'normal') {
+      // Return all 'normal' columns of the table, regardless of whether the user has shown them.
+      return columns;
+    } else {
+      // Return *all* columns, including special invisible columns like manualSort.
+      return this._baseView.viewSection.table.peek().columns.peek().all();
+    }
   }
 }
 

app/plugin/GristAPI-ti.ts

@@ -30,9 +30,15 @@ export const GristDocAPI = t.iface([], {
   "getAccessToken": t.func("AccessTokenResult", t.param("options", "AccessTokenOptions")),
 });
 
+export const FetchSelectedOptions = t.iface([], {
+  "keepEncoded": t.opt("boolean"),
+  "format": t.opt(t.union(t.lit('rows'), t.lit('columns'))),
+  "includeColumns": t.opt(t.union(t.lit('shown'), t.lit('normal'), t.lit('all'))),
+});
+
 export const GristView = t.iface([], {
-  "fetchSelectedTable": t.func("any"),
-  "fetchSelectedRecord": t.func("any", t.param("rowId", "number")),
+  "fetchSelectedTable": t.func("any", t.param("options", "FetchSelectedOptions", true)),
+  "fetchSelectedRecord": t.func("any", t.param("rowId", "number"), t.param("options", "FetchSelectedOptions", true)),
   "allowSelectBy": t.func("void"),
   "setSelectedRows": t.func("void", t.param("rowIds", t.union(t.array("number"), "null"))),
   "setCursorPos": t.func("void", t.param("pos", "CursorPos")),
@@ -54,6 +60,7 @@ const exportedTypeSuite: t.ITypeSuite = {
   ComponentKind,
   GristAPI,
   GristDocAPI,
+  FetchSelectedOptions,
   GristView,
   AccessTokenOptions,
   AccessTokenResult,

app/plugin/GristAPI.ts

@@ -134,21 +134,54 @@ export interface GristDocAPI {
   getAccessToken(options: AccessTokenOptions): Promise<AccessTokenResult>;
 }
 
+/**
+ * Options for functions which fetch data from the selected table or record:
+ *
+ * - [[onRecords]]
+ * - [[onRecord]]
+ * - [[fetchSelectedRecord]]
+ * - [[fetchSelectedTable]]
+ * - [[GristView.fetchSelectedRecord]]
+ * - [[GristView.fetchSelectedTable]]
+ *
+ * The different methods have different default values for `keepEncoded` and `format`.
+ **/
+export interface FetchSelectedOptions {
+  /**
+   * - `true`: the returned data will contain raw `CellValue`s.
+   * - `false`: the values will be decoded, replacing e.g. `['D', timestamp]` with a moment date.
+   */
+  keepEncoded?: boolean;
+
+  /**
+   * - `rows`, the returned data will be an array of objects, one per row, with column names as keys.
+   * - `columns`, the returned data will be an object with column names as keys, and arrays of values.
+   */
+  format?: 'rows' | 'columns';
+
+  /**
+   * - `shown` (default): return only columns that are explicitly shown
+   *   in the right panel configuration of the widget. This is the only value that doesn't require full access.
+   * - `normal`: return all 'normal' columns, regardless of whether the user has shown them.
+   * - `all`: also return special invisible columns like `manualSort` and display helper columns.
+   */
+  includeColumns?: 'shown' | 'normal' | 'all';
+}
+
 /**
  * Interface for the data backing a single widget.
  */
 export interface GristView {
   /**
    * Like [[GristDocAPI.fetchTable]], but gets data for the custom section specifically, if there is any.
+   * By default, `options.keepEncoded` is `true` and `format` is `columns`.
    */
-  fetchSelectedTable(): Promise<any>;
-  // TODO: return type is Promise{[colId: string]: CellValue[]}> but cannot be specified
-  // because ts-interface-builder does not properly support index-signature.
+  fetchSelectedTable(options?: FetchSelectedOptions): Promise<any>;
 
   /**
-   * Fetches selected record by its `rowId`.
+   * Fetches selected record by its `rowId`. By default, `options.keepEncoded` is `true`.
    */
-  fetchSelectedRecord(rowId: number): Promise<any>;
+  fetchSelectedRecord(rowId: number, options?: FetchSelectedOptions): Promise<any>;
   // TODO: return type is Promise{[colId: string]: CellValue}> but cannot be specified
   // because ts-interface-builder does not properly support index-signature.
 

app/plugin/grist-plugin-api.ts

@@ -20,8 +20,10 @@
 
 import { ColumnsToMap, CustomSectionAPI, InteractionOptions, InteractionOptionsRequest,
          WidgetColumnMap } from './CustomSectionAPI';
-import { AccessTokenOptions, AccessTokenResult, GristAPI, GristDocAPI,
-         GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI';
+import {
+  AccessTokenOptions, AccessTokenResult, FetchSelectedOptions, GristAPI, GristDocAPI,
+  GristView, RPC_GRISTAPI_INTERFACE
+} from './GristAPI';
 import { RowRecord } from './GristData';
 import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI';
 import { decodeObject, mapValues } from './objtypes';
@@ -53,7 +55,34 @@ export const coreDocApi = rpc.getStub<GristDocAPI>('GristDocAPI@grist', checkers
 /**
  * Interface for the records backing a custom widget.
  */
-export const viewApi = rpc.getStub<GristView>('GristView', checkers.GristView);
+const viewApiStub = rpc.getStub<GristView>('GristView', checkers.GristView);
+export const viewApi: GristView = {
+  ...viewApiStub,
+  // Decoded objects aren't fully preserved over the RPC channel, so decoding has to happen on this side.
+  async fetchSelectedTable(options: FetchSelectedOptions = {}) {
+    let data = await viewApiStub.fetchSelectedTable(options);
+    if (options.keepEncoded === false) {
+      data = mapValues<any[], any[]>(data, (col) => col.map(decodeObject));
+    }
+    if (options.format === 'rows') {
+      const rows: RowRecord[] = [];
+      for (let i = 0; i < data.id.length; i++) {
+        const row: RowRecord = {id: data.id[i]};
+        for (const key of Object.keys(data)) {
+          row[key] = data[key][i];
+        }
+        rows.push(row);
+      }
+      return rows;
+    } else {
+      return data;
+    }
+  },
+  async fetchSelectedRecord(rowId: number, options: FetchSelectedOptions = {}) {
+    const rec = await viewApiStub.fetchSelectedRecord(rowId, options);
+    return options.keepEncoded === false ? mapValues(rec, decodeObject) : rec;
+  },
+};
 
 /**
  * Interface for the state of a custom widget.
@@ -84,25 +113,19 @@ export const setCursorPos = viewApi.setCursorPos;
 
 
 /**
- * Fetches data backing the widget as for [[GristView.fetchSelectedTable]],
- * but decoding data by default, replacing e.g. ['D', timestamp] with
- * a moment date. Option `keepEncoded` skips the decoding step.
+ * Same as [[GristView.fetchSelectedTable]], but the option `keepEncoded` is `false` by default.
  */
- export async function fetchSelectedTable(options: {keepEncoded?: boolean} = {}) {
-  const table = await viewApi.fetchSelectedTable();
-  return options.keepEncoded ? table :
-    mapValues<any[], any[]>(table, (col) => col.map(decodeObject));
+export async function fetchSelectedTable(options: FetchSelectedOptions = {}) {
+  options = {...options, keepEncoded: options.keepEncoded || false};
+  return await viewApi.fetchSelectedTable(options);
 }
 
 /**
- * Fetches current selected record as for [[GristView.fetchSelectedRecord]],
- * but decoding data by default, replacing e.g. ['D', timestamp] with
- * a moment date. Option `keepEncoded` skips the decoding step.
+ * Same as [[GristView.fetchSelectedRecord]], but the option `keepEncoded` is `false` by default.
  */
-export async function fetchSelectedRecord(rowId: number, options: {keepEncoded?: boolean} = {}) {
-  const rec = await viewApi.fetchSelectedRecord(rowId);
-  return options.keepEncoded ? rec :
-    mapValues(rec, decodeObject);
+export async function fetchSelectedRecord(rowId: number, options: FetchSelectedOptions = {}) {
+  options = {...options, keepEncoded: options.keepEncoded || false};
+  return await viewApi.fetchSelectedRecord(rowId, options);
 }
 
 
@@ -342,18 +365,34 @@ export function mapColumnNamesBack(data: any, options?: {
   return mapColumnNames(data, {...options, reverse: true});
 }
 
+/**
+ * While `fetchSelected(Record|Table)` check the access level on 'the Grist side',
+ * `onRecord(s)` needs to check this in advance for the caller to be able to handle the error.
+ */
+function checkAccessLevelForColumns(options: FetchSelectedOptions) {
+  const accessLevel = new URL(window.location.href).searchParams.get("access");
+  if (accessLevel !== "full" && options.includeColumns && options.includeColumns !== "shown") {
+    throw new Error("Access not granted. Current access level " + accessLevel);
+  }
+}
+
 /**
  * For custom widgets, add a handler that will be called whenever the
  * row with the cursor changes - either by switching to a different row, or
  * by some value within the row potentially changing.  Handler may
  * in the future be called with null if the cursor moves away from
  * any row.
+ * By default, `options.keepEncoded` is `false`.
  */
-export function onRecord(callback: (data: RowRecord | null, mappings: WidgetColumnMap | null) => unknown) {
+export function onRecord(
+  callback: (data: RowRecord | null, mappings: WidgetColumnMap | null) => unknown,
+  options: FetchSelectedOptions = {},
+) {
+  checkAccessLevelForColumns(options);
   // TODO: currently this will be called even if the content of a different row changes.
   on('message', async function(msg) {
     if (!msg.tableId || !msg.rowId || msg.rowId === 'new') { return; }
-    const rec = await docApi.fetchSelectedRecord(msg.rowId);
+    const rec = await docApi.fetchSelectedRecord(msg.rowId, options);
     callback(rec, await getMappingsIfChanged(msg));
   });
 }
@@ -372,22 +411,19 @@ export function onNewRecord(callback: (mappings: WidgetColumnMap | null) => unkn
 
 /**
  * For custom widgets, add a handler that will be called whenever the
- * selected records change.  Handler will be called with a list of records.
+ * selected records change.
+ * By default, `options.format` is `'rows'` and `options.keepEncoded` is `false`.
  */
-export function onRecords(callback: (data: RowRecord[], mappings: WidgetColumnMap | null) => unknown) {
+export function onRecords(
+  callback: (data: RowRecord[], mappings: WidgetColumnMap | null) => unknown,
+  options: FetchSelectedOptions = {},
+) {
+  checkAccessLevelForColumns(options);
+  options = {...options, format: options.format || 'rows'};
   on('message', async function(msg) {
     if (!msg.tableId || !msg.dataChange) { return; }
-    const data = await docApi.fetchSelectedTable();
-    if (!data.id) { return; }
-    const rows: RowRecord[] = [];
-    for (let i = 0; i < data.id.length; i++) {
-      const row: RowRecord = {id: data.id[i]};
-      for (const key of Object.keys(data)) {
-        row[key] = data[key][i];
-      }
-      rows.push(row);
-    }
-    callback(rows, await getMappingsIfChanged(msg));
+    const data = await docApi.fetchSelectedTable(options);
+    callback(data, await getMappingsIfChanged(msg));
   });
 }
 

test/fixtures/sites/fetchSelectedOptions/index.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <script src="/grist-plugin-api.js"></script>
+    <script src="page.js"></script>
+  </head>
+  <body>
+    <h1>FetchSelectedOptions</h1>
+    <pre id="data"></pre>
+  </body>
+</html>