feat(self-trigger): add backend pagination for the table (#959)

* feat: add backend pagination for the table

* fix: fix first render next button

* refactor: fix CR notes

* fix: fix CR notes

* refactor: fix CR notes

* chore: add description

* refactor: add behavior property to ChartProps.ts

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/chart/models/ChartProps.ts

@@ -1,6 +1,6 @@
 import { createSelector } from 'reselect';
-import { convertKeysToCamelCase, Datasource } from '../..';
-import { HandlerFunction, PlainObject, SetExtraFormDataHook } from '../types/Base';
+import { Behavior, convertKeysToCamelCase, Datasource, JsonObject } from '../..';
+import { HandlerFunction, PlainObject, SetDataMaskHook } from '../types/Base';
 import { QueryData, DataRecordFilters } from '..';
 
 // TODO: more specific typing for these fields of ChartProps
@@ -23,8 +23,8 @@ type Hooks = {
   onError?: HandlerFunction;
   /** use the vis as control to update state */
   setControlValue?: HandlerFunction;
-  /** handle native filters */
-  setExtraFormData?: SetExtraFormDataHook;
+  /** handle external filters */
+  setDataMask?: SetDataMaskHook;
   /** handle tooltip */
   setTooltip?: HandlerFunction;
 } & PlainObject;
@@ -51,6 +51,10 @@ export interface ChartPropsConfig {
   queriesData?: QueryData[];
   /** Chart width */
   width?: number;
+  /** Own chart state of object that saved in dashboard */
+  ownCurrentState?: JsonObject;
+  /** Set of actual behaviors that this instance of chart should use */
+  behaviors?: Behavior[];
 }
 
 const DEFAULT_WIDTH = 800;
@@ -75,18 +79,24 @@ export default class ChartProps<FormData extends RawFormData = RawFormData> {
 
   hooks: Hooks;
 
+  ownCurrentState: JsonObject;
+
   queriesData: QueryData[];
 
   width: number;
 
+  behaviors: Behavior[];
+
   constructor(config: ChartPropsConfig & { formData?: FormData } = {}) {
     const {
       annotationData = {},
       datasource = {},
       formData = {} as FormData,
       hooks = {},
+      ownCurrentState = {},
       initialValues = {},
       queriesData = [],
+      behaviors = [],
       width = DEFAULT_WIDTH,
       height = DEFAULT_HEIGHT,
     } = config;
@@ -100,6 +110,8 @@ export default class ChartProps<FormData extends RawFormData = RawFormData> {
     this.hooks = hooks;
     this.initialValues = initialValues;
     this.queriesData = queriesData;
+    this.ownCurrentState = ownCurrentState;
+    this.behaviors = behaviors;
   }
 }
 
@@ -114,7 +126,20 @@ ChartProps.createSelector = function create(): ChartPropsSelector {
     input => input.initialValues,
     input => input.queriesData,
     input => input.width,
-    (annotationData, datasource, formData, height, hooks, initialValues, queriesData, width) =>
+    input => input.ownCurrentState,
+    input => input.behaviors,
+    (
+      annotationData,
+      datasource,
+      formData,
+      height,
+      hooks,
+      initialValues,
+      queriesData,
+      width,
+      ownCurrentState,
+      behaviors,
+    ) =>
       new ChartProps({
         annotationData,
         datasource,
@@ -123,7 +148,9 @@ ChartProps.createSelector = function create(): ChartPropsSelector {
         hooks,
         initialValues,
         queriesData,
+        ownCurrentState,
         width,
+        behaviors,
       }),
   );
 };

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/chart/types/Base.ts

@@ -7,14 +7,25 @@ export enum Behavior {
   NATIVE_FILTER = 'NATIVE_FILTER',
 }
 
-export type SetExtraFormDataHook = {
-  ({
-    extraFormData,
-    currentState: { value },
-  }: {
-    extraFormData: ExtraFormData;
-    currentState: { value: any; [key: string]: any };
-  }): void;
+export type DataMaskCurrentState = { value?: any; [key: string]: any };
+
+export type DataMask = {
+  nativeFilters?: {
+    extraFormData?: ExtraFormData;
+    currentState: DataMaskCurrentState;
+  };
+  crossFilters?: {
+    extraFormData?: ExtraFormData;
+    currentState: DataMaskCurrentState;
+  };
+  ownFilters?: {
+    extraFormData?: ExtraFormData;
+    currentState: { [key: string]: any };
+  };
+};
+
+export type SetDataMaskHook = {
+  ({ nativeFilters, crossFilters, ownFilters }: DataMask): void;
 };
 
 export interface PlainObject {

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/query/buildQueryObject.ts

@@ -35,7 +35,7 @@ export default function buildQueryObject<T extends QueryFormData>(
     custom_params = {},
     ...residualFormData
   } = formData;
-  const { append_form_data = {}, override_form_data = {} } = extra_form_data;
+  const { append_form_data = {}, override_form_data = {}, custom_form_data = {} } = extra_form_data;
 
   const numericRowLimit = Number(row_limit);
   const numericRowOffset = Number(row_offset);
@@ -71,5 +71,5 @@ export default function buildQueryObject<T extends QueryFormData>(
   // append and override extra form data used by native filters
   queryObject = appendExtraFormData(queryObject, append_form_data);
   queryObject = overrideExtraFormData(queryObject, override_form_data);
-  return queryObject;
+  return { ...queryObject, custom_form_data };
 }

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/query/types/QueryFormData.ts

@@ -28,6 +28,7 @@ import { AnnotationLayer } from './AnnotationLayer';
 import { QueryObject } from './Query';
 import { TimeRange, TimeRangeEndpoints } from './Time';
 import { TimeGranularity } from '../../time-format';
+import { JsonObject } from '../../connection';
 
 /**
  * Metric definition/reference in query object.
@@ -98,8 +99,12 @@ export type QueryFormExtraFilter = {
 );
 
 export type ExtraFormData = {
+  /** params that will be passed to buildQuery and will be appended to request params */
   append_form_data?: Partial<QueryObject>;
+  /** params that will be passed to buildQuery and will override request params with same name */
   override_form_data?: Partial<QueryObject>;
+  /** custom params that will be passed to buildQuery and can be used for request customization */
+  custom_form_data?: JsonObject;
 };
 
 // Type signature for formData shared by all viz types

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-table/src/DataTable/DataTable.tsx

@@ -16,7 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import React, { useCallback, useRef, ReactNode, HTMLProps, MutableRefObject } from 'react';
+import React, {
+  useCallback,
+  useRef,
+  ReactNode,
+  HTMLProps,
+  MutableRefObject,
+  CSSProperties,
+} from 'react';
 import {
   useTable,
   usePagination,
@@ -29,10 +36,14 @@ import {
   Row,
 } from 'react-table';
 import { matchSorter, rankings } from 'match-sorter';
+import { SetDataMaskHook } from '@superset-ui/core';
 import GlobalFilter, { GlobalFilterProps } from './components/GlobalFilter';
 import SelectPageSize, { SelectPageSizeProps, SizeOption } from './components/SelectPageSize';
 import SimplePagination from './components/Pagination';
 import useSticky from './hooks/useSticky';
+import { updateExternalFormData } from './utils/externalAPIs';
+import ServerPagination from './components/ServerPagination';
+import { ServerPage } from '../types';
 
 export interface DataTableProps<D extends object> extends TableOptions<D> {
   tableClassName?: string;
@@ -43,9 +54,13 @@ export interface DataTableProps<D extends object> extends TableOptions<D> {
   hooks?: PluginHook<D>[]; // any additional hooks
   width?: string | number;
   height?: string | number;
+  serverPagination?: boolean;
+  setDataMask: SetDataMaskHook;
+  currentPage?: number;
   pageSize?: number;
   noResults?: string | ((filterString: string) => ReactNode);
   sticky?: boolean;
+  showNextButton: boolean;
   wrapperRef?: MutableRefObject<HTMLDivElement>;
 }
 
@@ -58,6 +73,7 @@ export default function DataTable<D extends object>({
   tableClassName,
   columns,
   data,
+  currentPage = 0,
   width: initialWidth = '100%',
   height: initialHeight = 300,
   pageSize: initialPageSize = 0,
@@ -66,9 +82,12 @@ export default function DataTable<D extends object>({
   maxPageItemCount = 9,
   sticky: doSticky,
   searchInput = true,
+  setDataMask,
+  showNextButton,
   selectPageSize,
   noResults: noResultsText = 'No data found',
   hooks,
+  serverPagination,
   wrapperRef: userWrapperRef,
   ...moreUseTableOptions
 }: DataTableProps<D>): JSX.Element {
@@ -79,16 +98,17 @@ export default function DataTable<D extends object>({
     doSticky ? useSticky : [],
     hooks || [],
   ].flat();
+  const resultsSize = data.length;
   const sortByRef = useRef([]); // cache initial `sortby` so sorting doesn't trigger page reset
-  const pageSizeRef = useRef([initialPageSize, data.length]);
-  const hasPagination = initialPageSize > 0 && data.length > 0; // pageSize == 0 means no pagination
+  const pageSizeRef = useRef([initialPageSize, resultsSize]);
+  const hasPagination = initialPageSize > 0 && resultsSize > 0; // pageSize == 0 means no pagination
   const hasGlobalControl = hasPagination || !!searchInput;
   const initialState = {
     ...initialState_,
     // zero length means all pages, the `usePagination` plugin does not
     // understand pageSize = 0
     sortBy: sortByRef.current,
-    pageSize: initialPageSize > 0 ? initialPageSize : data.length || 10,
+    pageSize: initialPageSize > 0 ? initialPageSize : resultsSize || 10,
   };
 
   const defaultWrapperRef = useRef<HTMLDivElement>(null);
@@ -109,7 +129,7 @@ export default function DataTable<D extends object>({
     }
     return undefined;
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [initialHeight, initialWidth, wrapperRef, hasPagination, hasGlobalControl]);
+  }, [initialHeight, initialWidth, wrapperRef, hasPagination, hasGlobalControl, showNextButton]);
 
   const defaultGlobalFilter: FilterType<D> = useCallback(
     (rows: Row<D>[], columnIds: IdType<D>[], filterValue: string) => {
@@ -149,9 +169,12 @@ export default function DataTable<D extends object>({
   );
   // make setPageSize accept 0
   const setPageSize = (size: number) => {
+    if (serverPagination) {
+      updateExternalFormData(setDataMask, 0, size);
+    }
     // keep the original size if data is empty
-    if (size || data.length !== 0) {
-      setPageSize_(size === 0 ? data.length : size);
+    if (size || resultsSize !== 0) {
+      setPageSize_(size === 0 ? resultsSize : size);
     }
   };
 
@@ -206,12 +229,22 @@ export default function DataTable<D extends object>({
     pageSizeRef.current[0] !== initialPageSize ||
     // when initialPageSize stays as zero, but total number of records changed,
     // we'd also need to update page size
-    (initialPageSize === 0 && pageSizeRef.current[1] !== data.length)
+    (initialPageSize === 0 && pageSizeRef.current[1] !== resultsSize)
   ) {
-    pageSizeRef.current = [initialPageSize, data.length];
+    pageSizeRef.current = [initialPageSize, resultsSize];
     setPageSize(initialPageSize);
   }
 
+  const goToBEPage = (direction: ServerPage) => {
+    updateExternalFormData(
+      setDataMask,
+      direction === ServerPage.NEXT ? currentPage + 1 : currentPage - 1,
+      pageSize,
+    );
+  };
+
+  const paginationStyle: CSSProperties = sticky.height ? {} : { visibility: 'hidden' };
+
   return (
     <div ref={wrapperRef} style={{ width: initialWidth, height: initialHeight }}>
       {hasGlobalControl ? (
@@ -220,7 +253,7 @@ export default function DataTable<D extends object>({
             <div className="col-sm-6">
               {hasPagination ? (
                 <SelectPageSize
-                  total={data.length}
+                  total={resultsSize}
                   current={pageSize}
                   options={pageSizeOptions}
                   selectRenderer={typeof selectPageSize === 'boolean' ? undefined : selectPageSize}
@@ -242,10 +275,19 @@ export default function DataTable<D extends object>({
         </div>
       ) : null}
       {wrapStickyTable ? wrapStickyTable(renderTable) : renderTable()}
-      {hasPagination ? (
+      {serverPagination && (
+        <ServerPagination
+          ref={paginationRef}
+          style={paginationStyle}
+          showNext={showNextButton}
+          showPrevious={currentPage !== 0}
+          onPageChange={goToBEPage}
+        />
+      )}
+      {!serverPagination && hasPagination ? (
         <SimplePagination
           ref={paginationRef}
-          style={sticky.height ? undefined : { visibility: 'hidden' }}
+          style={paginationStyle}
           maxPageItemCount={maxPageItemCount}
           pageCount={pageCount}
           currentPage={pageIndex}

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-table/src/DataTable/components/ServerPagination.tsx

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React, { CSSProperties } from 'react';
+import { t } from '@superset-ui/core';
+import { ServerPage } from '../../types';
+
+export interface ServerPaginationProps {
+  showNext: boolean;
+  showPrevious: boolean;
+  onPageChange: (direction: ServerPage) => void; // `page` next/previous
+  style?: CSSProperties;
+}
+
+export default React.memo(
+  React.forwardRef(function ServerPagination(
+    { style, onPageChange, showNext, showPrevious }: ServerPaginationProps,
+    ref: React.Ref<HTMLDivElement>,
+  ) {
+    const getButton = (name: ServerPage, label: string) => (
+      <li key={name}>
+        <a
+          href={undefined}
+          role="button"
+          onClick={e => {
+            e.preventDefault();
+            onPageChange(name);
+          }}
+        >
+          {label}
+        </a>
+      </li>
+    );
+    return (
+      <div ref={ref} className="dt-pagination" style={style}>
+        <ul className="pagination pagination-sm">
+          {showPrevious && getButton(ServerPage.PREVIOUS, t('table.previous_page'))}
+          {showNext && getButton(ServerPage.NEXT, t('table.next_page'))}
+        </ul>
+      </div>
+    );
+  }),
+);