feat(advanced analysis): support MultiIndex column in post processing…

… stage (#19116)
apache · Apr 3, 2022 · 9bc7633 · 9bc7633
1 parent f8a92de
commit 9bc7633
Show file tree

Hide file tree

Showing 55 changed files with 1,269 additions and 769 deletions.
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/boxplotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/boxplotOperator.ts
@@ -21,16 +21,16 @@ import {
   getColumnLabel,
   getMetricLabel,
   PostProcessingBoxplot,
+  BoxPlotQueryObjectWhiskerType,
 } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
 
-type BoxPlotQueryObjectWhiskerType =
-  PostProcessingBoxplot['options']['whisker_type'];
 const PERCENTILE_REGEX = /(\d+)\/(\d+) percentiles/;
 
-export const boxplotOperator: PostProcessingFactory<
-  PostProcessingBoxplot | undefined
-> = (formData, queryObject) => {
+export const boxplotOperator: PostProcessingFactory<PostProcessingBoxplot> = (
+  formData,
+  queryObject,
+) => {
   const { groupby, whiskerOptions } = formData;
 
   if (whiskerOptions) {

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/contributionOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/contributionOperator.ts
@@ -19,16 +19,15 @@
 import { PostProcessingContribution } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
 
-export const contributionOperator: PostProcessingFactory<
-  PostProcessingContribution | undefined
-> = (formData, queryObject) => {
-  if (formData.contributionMode) {
-    return {
-      operation: 'contribution',
-      options: {
-        orientation: formData.contributionMode,
-      },
-    };
-  }
-  return undefined;
-};
+export const contributionOperator: PostProcessingFactory<PostProcessingContribution> =
+  (formData, queryObject) => {
+    if (formData.contributionMode) {
+      return {
+        operation: 'contribution',
+        options: {
+          orientation: formData.contributionMode,
+        },
+      };
+    }
+    return undefined;
+  };
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/flattenOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/flattenOperator.ts
@@ -0,0 +1,26 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { PostProcessingFlatten } from '@superset-ui/core';
+import { PostProcessingFactory } from './types';
+
+export const flattenOperator: PostProcessingFactory<PostProcessingFlatten> = (
+  formData,
+  queryObject,
+) => ({ operation: 'flatten' });
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/index.ts
@@ -26,4 +26,5 @@ export { resampleOperator } from './resampleOperator';
 export { contributionOperator } from './contributionOperator';
 export { prophetOperator } from './prophetOperator';
 export { boxplotOperator } from './boxplotOperator';
+export { flattenOperator } from './flattenOperator';
 export * from './utils';
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts
@@ -24,19 +24,14 @@ import {
   PostProcessingPivot,
 } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
-import { isValidTimeCompare } from './utils';
-import { timeComparePivotOperator } from './timeComparePivotOperator';
 
-export const pivotOperator: PostProcessingFactory<
-  PostProcessingPivot | undefined
-> = (formData, queryObject) => {
+export const pivotOperator: PostProcessingFactory<PostProcessingPivot> = (
+  formData,
+  queryObject,
+) => {
   const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel);
   const { x_axis: xAxis } = formData;
   if ((xAxis || queryObject.is_timeseries) && metricLabels.length) {
-    if (isValidTimeCompare(formData, queryObject)) {
-      return timeComparePivotOperator(formData, queryObject);
-    }
-
     return {
       operation: 'pivot',
       options: {
@@ -48,6 +43,8 @@ export const pivotOperator: PostProcessingFactory<
           metricLabels.map(metric => [metric, { operator: 'mean' }]),
         ),
         drop_missing_columns: false,
+        flatten_columns: false,
+        reset_index: false,
       },
     };
   }

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts
@@ -19,9 +19,10 @@
 import { DTTM_ALIAS, PostProcessingProphet } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
 
-export const prophetOperator: PostProcessingFactory<
-  PostProcessingProphet | undefined
-> = (formData, queryObject) => {
+export const prophetOperator: PostProcessingFactory<PostProcessingProphet> = (
+  formData,
+  queryObject,
+) => {
   if (formData.forecastEnabled) {
     return {
       operation: 'prophet',

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts
@@ -17,36 +17,23 @@
  * specific language governing permissions and limitationsxw
  * under the License.
  */
-import {
-  DTTM_ALIAS,
-  ensureIsArray,
-  isPhysicalColumn,
-  PostProcessingResample,
-} from '@superset-ui/core';
+import { PostProcessingResample } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
 
-export const resampleOperator: PostProcessingFactory<
-  PostProcessingResample | undefined
-> = (formData, queryObject) => {
+export const resampleOperator: PostProcessingFactory<PostProcessingResample> = (
+  formData,
+  queryObject,
+) => {
   const resampleZeroFill = formData.resample_method === 'zerofill';
   const resampleMethod = resampleZeroFill ? 'asfreq' : formData.resample_method;
   const resampleRule = formData.resample_rule;
   if (resampleMethod && resampleRule) {
-    const groupby_columns = ensureIsArray(queryObject.columns).map(column => {
-      if (isPhysicalColumn(column)) {
-        return column;
-      }
-      return column.label;
-    });
-
     return {
       operation: 'resample',
       options: {
         method: resampleMethod,
         rule: resampleRule,
         fill_value: resampleZeroFill ? 0 : null,
-        time_column: formData.x_axis || DTTM_ALIAS,
-        groupby_columns,
       },
     };
   }

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts
@@ -18,39 +18,25 @@
  * under the License.
  */
 import {
-  ComparisionType,
   ensureIsArray,
   ensureIsInt,
   PostProcessingCum,
   PostProcessingRolling,
   RollingType,
 } from '@superset-ui/core';
-import {
-  getMetricOffsetsMap,
-  isValidTimeCompare,
-  TIME_COMPARISON_SEPARATOR,
-} from './utils';
+import { getMetricOffsetsMap, isValidTimeCompare } from './utils';
 import { PostProcessingFactory } from './types';
 
 export const rollingWindowOperator: PostProcessingFactory<
-  PostProcessingRolling | PostProcessingCum | undefined
+  PostProcessingRolling | PostProcessingCum
 > = (formData, queryObject) => {
   let columns: (string | undefined)[];
   if (isValidTimeCompare(formData, queryObject)) {
     const metricsMap = getMetricOffsetsMap(formData, queryObject);
-    const comparisonType = formData.comparison_type;
-    if (comparisonType === ComparisionType.Values) {
-      // time compare type: actual values
-      columns = [
-        ...Array.from(metricsMap.values()),
-        ...Array.from(metricsMap.keys()),
-      ];
-    } else {
-      // time compare type: difference / percentage / ratio
-      columns = Array.from(metricsMap.entries()).map(([offset, metric]) =>
-        [comparisonType, metric, offset].join(TIME_COMPARISON_SEPARATOR),
-      );
-    }
+    columns = [
+      ...Array.from(metricsMap.values()),
+      ...Array.from(metricsMap.keys()),
+    ];
   } else {
     columns = ensureIsArray(queryObject.metrics).map(metric => {
       if (typeof metric === 'string') {
@@ -67,7 +53,6 @@ export const rollingWindowOperator: PostProcessingFactory<
       options: {
         operator: 'sum',
         columns: columnsMap,
-        is_pivot_df: true,
       },
     };
   }
@@ -84,7 +69,6 @@ export const rollingWindowOperator: PostProcessingFactory<
         window: ensureIsInt(formData.rolling_periods, 1),
         min_periods: ensureIsInt(formData.min_periods, 0),
         columns: columnsMap,
-        is_pivot_df: true,
       },
     };
   }

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts
@@ -20,9 +20,10 @@
 import { DTTM_ALIAS, PostProcessingSort, RollingType } from '@superset-ui/core';
 import { PostProcessingFactory } from './types';
 
-export const sortOperator: PostProcessingFactory<
-  PostProcessingSort | undefined
-> = (formData, queryObject) => {
+export const sortOperator: PostProcessingFactory<PostProcessingSort> = (
+  formData,
+  queryObject,
+) => {
   const { x_axis: xAxis } = formData;
   if (
     (xAxis || queryObject.is_timeseries) &&

diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeCompareOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeCompareOperator.ts
@@ -21,26 +21,25 @@ import { ComparisionType, PostProcessingCompare } from '@superset-ui/core';
 import { getMetricOffsetsMap, isValidTimeCompare } from './utils';
 import { PostProcessingFactory } from './types';
 
-export const timeCompareOperator: PostProcessingFactory<
-  PostProcessingCompare | undefined
-> = (formData, queryObject) => {
-  const comparisonType = formData.comparison_type;
-  const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
+export const timeCompareOperator: PostProcessingFactory<PostProcessingCompare> =
+  (formData, queryObject) => {
+    const comparisonType = formData.comparison_type;
+    const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
 
-  if (
-    isValidTimeCompare(formData, queryObject) &&
-    comparisonType !== ComparisionType.Values
-  ) {
-    return {
-      operation: 'compare',
-      options: {
-        source_columns: Array.from(metricOffsetMap.values()),
-        compare_columns: Array.from(metricOffsetMap.keys()),
-        compare_type: comparisonType,
-        drop_original_columns: true,
-      },
-    };
-  }
+    if (
+      isValidTimeCompare(formData, queryObject) &&
+      comparisonType !== ComparisionType.Values
+    ) {
+      return {
+        operation: 'compare',
+        options: {
+          source_columns: Array.from(metricOffsetMap.values()),
+          compare_columns: Array.from(metricOffsetMap.keys()),
+          compare_type: comparisonType,
+          drop_original_columns: true,
+        },
+      };
+    }
 
-  return undefined;
-};
+    return undefined;
+  };
diff --git a/...et-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts b/...et-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts
@@ -18,54 +18,40 @@
  * under the License.
  */
 import {
-  ComparisionType,
   DTTM_ALIAS,
   ensureIsArray,
   getColumnLabel,
   NumpyFunction,
   PostProcessingPivot,
 } from '@superset-ui/core';
-import {
-  getMetricOffsetsMap,
-  isValidTimeCompare,
-  TIME_COMPARISON_SEPARATOR,
-} from './utils';
+import { getMetricOffsetsMap, isValidTimeCompare } from './utils';
 import { PostProcessingFactory } from './types';
 
-export const timeComparePivotOperator: PostProcessingFactory<
-  PostProcessingPivot | undefined
-> = (formData, queryObject) => {
-  const comparisonType = formData.comparison_type;
-  const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
+export const timeComparePivotOperator: PostProcessingFactory<PostProcessingPivot> =
+  (formData, queryObject) => {
+    const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
 
-  if (isValidTimeCompare(formData, queryObject)) {
-    const valuesAgg = Object.fromEntries(
-      [...metricOffsetMap.values(), ...metricOffsetMap.keys()].map(metric => [
-        metric,
-        // use the 'mean' aggregates to avoid drop NaN
-        { operator: 'mean' as NumpyFunction },
-      ]),
-    );
-    const changeAgg = Object.fromEntries(
-      [...metricOffsetMap.entries()]
-        .map(([offset, metric]) =>
-          [comparisonType, metric, offset].join(TIME_COMPARISON_SEPARATOR),
-        )
-        // use the 'mean' aggregates to avoid drop NaN
-        .map(metric => [metric, { operator: 'mean' as NumpyFunction }]),
-    );
+    if (isValidTimeCompare(formData, queryObject)) {
+      const aggregates = Object.fromEntries(
+        [...metricOffsetMap.values(), ...metricOffsetMap.keys()].map(metric => [
+          metric,
+          // use the 'mean' aggregates to avoid drop NaN
+          { operator: 'mean' as NumpyFunction },
+        ]),
+      );
 
-    return {
-      operation: 'pivot',
-      options: {
-        index: [formData.x_axis || DTTM_ALIAS],
-        columns: ensureIsArray(queryObject.columns).map(getColumnLabel),
-        aggregates:
-          comparisonType === ComparisionType.Values ? valuesAgg : changeAgg,
-        drop_missing_columns: false,
-      },
-    };
-  }
+      return {
+        operation: 'pivot',
+        options: {
+          index: [formData.x_axis || DTTM_ALIAS],
+          columns: ensureIsArray(queryObject.columns).map(getColumnLabel),
+          drop_missing_columns: false,
+          flatten_columns: false,
+          reset_index: false,
+          aggregates,
+        },
+      };
+    }
 
-  return undefined;
-};
+    return undefined;
+  };