feat: add resample operator to advanced analytic (#1349)

* chore: add resample operator to advanced analytic

* wip

* minor improvement

* wip

* fix UT

* minor fix

* udates

* fix type

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/index.ts

@@ -22,4 +22,5 @@ export { timeCompareOperator } from './timeCompareOperator';
 export { timeComparePivotOperator } from './timeComparePivotOperator';
 export { sortOperator } from './sortOperator';
 export { pivotOperator } from './pivotOperator';
+export { resampleOperator } from './resampleOperator';
 export * from './utils';

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts

@@ -0,0 +1,43 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { PostProcessingResample } from '@superset-ui/core';
+import { PostProcessingFactory } from './types';
+import { TIME_COLUMN } from './utils';
+
+export const resampleOperator: PostProcessingFactory<PostProcessingResample | undefined> = (
+  formData,
+  queryObject,
+) => {
+  const resampleZeroFill = formData.resample_method === 'zerofill';
+  const resampleMethod = resampleZeroFill ? 'asfreq' : formData.resample_method;
+  const resampleRule = formData.resample_rule;
+  if (resampleMethod && resampleRule) {
+    return {
+      operation: 'resample',
+      options: {
+        method: resampleMethod,
+        rule: resampleRule,
+        fill_value: resampleZeroFill ? 0 : null,
+        time_column: TIME_COLUMN,
+      },
+    };
+  }
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/sections/advancedAnalytics.tsx

@@ -126,5 +126,49 @@ export const advancedAnalyticsControls: ControlPanelSectionConfig = {
         },
       },
     ],
+    [<h1 className="section-header">{t('Resample')}</h1>],
+    [
+      {
+        name: 'resample_rule',
+        config: {
+          type: 'SelectControl',
+          freeForm: true,
+          label: t('Rule'),
+          default: null,
+          choices: [
+            ['1T', '1 minutely frequency'],
+            ['1H', '1 hourly frequency'],
+            ['1D', '1 calendar day frequency'],
+            ['7D', '7 calendar day frequency'],
+            ['1MS', '1 month start frequency'],
+            ['1M', '1 month end frequency'],
+            ['1AS', '1 year start frequency'],
+            ['1A', '1 year end frequency'],
+          ],
+          description: t('Pandas resample rule'),
+        },
+      },
+    ],
+    [
+      {
+        name: 'resample_method',
+        config: {
+          type: 'SelectControl',
+          freeForm: true,
+          label: t('Fill method'),
+          default: null,
+          choices: [
+            ['asfreq', 'Null imputation'],
+            ['zerofill', 'Zero imputation'],
+            ['ffill', 'Forward values'],
+            ['bfill', 'Backward values'],
+            ['median', 'Median values'],
+            ['mean', 'Mean values'],
+            ['sum', 'Sum values'],
+          ],
+          description: t('Pandas resample method'),
+        },
+      },
+    ],
   ],
 };

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { QueryObject, SqlaFormData } from '@superset-ui/core';
+import { resampleOperator } from '../../../src';
+
+const formData: SqlaFormData = {
+  metrics: ['count(*)', { label: 'sum(val)', expressionType: 'SQL', sqlExpression: 'sum(val)' }],
+  time_range: '2015 : 2016',
+  granularity: 'month',
+  datasource: 'foo',
+  viz_type: 'table',
+};
+const queryObject: QueryObject = {
+  metrics: ['count(*)', { label: 'sum(val)', expressionType: 'SQL', sqlExpression: 'sum(val)' }],
+  time_range: '2015 : 2016',
+  granularity: 'month',
+  post_processing: [
+    {
+      operation: 'pivot',
+      options: {
+        index: ['__timestamp'],
+        columns: ['nation'],
+        aggregates: {
+          'count(*)': {
+            operator: 'sum',
+          },
+        },
+      },
+    },
+  ],
+};
+
+describe('resampleOperator', () => {
+  it('should skip resampleOperator', () => {
+    expect(resampleOperator(formData, queryObject)).toEqual(undefined);
+    expect(resampleOperator({ ...formData, resample_method: 'ffill' }, queryObject)).toEqual(
+      undefined,
+    );
+    expect(resampleOperator({ ...formData, resample_rule: '1D' }, queryObject)).toEqual(undefined);
+  });
+
+  it('should do resample', () => {
+    expect(
+      resampleOperator({ ...formData, resample_method: 'ffill', resample_rule: '1D' }, queryObject),
+    ).toEqual({
+      operation: 'resample',
+      options: {
+        method: 'ffill',
+        rule: '1D',
+        fill_value: null,
+        time_column: '__timestamp',
+      },
+    });
+  });
+
+  it('should do zerofill resample', () => {
+    expect(
+      resampleOperator(
+        { ...formData, resample_method: 'zerofill', resample_rule: '1D' },
+        queryObject,
+      ),
+    ).toEqual({
+      operation: 'resample',
+      options: {
+        method: 'asfreq',
+        rule: '1D',
+        fill_value: 0,
+        time_column: '__timestamp',
+      },
+    });
+  });
+});

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/query/types/PostProcessing.ts

@@ -156,6 +156,16 @@ export interface PostProcessingSort {
   };
 }
 
+export interface PostProcessingResample {
+  operation: 'resample';
+  options: {
+    method: string;
+    rule: string;
+    fill_value?: number | null;
+    time_column: string;
+  };
+}
+
 /**
  * Parameters for chart data postprocessing.
  * See superset/utils/pandas_processing.py.
@@ -170,4 +180,5 @@ export type PostProcessingRule =
   | PostProcessingRolling
   | PostProcessingCum
   | PostProcessingCompare
-  | PostProcessingSort;
+  | PostProcessingSort
+  | PostProcessingResample;

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts

@@ -23,6 +23,7 @@ import {
   isValidTimeCompare,
   sortOperator,
   pivotOperator,
+  resampleOperator,
 } from '@superset-ui/chart-controls';
 
 export default function buildQuery(formData: QueryFormData) {
@@ -34,6 +35,7 @@ export default function buildQuery(formData: QueryFormData) {
       orderby: normalizeOrderBy(baseQueryObject).orderby,
       time_offsets: isValidTimeCompare(formData, baseQueryObject) ? formData.time_compare : [],
       post_processing: [
+        resampleOperator(formData, baseQueryObject),
         timeCompareOperator(formData, baseQueryObject),
         sortOperator(formData, { ...baseQueryObject, is_timeseries: true }),
         rollingWindowOperator(formData, baseQueryObject),