feat: Show used pre-aggregations and match rollup results in Playground

paveltiunov · paveltiunov · commit 4a673463b88c · 2019-11-25T23:23:04.000-08:00
diff --git a/packages/cubejs-api-gateway/index.js b/packages/cubejs-api-gateway/index.js
@@ -274,7 +274,10 @@ class ApiGateway {
     try {
       query = this.parseQueryParam(query);
       const normalizedQuery = await this.queryTransformer(normalizeQuery(query), context);
-      const sqlQuery = await this.getCompilerApi(context).getSql(coerceForSqlQuery(normalizedQuery, context));
+      const sqlQuery = await this.getCompilerApi(context).getSql(
+        coerceForSqlQuery(normalizedQuery, context),
+        { includeDebugInfo: process.env.NODE_ENV !== 'production' }
+      );
       res({
         sql: sqlQuery
       });
@@ -323,7 +326,10 @@ class ApiGateway {
         query: normalizedQuery,
         data: transformData(aliasToMemberNameMap, flattenAnnotation, response.data),
         lastRefreshTime: response.lastRefreshTime && response.lastRefreshTime.toISOString(),
-        refreshKeyValues: process.env.NODE_ENV === 'production' ? undefined : response.refreshKeyValues,
+        ...(process.env.NODE_ENV === 'production' ? undefined : {
+          refreshKeyValues: response.refreshKeyValues,
+          usedPreAggregations: response.usedPreAggregations
+        }),
         annotation
       });
     } catch (e) {
diff --git a/packages/cubejs-playground/src/ChartContainer.jsx b/packages/cubejs-playground/src/ChartContainer.jsx
@@ -11,6 +11,7 @@ import { QueryRenderer } from '@cubejs-client/react';
 import sqlFormatter from "sql-formatter";
 import PropTypes from 'prop-types';
 import PrismCode from './PrismCode';
+import CachePane from './components/CachePane';
 import { playgroundAction } from './events';
 
 export const frameworks = [{
@@ -296,47 +297,9 @@ class ChartContainer extends React.Component {
         );
       } else if (showCode === 'cache') {
         return (
-          <QueryRenderer
-            loadSql
-            query={{ ...query, renewQuery: true }}
+          <CachePane
+            query={query}
             cubejsApi={cubejsApi}
-            render={
-              ({ sqlQuery, resultSet: rs }) => (
-                <div>
-                  <h3>
-                    Last Refresh Time:
-                    {rs && rs.loadResponse.lastRefreshTime}
-                  </h3>
-                  <Table
-                    loading={!sqlQuery}
-                    pagination={false}
-                    columns={[
-                      {
-                        title: 'Refresh Key SQL',
-                        key: 'refreshKey',
-                        render: (text, record) => <PrismCode code={sqlFormatter.format(record[0])} />,
-                      },
-                      {
-                        title: 'Value',
-                        key: 'value',
-                        render: (text, record) => (
-                          <PrismCode
-                            code={
-                              rs && rs.loadResponse.refreshKeyValues
-                              && JSON.stringify(rs.loadResponse.refreshKeyValues[
-                                sqlQuery.sqlQuery.sql.cacheKeyQueries.queries.indexOf(record)
-                              ], null, 2)
-                            }
-                          />
-                        ),
-                      }
-                    ]}
-                    dataSource={
-                      sqlQuery && sqlQuery.sqlQuery.sql && sqlQuery.sqlQuery.sql.cacheKeyQueries.queries
-                    }
-                  />
-                </div>
-              )}
           />
         );
       }
diff --git a/packages/cubejs-playground/src/components/CachePane.js b/packages/cubejs-playground/src/components/CachePane.js
@@ -0,0 +1,136 @@
+import React from 'react';
+import {
+  Table, Icon, Tabs
+} from 'antd';
+import { QueryRenderer } from '@cubejs-client/react';
+import sqlFormatter from "sql-formatter";
+import PropTypes from 'prop-types';
+import PrismCode from '../PrismCode';
+
+const CachePane = ({ query, cubejsApi }) => (
+  <QueryRenderer
+    loadSql
+    query={{ ...query, renewQuery: true }}
+    cubejsApi={cubejsApi}
+    render={
+      ({ sqlQuery, resultSet: rs }) => (
+        <Tabs
+          defaultActiveKey="refreshKeys"
+          tabBarExtraContent={(
+            <span>
+              Last Refresh Time:&nbsp;
+              <b>{rs && rs.loadResponse.lastRefreshTime}</b>
+            </span>
+          )}
+        >
+          <Tabs.TabPane tab="Refresh Keys" key="refreshKeys">
+            <Table
+              loading={!sqlQuery}
+              pagination={false}
+              scroll={{ x: true }}
+              columns={[
+                {
+                  title: 'Refresh Key SQL',
+                  key: 'refreshKey',
+                  render: (text, record) => <PrismCode code={sqlFormatter.format(record[0])} />,
+                },
+                {
+                  title: 'Value',
+                  key: 'value',
+                  render: (text, record) => (
+                    <PrismCode
+                      code={
+                        rs && rs.loadResponse.refreshKeyValues
+                        && JSON.stringify(rs.loadResponse.refreshKeyValues[
+                          sqlQuery.sqlQuery.sql.cacheKeyQueries.queries.indexOf(record)
+                        ], null, 2)
+                      }
+                    />
+                  ),
+                }
+              ]}
+              dataSource={
+                sqlQuery && sqlQuery.sqlQuery.sql && sqlQuery.sqlQuery.sql.cacheKeyQueries.queries
+              }
+            />
+          </Tabs.TabPane>
+          <Tabs.TabPane tab="Pre-aggregations" key="preAggregations">
+            <Table
+              loading={!sqlQuery}
+              pagination={false}
+              scroll={{ x: true }}
+              columns={[
+                {
+                  title: 'Table Name',
+                  key: 'tableName',
+                  dataIndex: 'tableName',
+                  render: (text) => <b>{text}</b>
+                },
+                {
+                  title: 'Refresh Key SQL',
+                  key: 'refreshKey',
+                  dataIndex: 'invalidateKeyQueries',
+                  render: (refreshKeyQueries) => refreshKeyQueries
+                    .map(q => <PrismCode key={q[0]} code={sqlFormatter.format(q[0])} />),
+                },
+                {
+                  title: 'Refresh Key Value',
+                  key: 'value',
+                  render: (text, record) => rs && rs.loadResponse.usedPreAggregations
+                    && rs.loadResponse.usedPreAggregations[record.tableName].refreshKeyValues.map(k => (
+                      <PrismCode
+                        key={JSON.stringify(k)}
+                        code={JSON.stringify(k, null, 2)}
+                      />
+                    )),
+                }
+              ]}
+              dataSource={
+                sqlQuery && sqlQuery.sqlQuery.sql && sqlQuery.sqlQuery.sql.preAggregations
+              }
+            />
+          </Tabs.TabPane>
+          <Tabs.TabPane tab="Rollup Match Results" key="rollupMatchResults">
+            <Table
+              loading={!sqlQuery}
+              pagination={false}
+              scroll={{ x: true }}
+              columns={[
+                {
+                  title: 'Rollup Table Name',
+                  key: 'tableName',
+                  dataIndex: 'tableName',
+                  render: (text) => <b>{text}</b>
+                },
+                {
+                  title: 'Rollup Definition',
+                  key: 'rollup',
+                  dataIndex: 'references',
+                  render: (text) => <PrismCode code={JSON.stringify(text, null, 2)} />,
+                },
+                {
+                  title: 'Can Be Used',
+                  key: 'canUsePreAggregation',
+                  dataIndex: 'canUsePreAggregation',
+                  render: (text) => (
+                    text ? <Icon type="check" style={{ color: '#52c41a', fontSize: '2em' }}/>
+                      : <Icon type="close" style={{ color: '#c2371b', fontSize: '2em' }}/>
+                  ),
+                }
+              ]}
+              dataSource={
+                sqlQuery && sqlQuery.sqlQuery.sql && sqlQuery.sqlQuery.sql.rollupMatchResults
+              }
+            />
+          </Tabs.TabPane>
+        </Tabs>
+      )}
+  />
+);
+
+CachePane.propTypes = {
+  query: PropTypes.object.isRequired,
+  cubejsApi: PropTypes.object.isRequired
+};
+
+export default CachePane;
diff --git a/packages/cubejs-playground/src/prism.css b/packages/cubejs-playground/src/prism.css
@@ -55,7 +55,7 @@ pre[class*="language-"] {
 
 :not(pre) > code[class*="language-"],
 pre[class*="language-"] {
-	background: #fff;
+	background: none;
 }
 
 /* Inline code */
diff --git a/packages/cubejs-query-orchestrator/orchestrator/PreAggregations.js b/packages/cubejs-query-orchestrator/orchestrator/PreAggregations.js
@@ -187,15 +187,15 @@ class PreAggregationLoader {
         return this.loadPreAggregationWithKeys();
       }
     } else {
-      return this.loadPreAggregationWithKeys();
+      return {
+        targetTableName: await this.loadPreAggregationWithKeys(),
+        refreshKeyValues: await this.getInvalidationKeyValues()
+      };
     }
   }
 
   async loadPreAggregationWithKeys() {
-    const invalidationKeys = await Promise.all(
-      (this.preAggregation.invalidateKeyQueries || [])
-        .map(keyQuery => this.loadCache.keyQueryResult(keyQuery))
-    );
+    const invalidationKeys = await this.getInvalidationKeyValues();
     const contentVersion = version([this.preAggregation.loadSql, invalidationKeys]);
     const structureVersion = version(this.preAggregation.loadSql);
 
@@ -275,6 +275,13 @@ class PreAggregationLoader {
     return this.targetTableName(versionEntry);
   }
 
+  getInvalidationKeyValues() {
+    return Promise.all(
+      (this.preAggregation.invalidateKeyQueries || [])
+        .map(keyQuery => this.loadCache.keyQueryResult(keyQuery))
+    );
+  }
+
   scheduleRefresh(invalidationKeys, newVersionEntry) {
     this.logger('Refreshing pre-aggregation content', { preAggregation: this.preAggregation });
     this.executeInQueue(invalidationKeys, 0, newVersionEntry)
@@ -429,9 +436,10 @@ class PreAggregations {
         loadCache,
         { waitForRenew: queryBody.renewQuery }
       );
-      const preAggregationPromise = () => loader.loadPreAggregation().then(async tempTableName => {
-        await this.addTableUsed(tempTableName);
-        return [p.tableName, tempTableName];
+      const preAggregationPromise = () => loader.loadPreAggregation().then(async targetTableName => {
+        const usedPreAggregation = typeof targetTableName === 'string' ? { targetTableName } : targetTableName;
+        await this.addTableUsed(usedPreAggregation.targetTableName);
+        return [p.tableName, usedPreAggregation];
       });
       return preAggregationPromise().then(res => preAggregationsTablesToTempTables.concat([res]));
     }).reduce((promise, fn) => promise.then(fn), Promise.resolve([]));
diff --git a/packages/cubejs-query-orchestrator/orchestrator/QueryCache.js b/packages/cubejs-query-orchestrator/orchestrator/QueryCache.js
@@ -90,7 +90,7 @@ class QueryCache {
   static replacePreAggregationTableNames(queryAndParams, preAggregationsTablesToTempTables) {
     const [keyQuery, params] = Array.isArray(queryAndParams) ? queryAndParams : [queryAndParams, []];
     const replacedKeqQuery = preAggregationsTablesToTempTables.reduce(
-      (query, [tableName, tempTable]) => QueryCache.replaceAll(tableName, tempTable, query),
+      (query, [tableName, { targetTableName }]) => QueryCache.replaceAll(tableName, targetTableName, query),
       keyQuery
     );
     return Array.isArray(queryAndParams) ? [replacedKeqQuery, params] : replacedKeqQuery;
diff --git a/packages/cubejs-query-orchestrator/orchestrator/QueryOrchestrator.js b/packages/cubejs-query-orchestrator/orchestrator/QueryOrchestrator.js
@@ -1,3 +1,4 @@
+const R = require('ramda');
 const QueryCache = require('./QueryCache');
 const PreAggregations = require('./PreAggregations');
 
@@ -33,9 +34,15 @@ class QueryOrchestrator {
 
   async fetchQuery(queryBody) {
     return this.preAggregations.loadAllPreAggregationsIfNeeded(queryBody)
-      .then(preAggregationsTablesToTempTables => this.queryCache.cachedQueryResult(
-        queryBody, preAggregationsTablesToTempTables
-      ));
+      .then(async preAggregationsTablesToTempTables => {
+        const result = await this.queryCache.cachedQueryResult(
+          queryBody, preAggregationsTablesToTempTables
+        );
+        return {
+          ...result,
+          usedPreAggregations: R.fromPairs(preAggregationsTablesToTempTables)
+        };
+      });
   }
 
   async queryStage(queryBody) {
diff --git a/packages/cubejs-query-orchestrator/test/QueryOrchestratorTest.js b/packages/cubejs-query-orchestrator/test/QueryOrchestratorTest.js
@@ -0,0 +1,53 @@
+/* globals describe, it, should */
+const QueryOrchestrator = require('../orchestrator/QueryOrchestrator');
+
+class MockDriver {
+  constructor() {
+    this.tables = [];
+  }
+
+  async query(query) {
+    return [query];
+  }
+
+  async getTablesQuery(schema) {
+    return this.tables.map(t => ({ table_name: t.replace(`${schema}.`, '') }));
+  }
+
+  async createSchemaIfNotExists(schema) {
+    this.schema = schema;
+    return null;
+  }
+
+  async loadPreAggregationIntoTable(preAggregationTableName) {
+    this.tables.push(preAggregationTableName);
+  }
+}
+
+describe('QueryOrchestrator', () => {
+  const mockDriver = new MockDriver();
+  const queryOrchestrator = new QueryOrchestrator(
+    'TEST', async () => mockDriver, (msg, params) => console.log(msg, params)
+  );
+
+  it('basic', async () => {
+    const query = {
+      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
+      values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
+      cacheKeyQueries: {
+        renewalThreshold: 21600,
+        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
+      },
+      preAggregations: [{
+        preAggregationsSchema: "stb_pre_aggregations",
+        tableName: "stb_pre_aggregations.orders_number_and_count20191101",
+        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
+        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
+      }],
+      renewQuery: true
+    };
+    const result = await queryOrchestrator.fetchQuery(query);
+    console.log(result.data[0]);
+    should(result.data[0]).match(/orders_number_and_count20191101_kjypcoio_5yftl5il/);
+  });
+});
diff --git a/packages/cubejs-query-orchestrator/test/QueryQueueTest.js b/packages/cubejs-query-orchestrator/test/QueryQueueTest.js
@@ -1,6 +1,6 @@
+/* globals describe,it */
 const QueryQueue = require('../orchestrator/QueryQueue');
 const should = require('should');
-const redis = require('redis');
 
 const QueryQueueTest = (name, options) => {
   describe(`QueryQueue${name}`, function () {
@@ -16,7 +16,7 @@ const QueryQueueTest = (name, options) => {
           const result = query.result + delayCount;
           delayCount += 1;
           await setCancelHandler(result);
-          return await delayFn(result, query.delay);
+          return delayFn(result, query.delay);
         }
       },
       cancelHandlers: {
@@ -57,6 +57,7 @@ const QueryQueueTest = (name, options) => {
           await queue.executeInQueue('delay', query, { delay: 3000, result: '1' });
         } catch (e) {
           if (e.message === 'Continue wait') {
+            // eslint-disable-next-line no-continue
             continue;
           }
           errorString = e.toString();
@@ -89,7 +90,7 @@ const QueryQueueTest = (name, options) => {
 
     it('orphaned', async () => {
       for (let i = 1; i <= 4; i++) {
-        await queue.executeInQueue('delay', `11` + i, { delay: 50, result: '' + i }, 0);
+        await queue.executeInQueue('delay', `11` + i, { delay: 50, result: `${i}` }, 0);
       }
       cancelledQuery = null;
       delayCount = 0;
diff --git a/packages/cubejs-schema-compiler/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/adapter/PreAggregations.js
diff --git a/packages/cubejs-server-core/core/CompilerApi.js b/packages/cubejs-server-core/core/CompilerApi.js

Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,7 @@ pre[class*="language-"] {`
`55`	`55`
`56`	`56`	`:not(pre) > code[class*="language-"],`
`57`	`57`	`pre[class*="language-"] {`
`58`		`- background: #fff;`
	`58`	`+ background: none;`
`59`	`59`	`}`
`60`	`60`
`61`	`61`	`/* Inline code */`