Add support for auto generating metrics

packages/back-end/src/app.ts

@@ -483,6 +483,10 @@ app.get("/datasource/:id", datasourcesController.getDataSource);
 app.post("/datasources", datasourcesController.postDataSources);
 app.put("/datasource/:id", datasourcesController.putDataSource);
 app.delete("/datasource/:id", datasourcesController.deleteDataSource);
+app.post(
+  "/datasource/:id/auto-metrics",
+  datasourcesController.postAutoGeneratedMetricsToCreate
+);
 
 // Information Schemas
 app.get(

packages/back-end/src/controllers/datasources.ts

@@ -42,6 +42,9 @@ import {
 import { EventAuditUserForResponseLocals } from "../events/event-types";
 import { deleteInformationSchemaById } from "../models/InformationSchemaModel";
 import { deleteInformationSchemaTablesByInformationSchemaId } from "../models/InformationSchemaTablesModel";
+import { queueCreateAutoGeneratedMetrics } from "../jobs/createAutoGeneratedMetrics";
+import { TrackedEventData } from "../types/Integration";
+import { MetricType } from "../../types/metric";
 
 export async function postSampleData(
   req: AuthRequest,
@@ -420,14 +423,23 @@ export async function putDataSource(
       params: DataSourceParams;
       settings: DataSourceSettings;
       projects?: string[];
+      metricsToCreate?: { name: string; type: MetricType; sql: string }[];
     },
     { id: string }
   >,
   res: Response
 ) {
   const { org } = getOrgFromReq(req);
   const { id } = req.params;
-  const { name, description, type, params, settings, projects } = req.body;
+  const {
+    name,
+    description,
+    type,
+    params,
+    settings,
+    projects,
+    metricsToCreate,
+  } = req.body;
 
   const datasource = await getDataSourceById(id, org.id);
   if (!datasource) {
@@ -455,6 +467,14 @@ export async function putDataSource(
     return;
   }
 
+  if (metricsToCreate?.length) {
+    await queueCreateAutoGeneratedMetrics(
+      datasource.id,
+      org.id,
+      metricsToCreate
+    );
+  }
+
   try {
     const updates: Partial<DataSourceInterface> = {
       dateUpdated: new Date(),
@@ -602,3 +622,58 @@ export async function testLimitedQuery(
     error,
   });
 }
+
+export async function postAutoGeneratedMetricsToCreate(
+  req: AuthRequest<null, { id: string }>,
+  res: Response
+) {
+  const { org } = getOrgFromReq(req);
+  const { id } = req.params;
+
+  const dataSourceObj = await getDataSourceById(id, org.id);
+  if (!dataSourceObj) {
+    res.status(404).json({
+      status: 404,
+      message: "Invalid data source: " + id,
+    });
+    return;
+  }
+
+  req.checkPermissions(
+    "createMetrics",
+    dataSourceObj.projects?.length ? dataSourceObj.projects : ""
+  );
+
+  const integration = getSourceIntegrationObject(dataSourceObj);
+
+  try {
+    if (
+      !integration.getEventsTrackedByDatasource ||
+      !integration.settings.schemaFormat ||
+      !integration.getSourceProperties().supportsAutoGeneratedMetrics
+    ) {
+      throw new Error("Datasource does not support auto-metrics");
+    }
+
+    const trackedEvents: TrackedEventData[] = await integration.getEventsTrackedByDatasource(
+      integration.settings.schemaFormat
+    );
+
+    if (!trackedEvents.length) {
+      throw new Error("No recent events found");
+    }
+
+    return res.status(200).json({
+      status: 200,
+      trackedEvents,
+    });
+  } catch (e) {
+    res.status(200).json({
+      status: 200,
+      trackedEvents: [],
+      message:
+        "We were unable to identify any metrics to generate for you automatically. You can still create metrics manually.",
+    });
+    return;
+  }
+}

packages/back-end/src/init/queue.ts

@@ -6,6 +6,7 @@ import addMetricUpdateJob from "../jobs/updateMetrics";
 import addProxyUpdateJob from "../jobs/proxyUpdate";
 import createInformationSchemaJob from "../jobs/createInformationSchema";
 import updateInformationSchemaJob from "../jobs/updateInformationSchema";
+import createAutoGeneratedMetrics from "../jobs/createAutoGeneratedMetrics";
 import { CRON_ENABLED } from "../util/secrets";
 import { getAgendaInstance } from "../services/queueing";
 import updateStaleInformationSchemaTable from "../jobs/updateStaleInformationSchemaTable";
@@ -23,6 +24,7 @@ export async function queueInit() {
   addProxyUpdateJob(agenda);
   createInformationSchemaJob(agenda);
   updateInformationSchemaJob(agenda);
+  createAutoGeneratedMetrics(agenda);
   updateStaleInformationSchemaTable(agenda);
 
   await agenda.start();

packages/back-end/src/integrations/Athena.ts

@@ -2,13 +2,13 @@ import { decryptDataSourceParams } from "../services/datasource";
 import { runAthenaQuery } from "../services/athena";
 import { AthenaConnectionParams } from "../../types/integrations/athena";
 import { FormatDialect } from "../util/sql";
-import { MissingDatasourceParamsError } from "../types/Integration";
 import SqlIntegration from "./SqlIntegration";
 
 export default class Athena extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: AthenaConnectionParams;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<AthenaConnectionParams>(
       encryptedParams
@@ -49,14 +49,7 @@ export default class Athena extends SqlIntegration {
   ensureFloat(col: string): string {
     return `1.0*${col}`;
   }
-  getInformationSchemaFromClause(): string {
-    if (!this.params.catalog)
-      throw new MissingDatasourceParamsError(
-        "To view the information schema for an Athena data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings."
-      );
-    return `${this.params.catalog}.information_schema.columns`;
-  }
-  getInformationSchemaTableFromClause(databaseName: string): string {
-    return `${databaseName}.information_schema.columns`;
+  getDefaultDatabase() {
+    return this.params.catalog || "";
   }
 }

packages/back-end/src/integrations/BigQuery.ts

@@ -4,13 +4,13 @@ import { decryptDataSourceParams } from "../services/datasource";
 import { BigQueryConnectionParams } from "../../types/integrations/bigquery";
 import { IS_CLOUD } from "../util/secrets";
 import { FormatDialect } from "../util/sql";
-import { MissingDatasourceParamsError } from "../types/Integration";
 import SqlIntegration from "./SqlIntegration";
 
 export default class BigQuery extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: BigQueryConnectionParams;
+  requiresEscapingPath = true;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<BigQueryConnectionParams>(
       encryptedParams
@@ -82,21 +82,17 @@ export default class BigQuery extends SqlIntegration {
   castUserDateCol(column: string): string {
     return `CAST(${column} as DATETIME)`;
   }
-  getInformationSchemaFromClause(): string {
-    if (!this.params.projectId)
-      throw new Error(
-        "No projectId provided. In order to get the information schema, you must provide a projectId."
-      );
-    if (!this.params.defaultDataset)
-      throw new MissingDatasourceParamsError(
-        "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings."
-      );
-    return `\`${this.params.projectId}.${this.params.defaultDataset}.INFORMATION_SCHEMA.COLUMNS\``;
-  }
-  getInformationSchemaTableFromClause(
-    databaseName: string,
-    tableSchema: string
-  ): string {
-    return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``;
+  getDefaultDatabase() {
+    return this.params.projectId || "";
+  }
+  getDefaultSchema() {
+    return this.params.defaultDataset;
+  }
+  getInformationSchemaTable(schema?: string, database?: string): string {
+    return this.generateTablePath(
+      "INFORMATION_SCHEMA.COLUMNS",
+      schema,
+      database
+    );
   }
 }

packages/back-end/src/integrations/ClickHouse.ts

@@ -7,6 +7,8 @@ export default class ClickHouse extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: ClickHouseConnectionParams;
+  requiresDatabase = false;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<ClickHouseConnectionParams>(
       encryptedParams

packages/back-end/src/integrations/Databricks.ts

@@ -6,6 +6,8 @@ import SqlIntegration from "./SqlIntegration";
 
 export default class Databricks extends SqlIntegration {
   params!: DatabricksConnectionParams;
+  requiresDatabase = false;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<DatabricksConnectionParams>(
       encryptedParams

packages/back-end/src/integrations/Mssql.ts

@@ -1,14 +1,14 @@
 import { MssqlConnectionParams } from "../../types/integrations/mssql";
 import { decryptDataSourceParams } from "../services/datasource";
 import { FormatDialect } from "../util/sql";
-import { MissingDatasourceParamsError } from "../types/Integration";
 import { findOrCreateConnection } from "../util/mssqlPoolManager";
 import SqlIntegration from "./SqlIntegration";
 
 export default class Mssql extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: MssqlConnectionParams;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<MssqlConnectionParams>(
       encryptedParams
@@ -67,14 +67,7 @@ export default class Mssql extends SqlIntegration {
   formatDateTimeString(col: string): string {
     return `CONVERT(VARCHAR(25), ${col}, 121)`;
   }
-  getInformationSchemaFromClause(): string {
-    if (!this.params.database)
-      throw new MissingDatasourceParamsError(
-        "To view the information schema for a MS Sql dataset, you must define a default database. Please add a default database by editing the datasource's connection settings."
-      );
-    return `${this.params.database}.information_schema.columns`;
-  }
-  getInformationSchemaTableFromClause(databaseName: string): string {
-    return `${databaseName}.information_schema.columns`;
+  getDefaultDatabase() {
+    return this.params.database;
   }
 }

packages/back-end/src/integrations/Mysql.ts

@@ -9,6 +9,8 @@ export default class Mysql extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: MysqlConnectionParams;
+  requiresDatabase = false;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<MysqlConnectionParams>(
       encryptedParams

packages/back-end/src/integrations/Postgres.ts

@@ -6,6 +6,8 @@ import SqlIntegration from "./SqlIntegration";
 
 export default class Postgres extends SqlIntegration {
   params!: PostgresConnectionParams;
+  requiresDatabase = false;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<PostgresConnectionParams>(
       encryptedParams

packages/back-end/src/integrations/Presto.ts

@@ -3,7 +3,6 @@ import { Client, IPrestoClientOptions } from "presto-client";
 import { decryptDataSourceParams } from "../services/datasource";
 import { PrestoConnectionParams } from "../../types/integrations/presto";
 import { FormatDialect } from "../util/sql";
-import { MissingDatasourceParamsError } from "../types/Integration";
 import SqlIntegration from "./SqlIntegration";
 
 // eslint-disable-next-line
@@ -13,6 +12,7 @@ export default class Presto extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: PrestoConnectionParams;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<PrestoConnectionParams>(
       encryptedParams
@@ -106,14 +106,7 @@ export default class Presto extends SqlIntegration {
   ensureFloat(col: string): string {
     return `CAST(${col} AS DOUBLE)`;
   }
-  getInformationSchemaFromClause(): string {
-    if (!this.params.catalog)
-      throw new MissingDatasourceParamsError(
-        "To view the information schema for a Presto data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings."
-      );
-    return `${this.params.catalog}.information_schema.columns`;
-  }
-  getInformationSchemaTableFromClause(databaseName: string): string {
-    return `${databaseName}.information_schema.columns`;
+  getDefaultDatabase() {
+    return this.params.catalog || "";
   }
 }

packages/back-end/src/integrations/Redshift.ts

@@ -34,10 +34,7 @@ export default class Redshift extends SqlIntegration {
   ensureFloat(col: string): string {
     return `${col}::float`;
   }
-  getInformationSchemaFromClause(): string {
-    return "SVV_COLUMNS";
-  }
-  getInformationSchemaTableFromClause(): string {
+  getInformationSchemaTable(): string {
     return "SVV_COLUMNS";
   }
 }

packages/back-end/src/integrations/Snowflake.ts

@@ -8,6 +8,7 @@ export default class Snowflake extends SqlIntegration {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   params: SnowflakeConnectionParams;
+  requiresSchema = false;
   setParams(encryptedParams: string) {
     this.params = decryptDataSourceParams<SnowflakeConnectionParams>(
       encryptedParams
@@ -34,17 +35,10 @@ export default class Snowflake extends SqlIntegration {
   ensureFloat(col: string): string {
     return `CAST(${col} AS DOUBLE)`;
   }
-  getInformationSchemaFromClause(): string {
-    if (!this.params.database)
-      throw new Error(
-        "No database provided. In order to get the information schema, you must provide a database."
-      );
-    return `${this.params.database}.information_schema.columns`;
-  }
   getInformationSchemaWhereClause(): string {
     return "table_schema NOT IN ('INFORMATION_SCHEMA')";
   }
-  getInformationSchemaTableFromClause(databaseName: string): string {
-    return `${databaseName}.information_schema.columns`;
+  getDefaultDatabase() {
+    return this.params.database || "";
   }
 }