From 8b2b643306e9ff601955ef73c3e85c90ff3ad49c Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 12 May 2023 16:10:45 -0400 Subject: [PATCH 01/39] POC for auto-generating metrics from Segment/BigQuery --- packages/back-end/src/app.ts | 1 + .../back-end/src/controllers/datasources.ts | 24 +++- packages/back-end/src/controllers/metrics.ts | 58 +++++++++ packages/back-end/src/init/queue.ts | 2 + .../back-end/src/integrations/BigQuery.ts | 11 ++ .../src/integrations/SqlIntegration.ts | 26 ++++ .../src/jobs/createAutomaticMetrics.ts | 77 ++++++++++++ packages/back-end/src/types/Integration.ts | 3 + .../components/Settings/NewDataSourceForm.tsx | 112 +++++++++++++++++- 9 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 packages/back-end/src/jobs/createAutomaticMetrics.ts diff --git a/packages/back-end/src/app.ts b/packages/back-end/src/app.ts index 64154cd6661..41227826841 100644 --- a/packages/back-end/src/app.ts +++ b/packages/back-end/src/app.ts @@ -346,6 +346,7 @@ app.get( metricsController.getMetricAnalysisStatus ); app.post("/metric/:id/analysis/cancel", metricsController.cancelMetricAnalysis); +app.get("/metrics/generate/:dataSourceId", metricsController.getAutoMetrics); // Experiments app.get("/experiments", experimentsController.getExperiments); diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index ad42fc81684..f621088abad 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import { Response } from "express"; import uniqid from "uniqid"; import { @@ -44,6 +45,7 @@ import { import { EventAuditUserForResponseLocals } from "../events/event-types"; import { deleteInformationSchemaById } from "../models/InformationSchemaModel"; import { deleteInformationSchemaTablesByInformationSchemaId } from "../models/InformationSchemaTablesModel"; +import { queueCreateAutomaticMetrics } from "../jobs/createAutomaticMetrics"; export async function postSampleData( req: AuthRequest, @@ -418,6 +420,11 @@ export async function putDataSource( params: DataSourceParams; settings: DataSourceSettings; projects?: string[]; + metricsToCreate?: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }[]; }, { id: string } >, @@ -425,7 +432,18 @@ export async function putDataSource( ) { const { org } = getOrgFromReq(req); const { id } = req.params; - const { name, description, type, params, settings, projects } = req.body; + const { + name, + description, + type, + params, + settings, + projects, + metricsToCreate, + } = req.body; + + console.log("made it to the putDataSource method"); + console.log("metricsToCreate", metricsToCreate); const datasource = await getDataSourceById(id, org.id); if (!datasource) { @@ -453,6 +471,10 @@ export async function putDataSource( return; } + if (metricsToCreate?.length) { + await queueCreateAutomaticMetrics(datasource.id, org.id, metricsToCreate); + } + try { const updates: Partial = { dateUpdated: new Date(), diff --git a/packages/back-end/src/controllers/metrics.ts b/packages/back-end/src/controllers/metrics.ts index 272126412a9..d2340709b86 100644 --- a/packages/back-end/src/controllers/metrics.ts +++ b/packages/back-end/src/controllers/metrics.ts @@ -31,6 +31,7 @@ import { auditDetailsDelete, } from "../services/audit"; import { EventAuditUserForResponseLocals } from "../events/event-types"; +import { getSourceIntegrationObject } from "../services/datasource"; export async function deleteMetric( req: AuthRequest, @@ -458,3 +459,60 @@ export async function putMetric( }), }); } + +export async function getAutoMetrics( + req: AuthRequest, + res: Response +) { + const { org } = getOrgFromReq(req); + const { dataSourceId } = req.params; + + const dataSourceObj = await getDataSourceById(dataSourceId, org.id); + if (!dataSourceObj) { + res.status(403).json({ + status: 403, + message: "Invalid data source: " + dataSourceId, + }); + return; + } + + req.checkPermissions( + "createMetrics", + dataSourceObj.projects?.length ? dataSourceObj.projects : "" + ); + + const integration = getSourceIntegrationObject(dataSourceObj); + + if (!integration.getTrackedEvents) { + //TODO: Is this the correct error code? + res.status(403).json({ + status: 403, + message: "This datasource does not support automatic metric generation.", + }); + return; + } + + try { + const results = await integration.getTrackedEvents(); + + if (results.length) { + return res.status(200).json({ + status: 200, + results, + }); + } else { + return res.status(200).json({ + status: 200, + results, + message: "No events found.", + }); + } + } catch (e) { + res.status(400).json({ + status: 400, + results: [], + message: e.message, + }); + return; + } +} diff --git a/packages/back-end/src/init/queue.ts b/packages/back-end/src/init/queue.ts index 24e64e14bab..3fc227980f7 100644 --- a/packages/back-end/src/init/queue.ts +++ b/packages/back-end/src/init/queue.ts @@ -6,6 +6,7 @@ import addMetricUpdateJob from "../jobs/updateMetrics"; import addProxyUpdateJob from "../jobs/proxyUpdate"; import createInformationSchemaJob from "../jobs/createInformationSchema"; import updateInformationSchemaJob from "../jobs/updateInformationSchema"; +import createAutomaticMetrics from "../jobs/createAutomaticMetrics"; import { CRON_ENABLED } from "../util/secrets"; import { getAgendaInstance } from "../services/queueing"; import updateStaleInformationSchemaTable from "../jobs/updateStaleInformationSchemaTable"; @@ -23,6 +24,7 @@ export async function queueInit() { addProxyUpdateJob(agenda); createInformationSchemaJob(agenda); updateInformationSchemaJob(agenda); + createAutomaticMetrics(agenda); updateStaleInformationSchemaTable(agenda); await agenda.start(); diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 6ac5cfd0893..f6a96265d8a 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -99,4 +99,15 @@ export default class BigQuery extends SqlIntegration { ): string { return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``; } + getTrackedEventsFromClause(): string { + if (!this.params.projectId) + throw new Error( + "No projectId provided. In order to get the information schema, you must provide a projectId." + ); + if (!this.params.defaultDataset) + throw new MissingDatasourceParamsError( + "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." + ); + return `\`${this.params.projectId}.${this.params.defaultDataset}.tracks\``; + } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index b99199afd60..6164ab53418 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1288,6 +1288,32 @@ export default abstract class SqlIntegration return { tableData }; } + getTrackedEventsFromClause(): string { + return "TODO"; + } + + async getTrackedEvents(): Promise< + { event: string; hasUserId: boolean; createForUser: boolean }[] + > { + const sql = ` + SELECT + event, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId + FROM + ${this.getTrackedEventsFromClause()} + GROUP BY event`; + + const results = await this.runQuery(format(sql, this.getFormatDialect())); + + results.forEach((result) => (result.createForUser = true)); + + if (!results) { + throw new Error(`No events found.`); + } + + return results; + } + private getMetricQueryFormat(metric: MetricInterface) { return metric.queryFormat || (metric.sql ? "sql" : "builder"); } diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts new file mode 100644 index 00000000000..396264b8155 --- /dev/null +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -0,0 +1,77 @@ +import Agenda, { Job } from "agenda"; +import uniqid from "uniqid"; +import { getDataSourceById } from "../models/DataSourceModel"; +import { insertMetric } from "../models/MetricModel"; +import { MetricInterface } from "../../types/metric"; + +const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; + +type CreateAutomaticMetricsJob = Job<{ + organization: string; + datasourceId: string; + metricsToCreate: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }[]; +}>; + +let agenda: Agenda; +export default function (ag: Agenda) { + agenda = ag; + + agenda.define( + CREATE_AUTOMATIC_METRICS_JOB_NAME, + async (job: CreateAutomaticMetricsJob) => { + const { datasourceId, organization, metricsToCreate } = job.attrs.data; + + if (!datasourceId || !organization || !metricsToCreate) return; + + const datasource = await getDataSourceById(datasourceId, organization); + + if (!datasource) return; + + try { + // This is where I guess I'll loop through the metricsToCreate & call the createMetric function? + const createdMetrics = metricsToCreate.map(async (metric) => { + const data: Partial = { + id: uniqid("met_"), + organization, + datasource: datasourceId, + name: metric.event, + type: "binomial", + sql: "SELECT * FROM SOMETHING", + dateCreated: new Date(), + dateUpdated: new Date(), + conversionWindowHours: 72, + }; + return await insertMetric(data); + }); + console.log("createdMetrics", createdMetrics); + } catch (e) { + // Not sure what to do here yet - catch the errors, but what should I do with them? + } + } + ); +} + +export async function queueCreateAutomaticMetrics( + datasourceId: string, + organization: string, + metricsToCreate: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }[] +) { + if (!datasourceId || !organization || !metricsToCreate) return; + + const job = agenda.create(CREATE_AUTOMATIC_METRICS_JOB_NAME, { + organization, + datasourceId, + metricsToCreate, + }) as CreateAutomaticMetricsJob; + job.unique({ datasourceId, organization, metricsToCreate }); + job.schedule(new Date()); + await job.save(); +} diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 4f898ecda00..99e65080483 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -292,4 +292,7 @@ export interface SourceIntegrationInterface { query: string ): Promise; runPastExperimentQuery(query: string): Promise; + getTrackedEvents?: () => Promise< + { event: string; hasUserId: boolean; createForUser: boolean }[] + >; } diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index d36ab4b566d..bb5ba54bbf8 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -5,8 +5,12 @@ import { ChangeEventHandler, ReactElement, } from "react"; -import { DataSourceInterfaceWithParams } from "back-end/types/datasource"; +import { + DataSourceInterfaceWithParams, + DataSourceSettings, +} from "back-end/types/datasource"; import { useForm } from "react-hook-form"; +import { cloneDeep } from "lodash"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -22,6 +26,8 @@ import SelectField from "../Forms/SelectField"; import Field from "../Forms/Field"; import Modal from "../Modal"; import { GBCircleArrowLeft } from "../Icons"; +import Button from "../Button"; +import Toggle from "../Forms/Toggle"; import EventSourceList from "./EventSourceList"; import ConnectionSettings from "./ConnectionSettings"; @@ -50,9 +56,17 @@ const NewDataSourceForm: FC<{ const [dataSourceId, setDataSourceId] = useState( data?.id || null ); + const [autoMetricError, setAutoMetricError] = useState(""); const [possibleTypes, setPossibleTypes] = useState( dataSourceConnections.map((d) => d.type) ); + const [metricsToCreate, setMetricsToCreate] = useState< + { + event: string; + hasUserId: boolean; + createForUser: boolean; + }[] + >([]); const permissions = usePermissions(); @@ -64,9 +78,18 @@ const NewDataSourceForm: FC<{ name: "My Datasource", settings: {}, }; - const form = useForm({ + + const form = useForm<{ + settings: DataSourceSettings | undefined; + metricsToCreate: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }[]; + }>({ defaultValues: { settings: data?.settings || DEFAULT_DATA_SOURCE.settings, + metricsToCreate: [], }, }); const schemasMap = new Map(); @@ -88,6 +111,10 @@ const NewDataSourceForm: FC<{ }); }, [source]); + useEffect(() => { + form.setValue("metricsToCreate", metricsToCreate); + }, [form, metricsToCreate]); + const { apiCall } = useAuth(); if (!datasource) { @@ -178,9 +205,11 @@ const NewDataSourceForm: FC<{ if (!dataSourceId) { throw new Error("Could not find existing data source id"); } + const newVal = { ...datasource, settings, + metricsToCreate, }; setDatasource(newVal as Partial); await apiCall<{ status: number; message: string }>( @@ -213,6 +242,11 @@ const NewDataSourceForm: FC<{ }); }; + const metricsToCreateForUser = form.watch("metricsToCreate"); + + console.log("form.watch('metricsToCreate')", form.watch("metricsToCreate")); + console.log("metricsToCreateForUser", metricsToCreateForUser); + const setSchemaSettings = (s: eventSchema) => { setSchema(s.value); form.setValue("settings.schemaFormat", s.value); @@ -500,6 +534,80 @@ const NewDataSourceForm: FC<{ /> ))} + {schemasMap.get(schema)?.label === "Segment" && ( +
+

Metric Options

+ {metricsToCreate.length === 0 ? ( +
+
+ With Segment, we may be able to generate metrics for you + automatically,{" "} + + saving you and your team valuable time. (It's Free) + +
+
+ +
+
+ ) : ( +
+

+ These are the metrics we've found that we can generate + for you automatically. Once created, you can always edit + these if you need to. +

+ {metricsToCreate.map((metric, i) => { + return ( + <> + { + const newMetricsToCreate = cloneDeep( + metricsToCreate + ); + newMetricsToCreate[i].createForUser = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> + + + ); + })} +
+ )} +
+ )} + {autoMetricError && ( +
{autoMetricError}
+ )} ); From 49734e171f21a754ec05bf2b0432ac401c28c839 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Tue, 16 May 2023 09:56:04 -0400 Subject: [PATCH 02/39] Adds some logic to build the metric sql query and only builds those the user has indicated they want us to create for them. --- .../back-end/src/integrations/BigQuery.ts | 12 +++++++ .../src/integrations/SqlIntegration.ts | 21 +++++++++++ .../src/jobs/createAutomaticMetrics.ts | 36 +++++++++++-------- packages/back-end/src/types/Integration.ts | 5 +++ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 3f917b56d17..3d3cdb8d183 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -110,4 +110,16 @@ export default class BigQuery extends SqlIntegration { ); return `\`${this.params.projectId}.${this.params.defaultDataset}.tracks\``; } + + getAutomaticMetricFromClause(event: string): string { + if (!this.params.projectId) + throw new Error( + "No projectId provided. In order to get the information schema, you must provide a projectId." + ); + if (!this.params.defaultDataset) + throw new MissingDatasourceParamsError( + "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." + ); + return `\`${this.params.projectId}.${this.params.defaultDataset}.${event}\``; + } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 42cffd19fe1..c19eb398bbf 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1315,6 +1315,27 @@ export default abstract class SqlIntegration return results; } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getAutomaticMetricFromClause(event: string): string { + return "TODO"; + } + + getAutomaticMetricSqlQuery(metric: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }): string { + return ` + SELECT + ${ + metric.hasUserId ? "user_id, " : "" + //TODO: I need to figure out how to customize the timestamp column based on the type (e.g. Segment vs GA4) + }anonymous_id, loaded_at as timestamp FROM ${this.getAutomaticMetricFromClause( + metric.event + )} + `; + } + private getMetricQueryFormat(metric: MetricInterface) { return metric.queryFormat || (metric.sql ? "sql" : "builder"); } diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 396264b8155..156dcf1b436 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -3,6 +3,7 @@ import uniqid from "uniqid"; import { getDataSourceById } from "../models/DataSourceModel"; import { insertMetric } from "../models/MetricModel"; import { MetricInterface } from "../../types/metric"; +import { getSourceIntegrationObject } from "../services/datasource"; const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; @@ -31,23 +32,30 @@ export default function (ag: Agenda) { if (!datasource) return; + const integration = getSourceIntegrationObject(datasource); + try { // This is where I guess I'll loop through the metricsToCreate & call the createMetric function? - const createdMetrics = metricsToCreate.map(async (metric) => { - const data: Partial = { - id: uniqid("met_"), - organization, - datasource: datasourceId, - name: metric.event, - type: "binomial", - sql: "SELECT * FROM SOMETHING", - dateCreated: new Date(), - dateUpdated: new Date(), - conversionWindowHours: 72, - }; - return await insertMetric(data); + metricsToCreate.map(async (metric) => { + if (metric.createForUser) { + // We need to build the SQL query + if (!integration.getAutomaticMetricSqlQuery) return; //TODO: Throw an error? + const sqlQuery = integration.getAutomaticMetricSqlQuery(metric); + + //TODO: Should I create a new method on the metric controller where we pass it an array of metrics to create rather than doing it one by one? + const data: Partial = { + id: uniqid("met_"), + organization, + datasource: datasourceId, + name: metric.event, + type: "binomial", //TODO: I need to come up with a way to build non-binomial metrics + sql: sqlQuery, + dateCreated: new Date(), + dateUpdated: new Date(), + }; + return await insertMetric(data); + } }); - console.log("createdMetrics", createdMetrics); } catch (e) { // Not sure what to do here yet - catch the errors, but what should I do with them? } diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 99e65080483..cf20081c1bb 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -295,4 +295,9 @@ export interface SourceIntegrationInterface { getTrackedEvents?: () => Promise< { event: string; hasUserId: boolean; createForUser: boolean }[] >; + getAutomaticMetricSqlQuery?(metric: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }): string; } From 81cab6f0b02cfc90461770d699da7fe63b1072b6 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Tue, 16 May 2023 13:21:55 -0400 Subject: [PATCH 03/39] Adds logic for schemaFormat to customize timestamp column, and adds very messy logic to customize metric type. --- .../src/integrations/SqlIntegration.ts | 30 +++- .../src/jobs/createAutomaticMetrics.ts | 158 ++++++++++++++++-- packages/back-end/src/models/MetricModel.ts | 7 + packages/back-end/src/types/Integration.ts | 14 +- 4 files changed, 183 insertions(+), 26 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index c19eb398bbf..940ad5e5113 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -7,6 +7,7 @@ import { DataSourceProperties, ExposureQuery, DataSourceType, + SchemaFormat, } from "../../types/datasource"; import { MetricValueParams, @@ -1320,19 +1321,30 @@ export default abstract class SqlIntegration return "TODO"; } - getAutomaticMetricSqlQuery(metric: { - event: string; - hasUserId: boolean; - createForUser: boolean; - }): string { + getAutomaticMetricTimestampColumn(schemaFormat: SchemaFormat): string { + switch (schemaFormat) { + case "segment": + return "loaded_at"; + default: + return "timestamp"; + } + } + + getAutomaticMetricSqlQuery( + metric: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }, + schemaFormat: SchemaFormat + ): string { return ` SELECT ${ metric.hasUserId ? "user_id, " : "" - //TODO: I need to figure out how to customize the timestamp column based on the type (e.g. Segment vs GA4) - }anonymous_id, loaded_at as timestamp FROM ${this.getAutomaticMetricFromClause( - metric.event - )} + }anonymous_id, ${this.getAutomaticMetricTimestampColumn( + schemaFormat + )} as timestamp FROM ${this.getAutomaticMetricFromClause(metric.event)} `; } diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 156dcf1b436..3fbdb47eb4a 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -1,9 +1,14 @@ import Agenda, { Job } from "agenda"; import uniqid from "uniqid"; import { getDataSourceById } from "../models/DataSourceModel"; -import { insertMetric } from "../models/MetricModel"; -import { MetricInterface } from "../../types/metric"; +import { insertMetrics } from "../models/MetricModel"; +import { MetricInterface, MetricType } from "../../types/metric"; import { getSourceIntegrationObject } from "../services/datasource"; +import { getInformationSchemaById } from "../models/InformationSchemaModel"; +import { getInformationSchemaTableById } from "../models/InformationSchemaTablesModel"; +import { fetchTableData } from "../services/informationSchema"; +import { getPath } from "../util/informationSchemas"; +import { Column } from "../types/Integration"; const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; @@ -35,27 +40,156 @@ export default function (ag: Agenda) { const integration = getSourceIntegrationObject(datasource); try { - // This is where I guess I'll loop through the metricsToCreate & call the createMetric function? - metricsToCreate.map(async (metric) => { + console.log("made it to the try block"); + const metrics: Partial[] = []; + for (const metric of metricsToCreate) { + console.log("metric", metric); + // sampleMetric = {event: "signed_up", hasUserId: true, createForUser: true}; if (metric.createForUser) { - // We need to build the SQL query + // We need to build the SQL query for the metric if (!integration.getAutomaticMetricSqlQuery) return; //TODO: Throw an error? - const sqlQuery = integration.getAutomaticMetricSqlQuery(metric); + // But before we can do that, we need to know if the event (aka, the table_name) has a revenue or count column + // If so, that changes the type, which also affects the query - //TODO: Should I create a new method on the metric controller where we pass it an array of metrics to create rather than doing it one by one? - const data: Partial = { + // To do that, I need to get the `information_schema_table`'s id, and then get all of the columns for that table. + const informationSchemaId = datasource.settings.informationSchemaId; + console.log("informationSchemaId", informationSchemaId); + + if (!informationSchemaId) return; //TODO: Throw an error? + + const informationSchema = await getInformationSchemaById( + organization, + informationSchemaId + ); + + console.log("informationSchema", informationSchema); + + if (!informationSchema) return; //TODO: Throw an error? + + let informationSchemaTableId = ""; + + informationSchema.databases.forEach((database) => { + database.schemas.forEach((schema) => { + schema.tables.forEach((table) => { + if (table.tableName === metric.event) { + informationSchemaTableId = table.id; + } + }); + }); + }); + + console.log("informationSchemaTableId", informationSchemaTableId); + + const informationSchemaTable = await getInformationSchemaTableById( + organization, + informationSchemaTableId + ); + + console.log("informationSchemaTable", informationSchemaTable); + + let metricType: MetricType = "binomial"; + + if (!informationSchemaTable) { + // I need to fetch it and set it within Mongo, + const { + tableData, + databaseName, + tableSchema, + tableName, + } = await fetchTableData( + datasource, + informationSchema, + informationSchemaTableId + ); + + if (!tableData) return; //TODO: Throw an error? + + const columns: Column[] = tableData?.map( + (row: { column_name: string; data_type: string }) => { + return { + columnName: row.column_name, + dataType: row.data_type, + path: getPath(datasource.type, { + tableCatalog: databaseName, + tableSchema: tableSchema, + tableName: tableName, + columnName: row.column_name, + }), + }; + } + ); + + if (columns.length) { + if (columns.some((column) => column.columnName === "revenue")) { + metricType = "revenue"; + } else if ( + columns.some((column) => column.columnName === "count") + ) { + metricType = "count"; + } + } + } + + if (informationSchemaTable?.columns.length) { + if ( + informationSchemaTable.columns.some( + (column) => column.columnName === "revenue" + ) + ) { + metricType = "revenue"; + } else if ( + informationSchemaTable.columns.some( + (column) => column.columnName === "count" + ) + ) { + metricType = "count"; + } + } + // else { + // const { tableData } = await fetchTableData( + // datasource, + // informationSchema, + // informationSchemaTableId + // ); + // if ( + // tableData?.some( + // (column: { data_type: string; column_name: string }) => + // column_name === "revenue" + // ) + // ) { + // metricType = "revenue"; + // } else if (tableData.some((column) => column_name === "count")) { + // metricType = "count"; + // } + // } + + // From there, I can check if there's a revenue or count column + + // if (tableColumnData?.some((column) => columnName === "revenue")) { + // metricType = "revenue"; + // } else if ( + // tableColumnData?.some((column) => columnName === "count") + // ) { + // metricType = "count"; + // } + + const sqlQuery = integration.getAutomaticMetricSqlQuery( + metric, + integration.settings.schemaFormat || "custom" + ); + metrics.push({ id: uniqid("met_"), organization, datasource: datasourceId, name: metric.event, - type: "binomial", //TODO: I need to come up with a way to build non-binomial metrics + type: metricType, //TODO: I need to come up with a way to build non-binomial metrics sql: sqlQuery, dateCreated: new Date(), dateUpdated: new Date(), - }; - return await insertMetric(data); + }); } - }); + } + await insertMetrics(metrics); } catch (e) { // Not sure what to do here yet - catch the errors, but what should I do with them? } diff --git a/packages/back-end/src/models/MetricModel.ts b/packages/back-end/src/models/MetricModel.ts index a6b0bf0df1a..7ee762e8f1c 100644 --- a/packages/back-end/src/models/MetricModel.ts +++ b/packages/back-end/src/models/MetricModel.ts @@ -108,6 +108,13 @@ export async function insertMetric(metric: Partial) { return toInterface(await MetricModel.create(metric)); } +export async function insertMetrics(metrics: Partial[]) { + if (usingFileConfig()) { + throw new Error("Cannot add. Metrics managed by config.yml"); + } + return (await MetricModel.insertMany(metrics)).map(toInterface); +} + export async function deleteMetricById(id: string, organization: string) { if (usingFileConfig()) { throw new Error("Cannot delete. Metrics managed by config.yml"); diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index cf20081c1bb..6f25a6c518e 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -1,6 +1,7 @@ import { DataSourceProperties, DataSourceSettings, + SchemaFormat, } from "../../types/datasource"; import { DimensionInterface } from "../../types/dimension"; import { ExperimentInterface, ExperimentPhase } from "../../types/experiment"; @@ -295,9 +296,12 @@ export interface SourceIntegrationInterface { getTrackedEvents?: () => Promise< { event: string; hasUserId: boolean; createForUser: boolean }[] >; - getAutomaticMetricSqlQuery?(metric: { - event: string; - hasUserId: boolean; - createForUser: boolean; - }): string; + getAutomaticMetricSqlQuery?( + metric: { + event: string; + hasUserId: boolean; + createForUser: boolean; + }, + schemaFormat: SchemaFormat + ): string; } From 75383614e5095c32d92ebc8bd7a40849a57d50c6 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Tue, 16 May 2023 14:58:51 -0400 Subject: [PATCH 04/39] Adds basic support for count and revenue metric types. Still not sure how to go about duration types, atleast with Segment. --- .../src/integrations/SqlIntegration.ts | 25 ++++++++++++--- .../src/jobs/createAutomaticMetrics.ts | 32 ++----------------- packages/back-end/src/types/Integration.ts | 3 +- .../components/Settings/NewDataSourceForm.tsx | 4 +-- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 940ad5e5113..a6a43b54b27 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1,7 +1,7 @@ import cloneDeep from "lodash/cloneDeep"; import { DEFAULT_REGRESSION_ADJUSTMENT_DAYS } from "shared/constants"; import { getValidDate } from "shared/dates"; -import { MetricInterface } from "../../types/metric"; +import { MetricInterface, MetricType } from "../../types/metric"; import { DataSourceSettings, DataSourceProperties, @@ -1330,22 +1330,39 @@ export default abstract class SqlIntegration } } + getAutomaticMetricAggregatorColumn(metricType: MetricType): string { + switch (metricType) { + case "count": + return ", count as value"; + case "revenue": + return ", revenue as value"; + default: + return ""; + } + } + getAutomaticMetricSqlQuery( metric: { event: string; hasUserId: boolean; createForUser: boolean; }, - schemaFormat: SchemaFormat + schemaFormat: SchemaFormat, + metricType: MetricType ): string { - return ` + const sqlQuery = ` SELECT ${ metric.hasUserId ? "user_id, " : "" }anonymous_id, ${this.getAutomaticMetricTimestampColumn( schemaFormat - )} as timestamp FROM ${this.getAutomaticMetricFromClause(metric.event)} + )} as timestamp + ${this.getAutomaticMetricAggregatorColumn( + metricType + )} FROM ${this.getAutomaticMetricFromClause(metric.event)} `; + + return format(sqlQuery, this.getFormatDialect()); } private getMetricQueryFormat(metric: MetricInterface) { diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 3fbdb47eb4a..acfb0edb569 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -145,44 +145,18 @@ export default function (ag: Agenda) { metricType = "count"; } } - // else { - // const { tableData } = await fetchTableData( - // datasource, - // informationSchema, - // informationSchemaTableId - // ); - // if ( - // tableData?.some( - // (column: { data_type: string; column_name: string }) => - // column_name === "revenue" - // ) - // ) { - // metricType = "revenue"; - // } else if (tableData.some((column) => column_name === "count")) { - // metricType = "count"; - // } - // } - - // From there, I can check if there's a revenue or count column - - // if (tableColumnData?.some((column) => columnName === "revenue")) { - // metricType = "revenue"; - // } else if ( - // tableColumnData?.some((column) => columnName === "count") - // ) { - // metricType = "count"; - // } const sqlQuery = integration.getAutomaticMetricSqlQuery( metric, - integration.settings.schemaFormat || "custom" + integration.settings.schemaFormat || "custom", + metricType ); metrics.push({ id: uniqid("met_"), organization, datasource: datasourceId, name: metric.event, - type: metricType, //TODO: I need to come up with a way to build non-binomial metrics + type: metricType, sql: sqlQuery, dateCreated: new Date(), dateUpdated: new Date(), diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 6f25a6c518e..5a3e9216154 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -302,6 +302,7 @@ export interface SourceIntegrationInterface { hasUserId: boolean; createForUser: boolean; }, - schemaFormat: SchemaFormat + schemaFormat: SchemaFormat, + metricType: MetricType ): string; } diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index bb5ba54bbf8..774fdb53b93 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -585,7 +585,7 @@ const NewDataSourceForm: FC<{

{metricsToCreate.map((metric, i) => { return ( - <> +
- +
); })} From 6d4be65205d11aae01cb0bf0d6fdbf0fe7739d9d Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Wed, 17 May 2023 14:55:54 -0400 Subject: [PATCH 05/39] Adds some flexibility to the SQl queries to handle different table names for the tracked events, along with different event column names based on the schemaFormat. --- packages/back-end/src/controllers/metrics.ts | 10 ++++- .../back-end/src/integrations/BigQuery.ts | 4 +- .../src/integrations/SqlIntegration.ts | 44 ++++++++++++------- packages/back-end/src/types/Integration.ts | 6 +-- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/packages/back-end/src/controllers/metrics.ts b/packages/back-end/src/controllers/metrics.ts index d2340709b86..bfdc5a4a769 100644 --- a/packages/back-end/src/controllers/metrics.ts +++ b/packages/back-end/src/controllers/metrics.ts @@ -483,7 +483,11 @@ export async function getAutoMetrics( const integration = getSourceIntegrationObject(dataSourceObj); - if (!integration.getTrackedEvents) { + if ( + !integration.getTrackedEvents || + !integration.settings.schemaFormat || + integration.settings.schemaFormat === "custom" //MKTODO: Is this logic correct? + ) { //TODO: Is this the correct error code? res.status(403).json({ status: 403, @@ -493,7 +497,9 @@ export async function getAutoMetrics( } try { - const results = await integration.getTrackedEvents(); + const results = await integration.getTrackedEvents( + integration.settings.schemaFormat + ); if (results.length) { return res.status(200).json({ diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 3d3cdb8d183..496a0f4dae7 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -99,7 +99,7 @@ export default class BigQuery extends SqlIntegration { ): string { return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``; } - getTrackedEventsFromClause(): string { + getTrackedEventsFromClause(trackedEventTableName: string): string { if (!this.params.projectId) throw new Error( "No projectId provided. In order to get the information schema, you must provide a projectId." @@ -108,7 +108,7 @@ export default class BigQuery extends SqlIntegration { throw new MissingDatasourceParamsError( "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." ); - return `\`${this.params.projectId}.${this.params.defaultDataset}.tracks\``; + return `\`${this.params.projectId}.${this.params.defaultDataset}.${trackedEventTableName}\``; } getAutomaticMetricFromClause(event: string): string { diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index a6a43b54b27..c8ec9d37785 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1290,25 +1290,34 @@ export default abstract class SqlIntegration return { tableData }; } - getTrackedEventsFromClause(): string { - return "TODO"; + getTrackedEventsFromClause(trackedEventTableName: string): string { + return trackedEventTableName; } - async getTrackedEvents(): Promise< - { event: string; hasUserId: boolean; createForUser: boolean }[] - > { + async getTrackedEvents( + schemaFormat: SchemaFormat + ): Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]> { + let eventColumn; + let trackedEventTableName; + + switch (schemaFormat) { + //TODO: Add cases for schemaFormats where the tracked events aren't stored in a column called "events" + default: + eventColumn = "event"; + trackedEventTableName = "tracks"; + } + const sql = ` SELECT - event, - (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId + ${eventColumn} as event, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + (CASE WHEN (1 > 0) THEN 1 ELSE 0 END) as createForUser FROM - ${this.getTrackedEventsFromClause()} + ${this.getTrackedEventsFromClause(trackedEventTableName)} GROUP BY event`; const results = await this.runQuery(format(sql, this.getFormatDialect())); - results.forEach((result) => (result.createForUser = true)); - if (!results) { throw new Error(`No events found.`); } @@ -1316,15 +1325,16 @@ export default abstract class SqlIntegration return results; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars getAutomaticMetricFromClause(event: string): string { - return "TODO"; + return event; } getAutomaticMetricTimestampColumn(schemaFormat: SchemaFormat): string { switch (schemaFormat) { - case "segment": - return "loaded_at"; + case "segment" || "rudderstack": + return "received_at"; + case "amplitude": + return "server_upload_time"; default: return "timestamp"; } @@ -1352,9 +1362,9 @@ export default abstract class SqlIntegration ): string { const sqlQuery = ` SELECT - ${ - metric.hasUserId ? "user_id, " : "" - }anonymous_id, ${this.getAutomaticMetricTimestampColumn( + ${metric.hasUserId ? "user_id, " : ""}${ + schemaFormat === "amplitude" ? "amplitude_id" : "anonymous_id" + } as anonymous_id, ${this.getAutomaticMetricTimestampColumn( schemaFormat )} as timestamp ${this.getAutomaticMetricAggregatorColumn( diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 5a3e9216154..18c5fd828c0 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -293,9 +293,9 @@ export interface SourceIntegrationInterface { query: string ): Promise; runPastExperimentQuery(query: string): Promise; - getTrackedEvents?: () => Promise< - { event: string; hasUserId: boolean; createForUser: boolean }[] - >; + getTrackedEvents?: ( + schemaFormat: SchemaFormat + ) => Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]>; getAutomaticMetricSqlQuery?( metric: { event: string; From 75aba06b4e25ca857d7d9ad79c38c3140d90e8ce Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Wed, 17 May 2023 16:34:37 -0400 Subject: [PATCH 06/39] Overall cleanup and renaming of methods. --- packages/back-end/src/app.ts | 5 +- .../back-end/src/controllers/datasources.ts | 68 +++++++++- packages/back-end/src/controllers/metrics.ts | 63 --------- .../back-end/src/integrations/BigQuery.ts | 4 +- .../src/integrations/SqlIntegration.ts | 35 +++-- .../src/jobs/createAutomaticMetrics.ts | 128 +++++++----------- packages/back-end/src/types/Integration.ts | 4 +- packages/back-end/types/datasource.d.ts | 1 + .../components/Settings/NewDataSourceForm.tsx | 2 +- 9 files changed, 148 insertions(+), 162 deletions(-) diff --git a/packages/back-end/src/app.ts b/packages/back-end/src/app.ts index 41227826841..962efbf103c 100644 --- a/packages/back-end/src/app.ts +++ b/packages/back-end/src/app.ts @@ -346,7 +346,6 @@ app.get( metricsController.getMetricAnalysisStatus ); app.post("/metric/:id/analysis/cancel", metricsController.cancelMetricAnalysis); -app.get("/metrics/generate/:dataSourceId", metricsController.getAutoMetrics); // Experiments app.get("/experiments", experimentsController.getExperiments); @@ -483,6 +482,10 @@ app.get("/datasource/:id", datasourcesController.getDataSource); app.post("/datasources", datasourcesController.postDataSources); app.put("/datasource/:id", datasourcesController.putDataSource); app.delete("/datasource/:id", datasourcesController.deleteDataSource); +app.get( + "/datasource/:id/auto-metrics", + datasourcesController.getAutoGeneratedMetricsToCreate +); // Information Schemas app.get( diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 3c45d9862de..f2338a8acdd 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -442,9 +442,6 @@ export async function putDataSource( metricsToCreate, } = req.body; - console.log("made it to the putDataSource method"); - console.log("metricsToCreate", metricsToCreate); - const datasource = await getDataSourceById(id, org.id); if (!datasource) { res.status(404).json({ @@ -622,3 +619,68 @@ export async function testLimitedQuery( error, }); } + +export async function getAutoGeneratedMetricsToCreate( + req: AuthRequest, + res: Response +) { + const { org } = getOrgFromReq(req); + const { id } = req.params; + + const dataSourceObj = await getDataSourceById(id, org.id); + if (!dataSourceObj) { + res.status(403).json({ + status: 403, + message: "Invalid data source: " + id, + }); + return; + } + + req.checkPermissions( + "createMetrics", + dataSourceObj.projects?.length ? dataSourceObj.projects : "" + ); + + const integration = getSourceIntegrationObject(dataSourceObj); + + if ( + !integration.getEventsTrackedByDatasource || + !integration.settings.schemaFormat || + !integration.getSourceProperties().supportsAutoGeneratedMetrics || + integration.settings.schemaFormat === "custom" //MKTODO: Is this logic correct? + ) { + //TODO: Is this the correct error code? + res.status(403).json({ + status: 403, + message: "This datasource does not support automatic metric generation.", + }); + return; + } + + try { + // Get the list of events tracked by this datasource - we can create metrics from these events. + const results = await integration.getEventsTrackedByDatasource( + integration.settings.schemaFormat + ); + + if (results.length) { + return res.status(200).json({ + status: 200, + results, + }); + } else { + return res.status(200).json({ + status: 200, + results, + message: "No events found.", + }); + } + } catch (e) { + res.status(400).json({ + status: 400, + results: [], + message: e.message, + }); + return; + } +} diff --git a/packages/back-end/src/controllers/metrics.ts b/packages/back-end/src/controllers/metrics.ts index bfdc5a4a769..7d5aa2aa9ce 100644 --- a/packages/back-end/src/controllers/metrics.ts +++ b/packages/back-end/src/controllers/metrics.ts @@ -459,66 +459,3 @@ export async function putMetric( }), }); } - -export async function getAutoMetrics( - req: AuthRequest, - res: Response -) { - const { org } = getOrgFromReq(req); - const { dataSourceId } = req.params; - - const dataSourceObj = await getDataSourceById(dataSourceId, org.id); - if (!dataSourceObj) { - res.status(403).json({ - status: 403, - message: "Invalid data source: " + dataSourceId, - }); - return; - } - - req.checkPermissions( - "createMetrics", - dataSourceObj.projects?.length ? dataSourceObj.projects : "" - ); - - const integration = getSourceIntegrationObject(dataSourceObj); - - if ( - !integration.getTrackedEvents || - !integration.settings.schemaFormat || - integration.settings.schemaFormat === "custom" //MKTODO: Is this logic correct? - ) { - //TODO: Is this the correct error code? - res.status(403).json({ - status: 403, - message: "This datasource does not support automatic metric generation.", - }); - return; - } - - try { - const results = await integration.getTrackedEvents( - integration.settings.schemaFormat - ); - - if (results.length) { - return res.status(200).json({ - status: 200, - results, - }); - } else { - return res.status(200).json({ - status: 200, - results, - message: "No events found.", - }); - } - } catch (e) { - res.status(400).json({ - status: 400, - results: [], - message: e.message, - }); - return; - } -} diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 496a0f4dae7..4d5046106e1 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -99,7 +99,9 @@ export default class BigQuery extends SqlIntegration { ): string { return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``; } - getTrackedEventsFromClause(trackedEventTableName: string): string { + getEventsTrackedByDatasourceFromClause( + trackedEventTableName: string + ): string { if (!this.params.projectId) throw new Error( "No projectId provided. In order to get the information schema, you must provide a projectId." diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index c8ec9d37785..5972bddef76 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -80,6 +80,7 @@ export default abstract class SqlIntegration activationDimension: true, pastExperiments: true, supportsInformationSchema: true, + supportsAutoGeneratedMetrics: this.isAutoGeneratedMetricSupported(), }; } @@ -88,6 +89,18 @@ export default abstract class SqlIntegration return true; } + isAutoGeneratedMetricSupported(): boolean { + const supportedEventTrackers = ["segment"]; + + if ( + this.settings.schemaFormat && + supportedEventTrackers.includes(this.settings.schemaFormat) + ) { + return true; + } + return false; + } + getSchema(): string { return ""; } @@ -1290,11 +1303,13 @@ export default abstract class SqlIntegration return { tableData }; } - getTrackedEventsFromClause(trackedEventTableName: string): string { + getEventsTrackedByDatasourceFromClause( + trackedEventTableName: string + ): string { return trackedEventTableName; } - async getTrackedEvents( + async getEventsTrackedByDatasource( schemaFormat: SchemaFormat ): Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]> { let eventColumn; @@ -1313,7 +1328,7 @@ export default abstract class SqlIntegration (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, (CASE WHEN (1 > 0) THEN 1 ELSE 0 END) as createForUser FROM - ${this.getTrackedEventsFromClause(trackedEventTableName)} + ${this.getEventsTrackedByDatasourceFromClause(trackedEventTableName)} GROUP BY event`; const results = await this.runQuery(format(sql, this.getFormatDialect())); @@ -1325,11 +1340,11 @@ export default abstract class SqlIntegration return results; } - getAutomaticMetricFromClause(event: string): string { + getAutoGeneratedMetricFromClause(event: string): string { return event; } - getAutomaticMetricTimestampColumn(schemaFormat: SchemaFormat): string { + getAutoGeneratedMetricTimestampColumn(schemaFormat: SchemaFormat): string { switch (schemaFormat) { case "segment" || "rudderstack": return "received_at"; @@ -1340,7 +1355,7 @@ export default abstract class SqlIntegration } } - getAutomaticMetricAggregatorColumn(metricType: MetricType): string { + getAutoGeneratedMetricValueColumn(metricType: MetricType): string { switch (metricType) { case "count": return ", count as value"; @@ -1351,7 +1366,7 @@ export default abstract class SqlIntegration } } - getAutomaticMetricSqlQuery( + getAutoGeneratedMetricSqlQuery( metric: { event: string; hasUserId: boolean; @@ -1364,12 +1379,12 @@ export default abstract class SqlIntegration SELECT ${metric.hasUserId ? "user_id, " : ""}${ schemaFormat === "amplitude" ? "amplitude_id" : "anonymous_id" - } as anonymous_id, ${this.getAutomaticMetricTimestampColumn( + } as anonymous_id, ${this.getAutoGeneratedMetricTimestampColumn( schemaFormat )} as timestamp - ${this.getAutomaticMetricAggregatorColumn( + ${this.getAutoGeneratedMetricValueColumn( metricType - )} FROM ${this.getAutomaticMetricFromClause(metric.event)} + )} FROM ${this.getAutoGeneratedMetricFromClause(metric.event)} `; return format(sqlQuery, this.getFormatDialect()); diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index acfb0edb569..1c1a3752d2c 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import Agenda, { Job } from "agenda"; import uniqid from "uniqid"; import { getDataSourceById } from "../models/DataSourceModel"; @@ -5,7 +6,6 @@ import { insertMetrics } from "../models/MetricModel"; import { MetricInterface, MetricType } from "../../types/metric"; import { getSourceIntegrationObject } from "../services/datasource"; import { getInformationSchemaById } from "../models/InformationSchemaModel"; -import { getInformationSchemaTableById } from "../models/InformationSchemaTablesModel"; import { fetchTableData } from "../services/informationSchema"; import { getPath } from "../util/informationSchemas"; import { Column } from "../types/Integration"; @@ -29,6 +29,7 @@ export default function (ag: Agenda) { agenda.define( CREATE_AUTOMATIC_METRICS_JOB_NAME, async (job: CreateAutomaticMetricsJob) => { + console.log("made it to the job"); const { datasourceId, organization, metricsToCreate } = job.attrs.data; if (!datasourceId || !organization || !metricsToCreate) return; @@ -39,35 +40,29 @@ export default function (ag: Agenda) { const integration = getSourceIntegrationObject(datasource); - try { - console.log("made it to the try block"); - const metrics: Partial[] = []; - for (const metric of metricsToCreate) { - console.log("metric", metric); - // sampleMetric = {event: "signed_up", hasUserId: true, createForUser: true}; - if (metric.createForUser) { - // We need to build the SQL query for the metric - if (!integration.getAutomaticMetricSqlQuery) return; //TODO: Throw an error? - // But before we can do that, we need to know if the event (aka, the table_name) has a revenue or count column - // If so, that changes the type, which also affects the query + if ( + !integration.getAutoGeneratedMetricSqlQuery || + !integration.getSourceProperties().supportsAutoGeneratedMetrics + ) + return; - // To do that, I need to get the `information_schema_table`'s id, and then get all of the columns for that table. - const informationSchemaId = datasource.settings.informationSchemaId; - console.log("informationSchemaId", informationSchemaId); + const informationSchemaId = datasource.settings.informationSchemaId; - if (!informationSchemaId) return; //TODO: Throw an error? + if (!informationSchemaId) return; //TODO: Throw an error? - const informationSchema = await getInformationSchemaById( - organization, - informationSchemaId - ); + const informationSchema = await getInformationSchemaById( + organization, + informationSchemaId + ); - console.log("informationSchema", informationSchema); + if (!informationSchema) return; //TODO: Throw an error? - if (!informationSchema) return; //TODO: Throw an error? - - let informationSchemaTableId = ""; + let informationSchemaTableId = ""; + try { + const metrics: Partial[] = []; + for (const metric of metricsToCreate) { + if (metric.createForUser) { informationSchema.databases.forEach((database) => { database.schemas.forEach((schema) => { schema.tables.forEach((table) => { @@ -78,75 +73,46 @@ export default function (ag: Agenda) { }); }); - console.log("informationSchemaTableId", informationSchemaTableId); - - const informationSchemaTable = await getInformationSchemaTableById( - organization, + const { + tableData, + databaseName, + tableSchema, + tableName, + } = await fetchTableData( + datasource, + informationSchema, informationSchemaTableId ); - console.log("informationSchemaTable", informationSchemaTable); - - let metricType: MetricType = "binomial"; - - if (!informationSchemaTable) { - // I need to fetch it and set it within Mongo, - const { - tableData, - databaseName, - tableSchema, - tableName, - } = await fetchTableData( - datasource, - informationSchema, - informationSchemaTableId - ); - - if (!tableData) return; //TODO: Throw an error? - - const columns: Column[] = tableData?.map( - (row: { column_name: string; data_type: string }) => { - return { + if (!tableData) return; //TODO: Throw an error? + + const columns: Column[] = tableData?.map( + (row: { column_name: string; data_type: string }) => { + return { + columnName: row.column_name, + dataType: row.data_type, + path: getPath(datasource.type, { + tableCatalog: databaseName, + tableSchema: tableSchema, + tableName: tableName, columnName: row.column_name, - dataType: row.data_type, - path: getPath(datasource.type, { - tableCatalog: databaseName, - tableSchema: tableSchema, - tableName: tableName, - columnName: row.column_name, - }), - }; - } - ); - - if (columns.length) { - if (columns.some((column) => column.columnName === "revenue")) { - metricType = "revenue"; - } else if ( - columns.some((column) => column.columnName === "count") - ) { - metricType = "count"; - } + }), + }; } - } + ); - if (informationSchemaTable?.columns.length) { - if ( - informationSchemaTable.columns.some( - (column) => column.columnName === "revenue" - ) - ) { + let metricType: MetricType = "binomial"; + + if (columns.length) { + if (columns.some((column) => column.columnName === "revenue")) { metricType = "revenue"; } else if ( - informationSchemaTable.columns.some( - (column) => column.columnName === "count" - ) + columns.some((column) => column.columnName === "count") ) { metricType = "count"; } } - - const sqlQuery = integration.getAutomaticMetricSqlQuery( + const sqlQuery = integration.getAutoGeneratedMetricSqlQuery( metric, integration.settings.schemaFormat || "custom", metricType diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 18c5fd828c0..48e8ead6bc1 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -293,10 +293,10 @@ export interface SourceIntegrationInterface { query: string ): Promise; runPastExperimentQuery(query: string): Promise; - getTrackedEvents?: ( + getEventsTrackedByDatasource?: ( schemaFormat: SchemaFormat ) => Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]>; - getAutomaticMetricSqlQuery?( + getAutoGeneratedMetricSqlQuery?( metric: { event: string; hasUserId: boolean; diff --git a/packages/back-end/types/datasource.d.ts b/packages/back-end/types/datasource.d.ts index ae8d436d505..64fffc0ebef 100644 --- a/packages/back-end/types/datasource.d.ts +++ b/packages/back-end/types/datasource.d.ts @@ -95,6 +95,7 @@ export interface DataSourceProperties { pastExperiments?: boolean; separateExperimentResultQueries?: boolean; supportsInformationSchema?: boolean; + supportsAutoGeneratedMetrics?: boolean; } type WithParams = Omit & { diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 774fdb53b93..6e2dfe263a3 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -558,7 +558,7 @@ const NewDataSourceForm: FC<{ createForUser: boolean; }[]; message?: string; - }>(`/metrics/generate/${dataSourceId}`, { + }>(`/datasource/${dataSourceId}/auto-metrics`, { method: "GET", }); if (res.message) { From 3503ef47e9a947b07ae3c5cb0314397aeef79a28 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 18 May 2023 06:50:02 -0400 Subject: [PATCH 07/39] Removing unused import. --- packages/back-end/src/controllers/metrics.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/back-end/src/controllers/metrics.ts b/packages/back-end/src/controllers/metrics.ts index 7d5aa2aa9ce..272126412a9 100644 --- a/packages/back-end/src/controllers/metrics.ts +++ b/packages/back-end/src/controllers/metrics.ts @@ -31,7 +31,6 @@ import { auditDetailsDelete, } from "../services/audit"; import { EventAuditUserForResponseLocals } from "../events/event-types"; -import { getSourceIntegrationObject } from "../services/datasource"; export async function deleteMetric( req: AuthRequest, From 4935fa38127ebb71b1bff00f5cb49afc2756e9a9 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 18 May 2023 09:52:07 -0400 Subject: [PATCH 08/39] Updates the front end to consume the new supportsAutoGeneratedMetrics integration property. --- .../back-end/src/controllers/datasources.ts | 15 ++++--- .../back-end/src/integrations/BigQuery.ts | 2 +- .../components/Settings/NewDataSourceForm.tsx | 41 +++++++++++++++---- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index f2338a8acdd..4ccc7dfb826 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -646,8 +646,7 @@ export async function getAutoGeneratedMetricsToCreate( if ( !integration.getEventsTrackedByDatasource || !integration.settings.schemaFormat || - !integration.getSourceProperties().supportsAutoGeneratedMetrics || - integration.settings.schemaFormat === "custom" //MKTODO: Is this logic correct? + !integration.getSourceProperties().supportsAutoGeneratedMetrics ) { //TODO: Is this the correct error code? res.status(403).json({ @@ -663,18 +662,18 @@ export async function getAutoGeneratedMetricsToCreate( integration.settings.schemaFormat ); - if (results.length) { - return res.status(200).json({ - status: 200, - results, - }); - } else { + if (!results.length) { return res.status(200).json({ status: 200, results, message: "No events found.", }); } + + return res.status(200).json({ + status: 200, + results, + }); } catch (e) { res.status(400).json({ status: 400, diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 4d5046106e1..2a356443817 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -113,7 +113,7 @@ export default class BigQuery extends SqlIntegration { return `\`${this.params.projectId}.${this.params.defaultDataset}.${trackedEventTableName}\``; } - getAutomaticMetricFromClause(event: string): string { + getAutoGeneratedMetricFromClause(event: string): string { if (!this.params.projectId) throw new Error( "No projectId provided. In order to get the information schema, you must provide a projectId." diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 6e2dfe263a3..c34f4316f98 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -50,7 +50,12 @@ const NewDataSourceForm: FC<{ inline, secondaryCTA, }) => { - const { projects, project } = useDefinitions(); + const { + projects, + project, + getDatasourceById, + mutateDefinitions, + } = useDefinitions(); const [step, setStep] = useState(0); const [schema, setSchema] = useState(""); const [dataSourceId, setDataSourceId] = useState( @@ -79,6 +84,27 @@ const NewDataSourceForm: FC<{ settings: {}, }; + const [ + datasourceSupportsAutoGeneratedMetrics, + setDatasourceSupportsAutoGeneratedMetrics, + ] = useState(false); + + useEffect(() => { + if (dataSourceId) { + mutateDefinitions(); + const datasourceObj = getDatasourceById(dataSourceId); + const supportsAutoGeneratedMetrics = + datasourceObj?.properties?.supportsAutoGeneratedMetrics || false; + setDatasourceSupportsAutoGeneratedMetrics(supportsAutoGeneratedMetrics); + } + }, [ + dataSourceId, + datasource.properties?.supportsAutoGeneratedMetrics, + datasourceSupportsAutoGeneratedMetrics, + getDatasourceById, + mutateDefinitions, + ]); + const form = useForm<{ settings: DataSourceSettings | undefined; metricsToCreate: { @@ -242,11 +268,6 @@ const NewDataSourceForm: FC<{ }); }; - const metricsToCreateForUser = form.watch("metricsToCreate"); - - console.log("form.watch('metricsToCreate')", form.watch("metricsToCreate")); - console.log("metricsToCreateForUser", metricsToCreateForUser); - const setSchemaSettings = (s: eventSchema) => { setSchema(s.value); form.setValue("settings.schemaFormat", s.value); @@ -534,14 +555,16 @@ const NewDataSourceForm: FC<{ /> ))} - {schemasMap.get(schema)?.label === "Segment" && ( + {datasourceSupportsAutoGeneratedMetrics && (

Metric Options

{metricsToCreate.length === 0 ? (
- With Segment, we may be able to generate metrics for you - automatically,{" "} + {`With ${ + schemasMap.get(schema).label + }, we may be able to generate metrics for you + automatically,`}{" "} saving you and your team valuable time. (It's Free) From eff2b945a717f9bde2cc5dbb9e559629eb0f5d86 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 18 May 2023 11:01:41 -0400 Subject: [PATCH 09/39] Expanding support for all data warehouses for Segment & Rudderstack - not fully tested yet. --- packages/back-end/src/integrations/Athena.ts | 9 +++++++++ packages/back-end/src/integrations/BigQuery.ts | 8 ++++---- packages/back-end/src/integrations/Mssql.ts | 9 +++++++++ packages/back-end/src/integrations/Snowflake.ts | 9 +++++++++ .../back-end/src/integrations/SqlIntegration.ts | 14 ++++++++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/packages/back-end/src/integrations/Athena.ts b/packages/back-end/src/integrations/Athena.ts index c84f583e9fb..fb491278c2a 100644 --- a/packages/back-end/src/integrations/Athena.ts +++ b/packages/back-end/src/integrations/Athena.ts @@ -59,4 +59,13 @@ export default class Athena extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } + getEventsTrackedByDatasourceFromClause( + trackedEventTableName: string + ): string { + if (!this.params.catalog) + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default catalog." + ); + return `${this.params.catalog}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 2a356443817..ebb6881ce58 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -104,11 +104,11 @@ export default class BigQuery extends SqlIntegration { ): string { if (!this.params.projectId) throw new Error( - "No projectId provided. In order to get the information schema, you must provide a projectId." + "No projectId provided. To automatically generate metrics you must provide a projectId." ); if (!this.params.defaultDataset) throw new MissingDatasourceParamsError( - "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." + "To automatically generate metrics for a BigQuery dataset, you must define a default dataset." ); return `\`${this.params.projectId}.${this.params.defaultDataset}.${trackedEventTableName}\``; } @@ -116,11 +116,11 @@ export default class BigQuery extends SqlIntegration { getAutoGeneratedMetricFromClause(event: string): string { if (!this.params.projectId) throw new Error( - "No projectId provided. In order to get the information schema, you must provide a projectId." + "No projectId provided. To automatically generate metrics you must provide a projectId." ); if (!this.params.defaultDataset) throw new MissingDatasourceParamsError( - "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." + "To automatically generate metrics for a BigQuery dataset, you must define a default dataset." ); return `\`${this.params.projectId}.${this.params.defaultDataset}.${event}\``; } diff --git a/packages/back-end/src/integrations/Mssql.ts b/packages/back-end/src/integrations/Mssql.ts index 69200f8d907..e154e621e41 100644 --- a/packages/back-end/src/integrations/Mssql.ts +++ b/packages/back-end/src/integrations/Mssql.ts @@ -77,4 +77,13 @@ export default class Mssql extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } + getEventsTrackedByDatasourceFromClause( + trackedEventTableName: string + ): string { + if (!this.params.database) + throw new Error( + "No database provided. To automatically generate metrics, you must provide a database." + ); + return `${this.params.database}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/Snowflake.ts b/packages/back-end/src/integrations/Snowflake.ts index 31d406016e2..b9514fbf44e 100644 --- a/packages/back-end/src/integrations/Snowflake.ts +++ b/packages/back-end/src/integrations/Snowflake.ts @@ -47,4 +47,13 @@ export default class Snowflake extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } + getEventsTrackedByDatasourceFromClause( + trackedEventTableName: string + ): string { + if (!this.params.database) + throw new Error( + "No database provided. To automatically generate metrics, you must provide a database." + ); + return `${this.params.database}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 5972bddef76..478e8612961 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -90,7 +90,7 @@ export default abstract class SqlIntegration } isAutoGeneratedMetricSupported(): boolean { - const supportedEventTrackers = ["segment"]; + const supportedEventTrackers: SchemaFormat[] = ["segment", "rudderstack"]; if ( this.settings.schemaFormat && @@ -1355,10 +1355,15 @@ export default abstract class SqlIntegration } } - getAutoGeneratedMetricValueColumn(metricType: MetricType): string { + getAutoGeneratedMetricValueColumn( + metricType: MetricType, + schemaFormat: SchemaFormat + ): string { + const countColumn = schemaFormat === "rudderstack" ? "value" : "count"; + switch (metricType) { case "count": - return ", count as value"; + return `, ${countColumn} as value`; case "revenue": return ", revenue as value"; default: @@ -1383,7 +1388,8 @@ export default abstract class SqlIntegration schemaFormat )} as timestamp ${this.getAutoGeneratedMetricValueColumn( - metricType + metricType, + schemaFormat )} FROM ${this.getAutoGeneratedMetricFromClause(metric.event)} `; From 775335866ba8cda5b527dca35cb6f7b22ae7a456 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 18 May 2023 16:20:20 -0400 Subject: [PATCH 10/39] Updates the mssql integration to include an optional default schema input as that is required for us to know which schema to query when looking for the tracked events coming from Rudderstack. Also updating the Snowflake from clause for the query to use the schema. --- packages/back-end/src/integrations/Mssql.ts | 6 +++--- packages/back-end/src/integrations/Snowflake.ts | 6 +++--- packages/back-end/types/integrations/mssql.d.ts | 1 + packages/front-end/components/Settings/MssqlForm.tsx | 11 +++++++++++ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/back-end/src/integrations/Mssql.ts b/packages/back-end/src/integrations/Mssql.ts index e154e621e41..82f1f1c4a05 100644 --- a/packages/back-end/src/integrations/Mssql.ts +++ b/packages/back-end/src/integrations/Mssql.ts @@ -80,10 +80,10 @@ export default class Mssql extends SqlIntegration { getEventsTrackedByDatasourceFromClause( trackedEventTableName: string ): string { - if (!this.params.database) + if (!this.params.defaultSchema) throw new Error( - "No database provided. To automatically generate metrics, you must provide a database." + "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where the Rudderstack tracked events are stored." ); - return `${this.params.database}.${trackedEventTableName}`; + return `${this.params.defaultSchema}.${trackedEventTableName}`; } } diff --git a/packages/back-end/src/integrations/Snowflake.ts b/packages/back-end/src/integrations/Snowflake.ts index b9514fbf44e..a8c6121dcbd 100644 --- a/packages/back-end/src/integrations/Snowflake.ts +++ b/packages/back-end/src/integrations/Snowflake.ts @@ -50,10 +50,10 @@ export default class Snowflake extends SqlIntegration { getEventsTrackedByDatasourceFromClause( trackedEventTableName: string ): string { - if (!this.params.database) + if (!this.params.schema) throw new Error( - "No database provided. To automatically generate metrics, you must provide a database." + "No schema provided. To automatically generate metrics, you must provide a schema." ); - return `${this.params.database}.${trackedEventTableName}`; + return `${this.params.schema}.${trackedEventTableName}`; } } diff --git a/packages/back-end/types/integrations/mssql.d.ts b/packages/back-end/types/integrations/mssql.d.ts index 407c0c66a29..42283405706 100644 --- a/packages/back-end/types/integrations/mssql.d.ts +++ b/packages/back-end/types/integrations/mssql.d.ts @@ -4,6 +4,7 @@ export interface MssqlConnectionParams { database: string; password: string; port: number; + defaultSchema?: string; options?: { encrypt?: boolean; // for azure trustServerCertificate?: boolean; // change to true for local dev / self-signed certs diff --git a/packages/front-end/components/Settings/MssqlForm.tsx b/packages/front-end/components/Settings/MssqlForm.tsx index 8062a0e8371..729d9ae897f 100644 --- a/packages/front-end/components/Settings/MssqlForm.tsx +++ b/packages/front-end/components/Settings/MssqlForm.tsx @@ -80,6 +80,17 @@ const MssqlForm: FC<{ placeholder={existing ? "(Keep existing)" : ""} />
+
+ + +
From 5eeed8fae1b26e118d27476d7f89d9ee17fa7dfb Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 19 May 2023 08:59:50 -0400 Subject: [PATCH 11/39] Refactors the code a bit to use the same method to get the from clause for both sql queries relating to the auto metrics - the query to get a list of tracked events and the query for the actual metric. --- packages/back-end/src/integrations/Athena.ts | 12 ++++++++---- packages/back-end/src/integrations/BigQuery.ts | 14 -------------- packages/back-end/src/integrations/ClickHouse.ts | 9 +++++++++ packages/back-end/src/integrations/Mssql.ts | 6 ++---- packages/back-end/src/integrations/Postgres.ts | 7 +++++++ packages/back-end/src/integrations/Redshift.ts | 13 +++++++++++++ packages/back-end/src/integrations/Snowflake.ts | 16 ++++++++++------ .../back-end/src/integrations/SqlIntegration.ts | 12 +++--------- 8 files changed, 52 insertions(+), 37 deletions(-) diff --git a/packages/back-end/src/integrations/Athena.ts b/packages/back-end/src/integrations/Athena.ts index fb491278c2a..de714fff851 100644 --- a/packages/back-end/src/integrations/Athena.ts +++ b/packages/back-end/src/integrations/Athena.ts @@ -59,13 +59,17 @@ export default class Athena extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getEventsTrackedByDatasourceFromClause( - trackedEventTableName: string - ): string { + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + // MKTODO: Test if the database is necessary here + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); + } if (!this.params.catalog) throw new MissingDatasourceParamsError( "To automatically generate metrics for an Athena data source, you must define a default catalog." ); - return `${this.params.catalog}.${trackedEventTableName}`; + return `${this.params.database}.${this.params.catalog}.${trackedEventTableName}`; } } diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index ebb6881ce58..0bc480f6d3b 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -99,20 +99,6 @@ export default class BigQuery extends SqlIntegration { ): string { return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``; } - getEventsTrackedByDatasourceFromClause( - trackedEventTableName: string - ): string { - if (!this.params.projectId) - throw new Error( - "No projectId provided. To automatically generate metrics you must provide a projectId." - ); - if (!this.params.defaultDataset) - throw new MissingDatasourceParamsError( - "To automatically generate metrics for a BigQuery dataset, you must define a default dataset." - ); - return `\`${this.params.projectId}.${this.params.defaultDataset}.${trackedEventTableName}\``; - } - getAutoGeneratedMetricFromClause(event: string): string { if (!this.params.projectId) throw new Error( diff --git a/packages/back-end/src/integrations/ClickHouse.ts b/packages/back-end/src/integrations/ClickHouse.ts index 858052b91e1..c8754351fb8 100644 --- a/packages/back-end/src/integrations/ClickHouse.ts +++ b/packages/back-end/src/integrations/ClickHouse.ts @@ -1,6 +1,7 @@ import { ClickHouse as ClickHouseClient } from "clickhouse"; import { decryptDataSourceParams } from "../services/datasource"; import { ClickHouseConnectionParams } from "../../types/integrations/clickhouse"; +import { MissingDatasourceParamsError } from "../types/Integration"; import SqlIntegration from "./SqlIntegration"; export default class ClickHouse extends SqlIntegration { @@ -90,4 +91,12 @@ export default class ClickHouse extends SqlIntegration { ); return `table_schema IN ('${this.params.database}')`; } + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "No database provided. To automatically generate metrics, you must provide a database." + ); + } + return `${this.params.database}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/Mssql.ts b/packages/back-end/src/integrations/Mssql.ts index 82f1f1c4a05..6476d4592f5 100644 --- a/packages/back-end/src/integrations/Mssql.ts +++ b/packages/back-end/src/integrations/Mssql.ts @@ -77,12 +77,10 @@ export default class Mssql extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getEventsTrackedByDatasourceFromClause( - trackedEventTableName: string - ): string { + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { if (!this.params.defaultSchema) throw new Error( - "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where the Rudderstack tracked events are stored." + "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." ); return `${this.params.defaultSchema}.${trackedEventTableName}`; } diff --git a/packages/back-end/src/integrations/Postgres.ts b/packages/back-end/src/integrations/Postgres.ts index bd52c31316c..5b8f5ed2c4c 100644 --- a/packages/back-end/src/integrations/Postgres.ts +++ b/packages/back-end/src/integrations/Postgres.ts @@ -38,4 +38,11 @@ export default class Postgres extends SqlIntegration { getInformationSchemaWhereClause(): string { return "table_schema NOT IN ('pg_catalog', 'information_schema', 'pg_toast')"; } + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + if (!this.params.defaultSchema) + throw new Error( + "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." + ); + return `${this.params.defaultSchema}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/Redshift.ts b/packages/back-end/src/integrations/Redshift.ts index 7fcbeb1a90f..dda4a8f9450 100644 --- a/packages/back-end/src/integrations/Redshift.ts +++ b/packages/back-end/src/integrations/Redshift.ts @@ -1,6 +1,7 @@ import { PostgresConnectionParams } from "../../types/integrations/postgres"; import { decryptDataSourceParams } from "../services/datasource"; import { runPostgresQuery } from "../services/postgres"; +import { MissingDatasourceParamsError } from "../types/Integration"; import { FormatDialect } from "../util/sql"; import SqlIntegration from "./SqlIntegration"; @@ -40,4 +41,16 @@ export default class Redshift extends SqlIntegration { getInformationSchemaTableFromClause(): string { return "SVV_COLUMNS"; } + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); + } + if (!this.params.defaultSchema) + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default catalog." + ); + return `${this.params.database}.${this.params.defaultSchema}.${trackedEventTableName}`; + } } diff --git a/packages/back-end/src/integrations/Snowflake.ts b/packages/back-end/src/integrations/Snowflake.ts index a8c6121dcbd..3ccead1cd50 100644 --- a/packages/back-end/src/integrations/Snowflake.ts +++ b/packages/back-end/src/integrations/Snowflake.ts @@ -1,6 +1,7 @@ import { SnowflakeConnectionParams } from "../../types/integrations/snowflake"; import { decryptDataSourceParams } from "../services/datasource"; import { runSnowflakeQuery } from "../services/snowflake"; +import { MissingDatasourceParamsError } from "../types/Integration"; import { FormatDialect } from "../util/sql"; import SqlIntegration from "./SqlIntegration"; @@ -36,7 +37,7 @@ export default class Snowflake extends SqlIntegration { } getInformationSchemaFromClause(): string { if (!this.params.database) - throw new Error( + throw new MissingDatasourceParamsError( "No database provided. In order to get the information schema, you must provide a database." ); return `${this.params.database}.information_schema.columns`; @@ -47,13 +48,16 @@ export default class Snowflake extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getEventsTrackedByDatasourceFromClause( - trackedEventTableName: string - ): string { + getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "No database provided. To automatically generate metrics, you must provide a database." + ); + } if (!this.params.schema) - throw new Error( + throw new MissingDatasourceParamsError( "No schema provided. To automatically generate metrics, you must provide a schema." ); - return `${this.params.schema}.${trackedEventTableName}`; + return `${this.params.database}.${this.params.schema}.${trackedEventTableName}`; } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 478e8612961..790d6120b26 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1303,10 +1303,8 @@ export default abstract class SqlIntegration return { tableData }; } - getEventsTrackedByDatasourceFromClause( - trackedEventTableName: string - ): string { - return trackedEventTableName; + getAutoGeneratedMetricFromClause(event: string): string { + return event; } async getEventsTrackedByDatasource( @@ -1328,7 +1326,7 @@ export default abstract class SqlIntegration (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, (CASE WHEN (1 > 0) THEN 1 ELSE 0 END) as createForUser FROM - ${this.getEventsTrackedByDatasourceFromClause(trackedEventTableName)} + ${this.getAutoGeneratedMetricFromClause(trackedEventTableName)} GROUP BY event`; const results = await this.runQuery(format(sql, this.getFormatDialect())); @@ -1340,10 +1338,6 @@ export default abstract class SqlIntegration return results; } - getAutoGeneratedMetricFromClause(event: string): string { - return event; - } - getAutoGeneratedMetricTimestampColumn(schemaFormat: SchemaFormat): string { switch (schemaFormat) { case "segment" || "rudderstack": From 6ccfdfe1f12aef034b3140759acfca33aef04717 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 19 May 2023 09:19:53 -0400 Subject: [PATCH 12/39] Improves error handling --- .../back-end/src/controllers/datasources.ts | 16 +++---- .../src/jobs/createAutomaticMetrics.ts | 46 ++++++++++--------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 4ccc7dfb826..12f957e450c 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -629,8 +629,8 @@ export async function getAutoGeneratedMetricsToCreate( const dataSourceObj = await getDataSourceById(id, org.id); if (!dataSourceObj) { - res.status(403).json({ - status: 403, + res.status(404).json({ + status: 404, message: "Invalid data source: " + id, }); return; @@ -648,9 +648,8 @@ export async function getAutoGeneratedMetricsToCreate( !integration.settings.schemaFormat || !integration.getSourceProperties().supportsAutoGeneratedMetrics ) { - //TODO: Is this the correct error code? - res.status(403).json({ - status: 403, + res.status(400).json({ + status: 400, message: "This datasource does not support automatic metric generation.", }); return; @@ -662,11 +661,12 @@ export async function getAutoGeneratedMetricsToCreate( integration.settings.schemaFormat ); - if (!results.length) { + if (results.length) { return res.status(200).json({ status: 200, - results, - message: "No events found.", + results: [], + message: + "We were unable to identify any metrics to generate automatically for you.", }); } diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 1c1a3752d2c..c02005f7d70 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -9,6 +9,7 @@ import { getInformationSchemaById } from "../models/InformationSchemaModel"; import { fetchTableData } from "../services/informationSchema"; import { getPath } from "../util/informationSchemas"; import { Column } from "../types/Integration"; +import { logger } from "../util/logger"; const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; @@ -32,34 +33,34 @@ export default function (ag: Agenda) { console.log("made it to the job"); const { datasourceId, organization, metricsToCreate } = job.attrs.data; - if (!datasourceId || !organization || !metricsToCreate) return; - - const datasource = await getDataSourceById(datasourceId, organization); + try { + const datasource = await getDataSourceById(datasourceId, organization); - if (!datasource) return; + if (!datasource) throw new Error("No datasource"); - const integration = getSourceIntegrationObject(datasource); + const integration = getSourceIntegrationObject(datasource); - if ( - !integration.getAutoGeneratedMetricSqlQuery || - !integration.getSourceProperties().supportsAutoGeneratedMetrics - ) - return; + if ( + !integration.getAutoGeneratedMetricSqlQuery || + !integration.getSourceProperties().supportsAutoGeneratedMetrics + ) + throw new Error( + "Auto generated metrics not supported for this data source" + ); - const informationSchemaId = datasource.settings.informationSchemaId; + const informationSchemaId = datasource.settings.informationSchemaId; - if (!informationSchemaId) return; //TODO: Throw an error? + if (!informationSchemaId) throw new Error("No information schema id"); - const informationSchema = await getInformationSchemaById( - organization, - informationSchemaId - ); + const informationSchema = await getInformationSchemaById( + organization, + informationSchemaId + ); - if (!informationSchema) return; //TODO: Throw an error? + if (!informationSchema) throw new Error("No information schema"); - let informationSchemaTableId = ""; + let informationSchemaTableId = ""; - try { const metrics: Partial[] = []; for (const metric of metricsToCreate) { if (metric.createForUser) { @@ -84,7 +85,7 @@ export default function (ag: Agenda) { informationSchemaTableId ); - if (!tableData) return; //TODO: Throw an error? + if (!tableData) throw new Error("No table data"); const columns: Column[] = tableData?.map( (row: { column_name: string; data_type: string }) => { @@ -131,7 +132,10 @@ export default function (ag: Agenda) { } await insertMetrics(metrics); } catch (e) { - // Not sure what to do here yet - catch the errors, but what should I do with them? + logger.error( + e, + "Failed to generate automatic metrics. Reason: " + e.message + ); } } ); From d10ef04b9944487208e9dd9356038fbad9aa6375 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 19 May 2023 09:51:35 -0400 Subject: [PATCH 13/39] Resets controller logic back after testing, and renames helper method. --- packages/back-end/src/controllers/datasources.ts | 4 ++-- packages/back-end/src/integrations/SqlIntegration.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 12f957e450c..20754a4303f 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -661,10 +661,10 @@ export async function getAutoGeneratedMetricsToCreate( integration.settings.schemaFormat ); - if (results.length) { + if (!results.length) { return res.status(200).json({ status: 200, - results: [], + results, message: "We were unable to identify any metrics to generate automatically for you.", }); diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 790d6120b26..ca058dbab96 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -80,7 +80,7 @@ export default abstract class SqlIntegration activationDimension: true, pastExperiments: true, supportsInformationSchema: true, - supportsAutoGeneratedMetrics: this.isAutoGeneratedMetricSupported(), + supportsAutoGeneratedMetrics: this.isAutoGeneratingMetricSupported(), }; } @@ -89,7 +89,7 @@ export default abstract class SqlIntegration return true; } - isAutoGeneratedMetricSupported(): boolean { + isAutoGeneratingMetricSupported(): boolean { const supportedEventTrackers: SchemaFormat[] = ["segment", "rudderstack"]; if ( From 145b9cd080d2d3d11806ff17d3eb0dc7afa52b48 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Wed, 24 May 2023 06:49:51 -0400 Subject: [PATCH 14/39] Refactored a bit based on Jeremy's feedback, needs a lot of cleanup, but committing in, instead of stashing it while I go bash a bug. --- .../back-end/src/controllers/datasources.ts | 2 + packages/back-end/src/integrations/Athena.ts | 29 ++- .../back-end/src/integrations/BigQuery.ts | 17 +- packages/back-end/src/integrations/Mssql.ts | 26 +-- packages/back-end/src/integrations/Presto.ts | 9 +- .../back-end/src/integrations/Redshift.ts | 27 +-- .../back-end/src/integrations/Snowflake.ts | 27 ++- .../src/integrations/SqlIntegration.ts | 166 ++++++++++++++---- .../src/jobs/createAutomaticMetrics.ts | 15 +- packages/back-end/types/datasource.d.ts | 8 + .../components/Settings/NewDataSourceForm.tsx | 8 +- 11 files changed, 224 insertions(+), 110 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 20754a4303f..9cb2c15e69e 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -424,6 +424,8 @@ export async function putDataSource( event: string; hasUserId: boolean; createForUser: boolean; + lastTrackedAt: Date; + count: number; }[]; }, { id: string } diff --git a/packages/back-end/src/integrations/Athena.ts b/packages/back-end/src/integrations/Athena.ts index de714fff851..1cf43e22946 100644 --- a/packages/back-end/src/integrations/Athena.ts +++ b/packages/back-end/src/integrations/Athena.ts @@ -49,27 +49,24 @@ export default class Athena extends SqlIntegration { ensureFloat(col: string): string { return `1.0*${col}`; } - getInformationSchemaFromClause(): string { - if (!this.params.catalog) - throw new MissingDatasourceParamsError( - "To view the information schema for an Athena data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings." - ); - return `${this.params.catalog}.information_schema.columns`; - } getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { - // MKTODO: Test if the database is necessary here - if (!this.params.database) { - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default database." - ); - } + generateTableName(tableName?: string): string { if (!this.params.catalog) throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default catalog." + "To view the information schema for an Athena data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings." ); - return `${this.params.database}.${this.params.catalog}.${trackedEventTableName}`; + if (tableName) { + // MKTODO: Test if the database is necessary here + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); + } + return `${this.params.database}.${this.params.catalog}.${tableName}`; + } else { + return `${this.params.catalog}.information_schema.columns`; + } } } diff --git a/packages/back-end/src/integrations/BigQuery.ts b/packages/back-end/src/integrations/BigQuery.ts index 0bc480f6d3b..4fb1b474533 100644 --- a/packages/back-end/src/integrations/BigQuery.ts +++ b/packages/back-end/src/integrations/BigQuery.ts @@ -82,24 +82,13 @@ export default class BigQuery extends SqlIntegration { castUserDateCol(column: string): string { return `CAST(${column} as DATETIME)`; } - getInformationSchemaFromClause(): string { - if (!this.params.projectId) - throw new Error( - "No projectId provided. In order to get the information schema, you must provide a projectId." - ); - if (!this.params.defaultDataset) - throw new MissingDatasourceParamsError( - "To view the information schema for a BigQuery dataset, you must define a default dataset. Please add a default dataset by editing the datasource's connection settings." - ); - return `\`${this.params.projectId}.${this.params.defaultDataset}.INFORMATION_SCHEMA.COLUMNS\``; - } getInformationSchemaTableFromClause( databaseName: string, tableSchema: string ): string { return `\`${databaseName}.${tableSchema}.INFORMATION_SCHEMA.COLUMNS\``; } - getAutoGeneratedMetricFromClause(event: string): string { + generateTableName(event?: string): string { if (!this.params.projectId) throw new Error( "No projectId provided. To automatically generate metrics you must provide a projectId." @@ -108,6 +97,8 @@ export default class BigQuery extends SqlIntegration { throw new MissingDatasourceParamsError( "To automatically generate metrics for a BigQuery dataset, you must define a default dataset." ); - return `\`${this.params.projectId}.${this.params.defaultDataset}.${event}\``; + return `\`${this.params.projectId}.${this.params.defaultDataset}.${ + event || "INFORMATION_SCHEMA.COLUMNS" + }\``; } } diff --git a/packages/back-end/src/integrations/Mssql.ts b/packages/back-end/src/integrations/Mssql.ts index 6476d4592f5..daa48395d7b 100644 --- a/packages/back-end/src/integrations/Mssql.ts +++ b/packages/back-end/src/integrations/Mssql.ts @@ -67,21 +67,23 @@ export default class Mssql extends SqlIntegration { formatDateTimeString(col: string): string { return `CONVERT(VARCHAR(25), ${col}, 121)`; } - getInformationSchemaFromClause(): string { - if (!this.params.database) - throw new MissingDatasourceParamsError( - "To view the information schema for a MS Sql dataset, you must define a default database. Please add a default database by editing the datasource's connection settings." - ); - return `${this.params.database}.information_schema.columns`; - } getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { - if (!this.params.defaultSchema) - throw new Error( - "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." + generateTableName(tableName?: string): string { + if (!this.params.database) + throw new MissingDatasourceParamsError( + "To view the information schema for a MS Sql dataset, you must define a default database. Please add a default database by editing the datasource's connection settings." ); - return `${this.params.defaultSchema}.${trackedEventTableName}`; + if (tableName) { + //MK TODO: Validate if the defaultSchema is necessary here + if (!this.params.defaultSchema) + throw new Error( + "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." + ); + return `${this.params.defaultSchema}.${tableName}`; + } else { + return `${this.params.database}.information_schema.columns`; + } } } diff --git a/packages/back-end/src/integrations/Presto.ts b/packages/back-end/src/integrations/Presto.ts index d9a6fb65d49..2a931d2f508 100644 --- a/packages/back-end/src/integrations/Presto.ts +++ b/packages/back-end/src/integrations/Presto.ts @@ -106,14 +106,15 @@ export default class Presto extends SqlIntegration { ensureFloat(col: string): string { return `CAST(${col} AS DOUBLE)`; } - getInformationSchemaFromClause(): string { + getInformationSchemaTableFromClause(databaseName: string): string { + return `${databaseName}.information_schema.columns`; + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + generateTableName(tableName?: string) { if (!this.params.catalog) throw new MissingDatasourceParamsError( "To view the information schema for a Presto data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings." ); return `${this.params.catalog}.information_schema.columns`; } - getInformationSchemaTableFromClause(databaseName: string): string { - return `${databaseName}.information_schema.columns`; - } } diff --git a/packages/back-end/src/integrations/Redshift.ts b/packages/back-end/src/integrations/Redshift.ts index dda4a8f9450..1f52d9ea4f0 100644 --- a/packages/back-end/src/integrations/Redshift.ts +++ b/packages/back-end/src/integrations/Redshift.ts @@ -35,22 +35,23 @@ export default class Redshift extends SqlIntegration { ensureFloat(col: string): string { return `${col}::float`; } - getInformationSchemaFromClause(): string { - return "SVV_COLUMNS"; - } getInformationSchemaTableFromClause(): string { return "SVV_COLUMNS"; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { - if (!this.params.database) { - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default database." - ); + generateTableName(tableName?: string): string { + if (tableName) { + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); + } + if (!this.params.defaultSchema) + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default catalog." + ); + return `${this.params.database}.${this.params.defaultSchema}.${tableName}`; + } else { + return "SVV_COLUMNS"; } - if (!this.params.defaultSchema) - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default catalog." - ); - return `${this.params.database}.${this.params.defaultSchema}.${trackedEventTableName}`; } } diff --git a/packages/back-end/src/integrations/Snowflake.ts b/packages/back-end/src/integrations/Snowflake.ts index 3ccead1cd50..1944120fb7c 100644 --- a/packages/back-end/src/integrations/Snowflake.ts +++ b/packages/back-end/src/integrations/Snowflake.ts @@ -35,29 +35,26 @@ export default class Snowflake extends SqlIntegration { ensureFloat(col: string): string { return `CAST(${col} AS DOUBLE)`; } - getInformationSchemaFromClause(): string { - if (!this.params.database) - throw new MissingDatasourceParamsError( - "No database provided. In order to get the information schema, you must provide a database." - ); - return `${this.params.database}.information_schema.columns`; - } getInformationSchemaWhereClause(): string { return "table_schema NOT IN ('INFORMATION_SCHEMA')"; } getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { - if (!this.params.database) { + generateTableName(tableName?: string): string { + if (!this.params.database) throw new MissingDatasourceParamsError( - "No database provided. To automatically generate metrics, you must provide a database." + "No database provided. In order to get the information schema, you must provide a database." ); + + if (tableName) { + if (!this.params.schema) + throw new MissingDatasourceParamsError( + "No schema provided. To automatically generate metrics, you must provide a schema." + ); + return `${this.params.database}.${this.params.schema}.${tableName}`; + } else { + return `${this.params.database}.information_schema.columns`; } - if (!this.params.schema) - throw new MissingDatasourceParamsError( - "No schema provided. To automatically generate metrics, you must provide a schema." - ); - return `${this.params.database}.${this.params.schema}.${trackedEventTableName}`; } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index ca058dbab96..e13638d7a5f 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -8,6 +8,7 @@ import { ExposureQuery, DataSourceType, SchemaFormat, + SchemaFormatConfig, } from "../../types/datasource"; import { MetricValueParams, @@ -1245,9 +1246,11 @@ export default abstract class SqlIntegration async getExperimentResults(): Promise { throw new Error("Not implemented"); } - getInformationSchemaFromClause(): string { - return "information_schema.columns"; + + generateTableName(tableName?: string): string { + return tableName || "information_schema.columns"; } + getInformationSchemaWhereClause(): string { return "table_schema NOT IN ('information_schema')"; } @@ -1267,7 +1270,7 @@ export default abstract class SqlIntegration table_schema as table_schema, count(column_name) as column_count FROM - ${this.getInformationSchemaFromClause()} + ${this.generateTableName()} WHERE ${this.getInformationSchemaWhereClause()} GROUP BY table_name, table_schema, table_catalog`; @@ -1303,50 +1306,140 @@ export default abstract class SqlIntegration return { tableData }; } - getAutoGeneratedMetricFromClause(event: string): string { - return event; + getSchemaFormatConfig(schemaFormat: SchemaFormat): SchemaFormatConfig { + switch (schemaFormat) { + // case "amplitude": TODO: Add support for amplitude later + // return { + // trackedEventTableName: "tracks", + // eventColumn: "event", + // timestampColumn: "server_upload_time", + // userIdColumn: "user_id", + // anonymousIdColumn: "amplitude_id", + // }; + // Segment & Rudderstack + default: + return { + trackedEventTableName: "tracks", + eventColumn: "event", + timestampColumn: "received_at", + userIdColumn: "user_id", + anonymousIdColumn: "anonymous_id", + }; + } + } + + toTitleCase(s: string): string { + return s + .replace(/^[-_]*(.)/, (_, c) => c.toUpperCase()) + .replace(/[-_]+(.)/g, (_, c) => " " + c.toUpperCase()); } async getEventsTrackedByDatasource( schemaFormat: SchemaFormat ): Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]> { - let eventColumn; - let trackedEventTableName; + const { + trackedEventTableName, + eventColumn, + timestampColumn, + } = this.getSchemaFormatConfig(schemaFormat); + + //MKTODO: Refactor how we're building the date string, it needs to be in the YYYY-mm-dd format + //MKTODO: Verify this sql works for non big query tables (it worked for MsSql & BigQuery) + // Create a date object from a date string + const currentDateTime = new Date(); + const sevenDaysAgo = new Date( + currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 + ); - switch (schemaFormat) { - //TODO: Add cases for schemaFormats where the tracked events aren't stored in a column called "events" - default: - eventColumn = "event"; - trackedEventTableName = "tracks"; - } + // Generate yyyy-mm-dd date string + const currentDateTimeString = + "'" + + currentDateTime.toLocaleString("default", { year: "numeric" }) + + "-" + + currentDateTime.toLocaleString("default", { + month: "2-digit", + }) + + "-" + + currentDateTime.toLocaleString("default", { day: "2-digit" }) + + "'"; + + // Generate yyyy-mm-dd date string + const sevenDaysAgoDateTimeString = + "'" + + sevenDaysAgo.toLocaleString("default", { year: "numeric" }) + + "-" + + sevenDaysAgo.toLocaleString("default", { + month: "2-digit", + }) + + "-" + + sevenDaysAgo.toLocaleString("default", { day: "2-digit" }) + + "'"; const sql = ` SELECT ${eventColumn} as event, + ${timestampColumn}, (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, - (CASE WHEN (1 > 0) THEN 1 ELSE 0 END) as createForUser + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt FROM - ${this.getAutoGeneratedMetricFromClause(trackedEventTableName)} - GROUP BY event`; + ${this.generateTableName(trackedEventTableName)} + WHERE received_at > ${sevenDaysAgoDateTimeString} AND received_at < ${currentDateTimeString} + AND event NOT IN ('experiment_viewed', 'experiment_started') + GROUP BY event, ${timestampColumn}`; + + const pageViewedSql = ` + SELECT + context_page_title as event, + ${timestampColumn}, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName("pages")} + GROUP BY event, ${timestampColumn} + ORDER BY lastTrackedAt DESC + LIMIT 1`; const results = await this.runQuery(format(sql, this.getFormatDialect())); - if (!results) { + const pageViewedResults = await this.runQuery( + format(pageViewedSql, this.getFormatDialect()) + ); + + console.log("results: ", results); + + if (!results && !pageViewedResults) { throw new Error(`No events found.`); } - return results; - } + const formattedResults: { + event: string; + displayName: string; + createForUser: boolean; + lastTrackedAt: Date; + hasUserId: boolean; + count: number; + }[] = []; + + results.forEach((result) => { + result.createForUser = true; + result.lastTrackedAt = new Date(result.lastTrackedAt.value); + result.displayName = + schemaFormat === "segment" + ? this.toTitleCase(result.event) + : result.event; + formattedResults.push(result); + }); - getAutoGeneratedMetricTimestampColumn(schemaFormat: SchemaFormat): string { - switch (schemaFormat) { - case "segment" || "rudderstack": - return "received_at"; - case "amplitude": - return "server_upload_time"; - default: - return "timestamp"; + if (pageViewedResults.length) { + pageViewedResults[0].event = "pages"; + pageViewedResults[0].displayName = "Page Viewed"; + pageViewedResults[0].createForUser = true; + formattedResults.push(pageViewedResults[0]); } + + return formattedResults; } getAutoGeneratedMetricValueColumn( @@ -1368,23 +1461,30 @@ export default abstract class SqlIntegration getAutoGeneratedMetricSqlQuery( metric: { event: string; + displayName: string; hasUserId: boolean; createForUser: boolean; + lastTrackedAt: Date; + count: number; }, schemaFormat: SchemaFormat, metricType: MetricType ): string { + const { + timestampColumn, + userIdColumn, + anonymousIdColumn, + } = this.getSchemaFormatConfig(schemaFormat); + const sqlQuery = ` SELECT - ${metric.hasUserId ? "user_id, " : ""}${ - schemaFormat === "amplitude" ? "amplitude_id" : "anonymous_id" - } as anonymous_id, ${this.getAutoGeneratedMetricTimestampColumn( - schemaFormat - )} as timestamp + ${ + metric.hasUserId ? `${userIdColumn}, ` : "" + }${anonymousIdColumn} as anonymous_id, ${timestampColumn} as timestamp ${this.getAutoGeneratedMetricValueColumn( metricType, schemaFormat - )} FROM ${this.getAutoGeneratedMetricFromClause(metric.event)} + )} FROM ${this.generateTableName(metric.event)} `; return format(sqlQuery, this.getFormatDialect()); diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index c02005f7d70..1bc4cdd8063 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -20,9 +20,16 @@ type CreateAutomaticMetricsJob = Job<{ event: string; hasUserId: boolean; createForUser: boolean; + displayName: string; + lastTrackedAt: Date; + count: number; }[]; }>; +function toSnakeCase(s: string): string { + return s.replace(" ", "_").toLowerCase(); +} + let agenda: Agenda; export default function (ag: Agenda) { agenda = ag; @@ -30,7 +37,6 @@ export default function (ag: Agenda) { agenda.define( CREATE_AUTOMATIC_METRICS_JOB_NAME, async (job: CreateAutomaticMetricsJob) => { - console.log("made it to the job"); const { datasourceId, organization, metricsToCreate } = job.attrs.data; try { @@ -67,7 +73,10 @@ export default function (ag: Agenda) { informationSchema.databases.forEach((database) => { database.schemas.forEach((schema) => { schema.tables.forEach((table) => { - if (table.tableName === metric.event) { + if ( + table.tableName === metric.event || + table.tableName === toSnakeCase(metric.event) + ) { informationSchemaTableId = table.id; } }); @@ -122,7 +131,7 @@ export default function (ag: Agenda) { id: uniqid("met_"), organization, datasource: datasourceId, - name: metric.event, + name: metric.displayName, type: metricType, sql: sqlQuery, dateCreated: new Date(), diff --git a/packages/back-end/types/datasource.d.ts b/packages/back-end/types/datasource.d.ts index 64fffc0ebef..54fbe41b95c 100644 --- a/packages/back-end/types/datasource.d.ts +++ b/packages/back-end/types/datasource.d.ts @@ -81,6 +81,14 @@ export interface SchemaInterface { getMetricSQL(name: string, type: MetricType, tablePrefix: string): string; } +export interface SchemaFormatConfig { + userIdColumn: string; + anonymousIdColumn: string; + trackedEventTableName: string; + eventColumn: string; + timestampColumn: string; +} + export interface DataSourceProperties { queryLanguage: QueryLanguage; metricCaps?: boolean; diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index c34f4316f98..7eddfc8255e 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -70,6 +70,9 @@ const NewDataSourceForm: FC<{ event: string; hasUserId: boolean; createForUser: boolean; + displayName: string; + lastTrackedAt: Date; + count: number; }[] >([]); @@ -577,8 +580,11 @@ const NewDataSourceForm: FC<{ const res = await apiCall<{ results: { event: string; + displayName: string; hasUserId: boolean; createForUser: boolean; + lastTrackedAt: Date; + count: number; }[]; message?: string; }>(`/datasource/${dataSourceId}/auto-metrics`, { @@ -620,7 +626,7 @@ const NewDataSourceForm: FC<{ setMetricsToCreate(newMetricsToCreate); }} /> - +
); })} From 08c37e143c1adc3075d01b2f5def50bfec92f377 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Wed, 24 May 2023 14:33:25 -0400 Subject: [PATCH 15/39] Starts scaffolding out how we can build additional metrics when retreiving the list of tracked events. --- .../src/integrations/SqlIntegration.ts | 188 ++++++++++-------- .../src/jobs/createAutomaticMetrics.ts | 89 +++++---- packages/back-end/src/types/Integration.ts | 3 +- 3 files changed, 152 insertions(+), 128 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index e13638d7a5f..a402161c7a1 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1328,10 +1328,63 @@ export default abstract class SqlIntegration } } - toTitleCase(s: string): string { - return s - .replace(/^[-_]*(.)/, (_, c) => c.toUpperCase()) - .replace(/[-_]+(.)/g, (_, c) => " " + c.toUpperCase()); + formatTrackedEventResults(results: any[], schemaFormat: SchemaFormat): any[] { + function toTitleCase(input: string): string { + return input + .replace(/^[-_]*(.)/, (_, c) => c.toUpperCase()) + .replace(/[-_]+(.)/g, (_, c) => " " + c.toUpperCase()); + } + + const formattedResults: { + event: string; + displayName: string; + createForUser: boolean; + lastTrackedAt: Date; + hasUserId: boolean; + count: number; + }[] = []; + + results.forEach((result) => { + result.createForUser = true; + result.lastTrackedAt = new Date(result.lastTrackedAt.value); + result.displayName = result.event; + result.type = "binomial"; + + if (schemaFormat === "segment") { + result.displayName = toTitleCase(result.event); + } + formattedResults.push(result); + }); + + return formattedResults; + } + + generateMetricsToCreate( + formattedResults: any[], + timestampColumn: string, + userIdColumn: string, + anonymousIdColumn: string + ): any[] { + const metricsToCreate = cloneDeep(formattedResults); + // What are we trying to do here? Let's say formattedResults has 4 metrics - Placed Order, Added to Cart, Viewed Product, Subscribed, & Page Viewed + formattedResults.forEach((result) => { + // Do something here + if (result.event === "placed_order") { + const metric = { + event: result.event, + displayName: "Average Order Value", + createForUser: true, + lastTrackedAt: undefined, + hasUserId: result.hasUserId, + type: "revenue", + valueClause: "AVG(revenue)", + groupByClause: `GROUP BY ${userIdColumn}, ${anonymousIdColumn}, ${timestampColumn}`, + }; + metricsToCreate.push(metric); + } + }); + + return metricsToCreate; } async getEventsTrackedByDatasource( @@ -1341,40 +1394,15 @@ export default abstract class SqlIntegration trackedEventTableName, eventColumn, timestampColumn, + userIdColumn, + anonymousIdColumn, } = this.getSchemaFormatConfig(schemaFormat); - //MKTODO: Refactor how we're building the date string, it needs to be in the YYYY-mm-dd format - //MKTODO: Verify this sql works for non big query tables (it worked for MsSql & BigQuery) - // Create a date object from a date string const currentDateTime = new Date(); const sevenDaysAgo = new Date( currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 ); - // Generate yyyy-mm-dd date string - const currentDateTimeString = - "'" + - currentDateTime.toLocaleString("default", { year: "numeric" }) + - "-" + - currentDateTime.toLocaleString("default", { - month: "2-digit", - }) + - "-" + - currentDateTime.toLocaleString("default", { day: "2-digit" }) + - "'"; - - // Generate yyyy-mm-dd date string - const sevenDaysAgoDateTimeString = - "'" + - sevenDaysAgo.toLocaleString("default", { year: "numeric" }) + - "-" + - sevenDaysAgo.toLocaleString("default", { - month: "2-digit", - }) + - "-" + - sevenDaysAgo.toLocaleString("default", { day: "2-digit" }) + - "'"; - const sql = ` SELECT ${eventColumn} as event, @@ -1384,62 +1412,53 @@ export default abstract class SqlIntegration MAX(${timestampColumn}) as lastTrackedAt FROM ${this.generateTableName(trackedEventTableName)} - WHERE received_at > ${sevenDaysAgoDateTimeString} AND received_at < ${currentDateTimeString} + WHERE received_at < '${currentDateTime + .toISOString() + .slice(0, 10)}' AND received_at > '${sevenDaysAgo + .toISOString() + .slice(0, 10)}' AND event NOT IN ('experiment_viewed', 'experiment_started') GROUP BY event, ${timestampColumn}`; - const pageViewedSql = ` - SELECT - context_page_title as event, - ${timestampColumn}, - (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, - COUNT (*) as count, - MAX(${timestampColumn}) as lastTrackedAt - FROM - ${this.generateTableName("pages")} - GROUP BY event, ${timestampColumn} - ORDER BY lastTrackedAt DESC - LIMIT 1`; - const results = await this.runQuery(format(sql, this.getFormatDialect())); - const pageViewedResults = await this.runQuery( - format(pageViewedSql, this.getFormatDialect()) - ); + if (schemaFormat === "segment") { + const pageViewedSql = ` + SELECT + 'page-viewed' as event, + ${timestampColumn}, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName("pages")} + GROUP BY event, ${timestampColumn} + ORDER BY lastTrackedAt DESC + LIMIT 1`; - console.log("results: ", results); + const pageViewedResults = await this.runQuery( + format(pageViewedSql, this.getFormatDialect()) + ); - if (!results && !pageViewedResults) { - throw new Error(`No events found.`); + pageViewedResults.forEach((result) => results.push(result)); } - const formattedResults: { - event: string; - displayName: string; - createForUser: boolean; - lastTrackedAt: Date; - hasUserId: boolean; - count: number; - }[] = []; - - results.forEach((result) => { - result.createForUser = true; - result.lastTrackedAt = new Date(result.lastTrackedAt.value); - result.displayName = - schemaFormat === "segment" - ? this.toTitleCase(result.event) - : result.event; - formattedResults.push(result); - }); - - if (pageViewedResults.length) { - pageViewedResults[0].event = "pages"; - pageViewedResults[0].displayName = "Page Viewed"; - pageViewedResults[0].createForUser = true; - formattedResults.push(pageViewedResults[0]); + if (!results) { + throw new Error(`No events found.`); } - return formattedResults; + const formattedResults = this.formatTrackedEventResults( + results, + schemaFormat + ); + + // Now that we have the formattedResults, is there where we should build logic to define additional metrics? + return this.generateMetricsToCreate( + formattedResults, + userIdColumn, + timestampColumn, + anonymousIdColumn + ); } getAutoGeneratedMetricValueColumn( @@ -1466,9 +1485,11 @@ export default abstract class SqlIntegration createForUser: boolean; lastTrackedAt: Date; count: number; + valueClause?: string; + type?: MetricType; + groupByClause?: string; }, - schemaFormat: SchemaFormat, - metricType: MetricType + schemaFormat: SchemaFormat ): string { const { timestampColumn, @@ -1481,10 +1502,13 @@ export default abstract class SqlIntegration ${ metric.hasUserId ? `${userIdColumn}, ` : "" }${anonymousIdColumn} as anonymous_id, ${timestampColumn} as timestamp - ${this.getAutoGeneratedMetricValueColumn( - metricType, - schemaFormat - )} FROM ${this.generateTableName(metric.event)} + ${ + metric.type !== "binomial" && metric.valueClause + ? `, ${metric.valueClause} as value` + : "" + } + FROM ${this.generateTableName(metric.event)} + ${metric.groupByClause || ""} `; return format(sqlQuery, this.getFormatDialect()); diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 1bc4cdd8063..d7239488ba6 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -6,9 +6,6 @@ import { insertMetrics } from "../models/MetricModel"; import { MetricInterface, MetricType } from "../../types/metric"; import { getSourceIntegrationObject } from "../services/datasource"; import { getInformationSchemaById } from "../models/InformationSchemaModel"; -import { fetchTableData } from "../services/informationSchema"; -import { getPath } from "../util/informationSchemas"; -import { Column } from "../types/Integration"; import { logger } from "../util/logger"; const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; @@ -23,6 +20,9 @@ type CreateAutomaticMetricsJob = Job<{ displayName: string; lastTrackedAt: Date; count: number; + type: MetricType; + valueClause: string; + groupByClause: string; }[]; }>; @@ -83,49 +83,50 @@ export default function (ag: Agenda) { }); }); - const { - tableData, - databaseName, - tableSchema, - tableName, - } = await fetchTableData( - datasource, - informationSchema, - informationSchemaTableId - ); - - if (!tableData) throw new Error("No table data"); - - const columns: Column[] = tableData?.map( - (row: { column_name: string; data_type: string }) => { - return { - columnName: row.column_name, - dataType: row.data_type, - path: getPath(datasource.type, { - tableCatalog: databaseName, - tableSchema: tableSchema, - tableName: tableName, - columnName: row.column_name, - }), - }; - } - ); - - let metricType: MetricType = "binomial"; - - if (columns.length) { - if (columns.some((column) => column.columnName === "revenue")) { - metricType = "revenue"; - } else if ( - columns.some((column) => column.columnName === "count") - ) { - metricType = "count"; - } - } + // const { + // tableData, + // databaseName, + // tableSchema, + // tableName, + // } = await fetchTableData( + // datasource, + // informationSchema, + // informationSchemaTableId + // ); + + // if (!tableData) throw new Error("No table data"); + + // const columns: Column[] = tableData?.map( + // (row: { column_name: string; data_type: string }) => { + // return { + // columnName: row.column_name, + // dataType: row.data_type, + // path: getPath(datasource.type, { + // tableCatalog: databaseName, + // tableSchema: tableSchema, + // tableName: tableName, + // columnName: row.column_name, + // }), + // }; + // } + // ); + + const metricType = metric.type; + + // let metricType: MetricType = "binomial"; + + // if (columns.length) { + // if (columns.some((column) => column.columnName === "revenue")) { + // metricType = "revenue"; + // } else if ( + // columns.some((column) => column.columnName === "count") + // ) { + // metricType = "count"; + // } + // } const sqlQuery = integration.getAutoGeneratedMetricSqlQuery( metric, - integration.settings.schemaFormat || "custom", - metricType + integration.settings.schemaFormat || "custom" ); metrics.push({ id: uniqid("met_"), diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 48e8ead6bc1..a3aac48d5ea 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -302,7 +302,6 @@ export interface SourceIntegrationInterface { hasUserId: boolean; createForUser: boolean; }, - schemaFormat: SchemaFormat, - metricType: MetricType + schemaFormat: SchemaFormat ): string; } From 63d97887f50a7f812c318af7708f008d4d8b653e Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Wed, 24 May 2023 16:21:28 -0400 Subject: [PATCH 16/39] Adds better typing and refactors logic to simplify some things, while also adding hooks to generate additional metrics from single tracked event. Very narrow in scope for now, but getting the building blocks in place. --- .../back-end/src/controllers/datasources.ts | 3 +- .../src/integrations/SqlIntegration.ts | 101 +++++++++--------- .../src/jobs/createAutomaticMetrics.ts | 41 +++---- packages/back-end/src/types/Integration.ts | 20 ++-- 4 files changed, 81 insertions(+), 84 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 9cb2c15e69e..74f807b2b8d 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -46,6 +46,7 @@ import { EventAuditUserForResponseLocals } from "../events/event-types"; import { deleteInformationSchemaById } from "../models/InformationSchemaModel"; import { deleteInformationSchemaTablesByInformationSchemaId } from "../models/InformationSchemaTablesModel"; import { queueCreateAutomaticMetrics } from "../jobs/createAutomaticMetrics"; +import { AutoGeneratedMetric } from "../types/Integration"; export async function postSampleData( req: AuthRequest, @@ -659,7 +660,7 @@ export async function getAutoGeneratedMetricsToCreate( try { // Get the list of events tracked by this datasource - we can create metrics from these events. - const results = await integration.getEventsTrackedByDatasource( + const results: AutoGeneratedMetric[] = await integration.getEventsTrackedByDatasource( integration.settings.schemaFormat ); diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index a402161c7a1..6ea750dd4d3 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -25,6 +25,7 @@ import { MetricAggregationType, InformationSchema, RawInformationSchema, + AutoGeneratedMetric, } from "../types/Integration"; import { ExperimentPhase, ExperimentInterface } from "../../types/experiment"; import { DimensionInterface } from "../../types/dimension"; @@ -1328,30 +1329,32 @@ export default abstract class SqlIntegration } } - formatTrackedEventResults(results: any[], schemaFormat: SchemaFormat): any[] { + formatTrackedEventResults( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + results: any[], + schemaFormat: SchemaFormat + ): Omit[] { function toTitleCase(input: string): string { return input .replace(/^[-_]*(.)/, (_, c) => c.toUpperCase()) .replace(/[-_]+(.)/g, (_, c) => " " + c.toUpperCase()); } - const formattedResults: { - event: string; - displayName: string; - createForUser: boolean; - lastTrackedAt: Date; - hasUserId: boolean; - count: number; - }[] = []; + const formattedResults: Omit< + AutoGeneratedMetric, + "type" | "valueClause" | "groupByClause" + >[] = []; results.forEach((result) => { + result.event; result.createForUser = true; result.lastTrackedAt = new Date(result.lastTrackedAt.value); result.displayName = result.event; - result.type = "binomial"; + result.count; if (schemaFormat === "segment") { - result.displayName = toTitleCase(result.event); + result.displayName = + result.event === "pages" ? "Paged Viewed" : toTitleCase(result.event); } formattedResults.push(result); }); @@ -1360,27 +1363,32 @@ export default abstract class SqlIntegration } generateMetricsToCreate( - formattedResults: any[], + formattedResults: Omit< + AutoGeneratedMetric, + "type" | "valueClause" | "groupByClause" + >[], timestampColumn: string, userIdColumn: string, anonymousIdColumn: string - ): any[] { - const metricsToCreate = cloneDeep(formattedResults); - // What are we trying to do here? Let's say formattedResults has 4 metrics - Placed Order, Added to Cart, Viewed Product, Subscribed, & Page Viewed + ): AutoGeneratedMetric[] { + const metricsToCreate: AutoGeneratedMetric[] = []; formattedResults.forEach((result) => { - // Do something here + const metric: AutoGeneratedMetric = { + event: result.event, + createForUser: true, + lastTrackedAt: result.lastTrackedAt, + hasUserId: !!result.hasUserId, + count: result.count, + displayName: result.displayName, + }; + metricsToCreate.push(metric); if (result.event === "placed_order") { - const metric = { - event: result.event, - displayName: "Average Order Value", - createForUser: true, - lastTrackedAt: undefined, - hasUserId: result.hasUserId, - type: "revenue", - valueClause: "AVG(revenue)", - groupByClause: `GROUP BY ${userIdColumn}, ${anonymousIdColumn}, ${timestampColumn}`, - }; - metricsToCreate.push(metric); + const newMetric = cloneDeep(metric); + newMetric.displayName = "Average Order Value"; + newMetric.type = "revenue"; + newMetric.valueClause = "AVG(revenue)"; + newMetric.groupByClause = `GROUP BY ${userIdColumn}, ${anonymousIdColumn}, ${timestampColumn}`; + metricsToCreate.push(newMetric); } }); @@ -1389,7 +1397,7 @@ export default abstract class SqlIntegration async getEventsTrackedByDatasource( schemaFormat: SchemaFormat - ): Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]> { + ): Promise { const { trackedEventTableName, eventColumn, @@ -1399,14 +1407,14 @@ export default abstract class SqlIntegration } = this.getSchemaFormatConfig(schemaFormat); const currentDateTime = new Date(); - const sevenDaysAgo = new Date( - currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 + //MKTODO: Re-think how to name this so if we change the calculation, it doesn't make the variable name incorrect + const ThirtyDaysAgo = new Date( + currentDateTime.valueOf() - 30 * 60 * 60 * 24 * 1000 ); const sql = ` SELECT ${eventColumn} as event, - ${timestampColumn}, (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt @@ -1414,9 +1422,10 @@ export default abstract class SqlIntegration ${this.generateTableName(trackedEventTableName)} WHERE received_at < '${currentDateTime .toISOString() - .slice(0, 10)}' AND received_at > '${sevenDaysAgo - .toISOString() - .slice(0, 10)}' + .slice( + 0, + 10 + )}' AND received_at > '${ThirtyDaysAgo.toISOString().slice(0, 10)}' AND event NOT IN ('experiment_viewed', 'experiment_started') GROUP BY event, ${timestampColumn}`; @@ -1425,8 +1434,7 @@ export default abstract class SqlIntegration if (schemaFormat === "segment") { const pageViewedSql = ` SELECT - 'page-viewed' as event, - ${timestampColumn}, + 'pages' as event, (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt @@ -1447,12 +1455,11 @@ export default abstract class SqlIntegration throw new Error(`No events found.`); } - const formattedResults = this.formatTrackedEventResults( - results, - schemaFormat - ); + const formattedResults: Omit< + AutoGeneratedMetric, + "type" | "valueClause" | "groupByClause" + >[] = this.formatTrackedEventResults(results, schemaFormat); - // Now that we have the formattedResults, is there where we should build logic to define additional metrics? return this.generateMetricsToCreate( formattedResults, userIdColumn, @@ -1478,17 +1485,7 @@ export default abstract class SqlIntegration } getAutoGeneratedMetricSqlQuery( - metric: { - event: string; - displayName: string; - hasUserId: boolean; - createForUser: boolean; - lastTrackedAt: Date; - count: number; - valueClause?: string; - type?: MetricType; - groupByClause?: string; - }, + metric: AutoGeneratedMetric, schemaFormat: SchemaFormat ): string { const { diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index d7239488ba6..6e03d72e62a 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -3,27 +3,18 @@ import Agenda, { Job } from "agenda"; import uniqid from "uniqid"; import { getDataSourceById } from "../models/DataSourceModel"; import { insertMetrics } from "../models/MetricModel"; -import { MetricInterface, MetricType } from "../../types/metric"; +import { MetricInterface } from "../../types/metric"; import { getSourceIntegrationObject } from "../services/datasource"; import { getInformationSchemaById } from "../models/InformationSchemaModel"; import { logger } from "../util/logger"; +import { AutoGeneratedMetric } from "../types/Integration"; const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; type CreateAutomaticMetricsJob = Job<{ organization: string; datasourceId: string; - metricsToCreate: { - event: string; - hasUserId: boolean; - createForUser: boolean; - displayName: string; - lastTrackedAt: Date; - count: number; - type: MetricType; - valueClause: string; - groupByClause: string; - }[]; + metricsToCreate: AutoGeneratedMetric[]; }>; function toSnakeCase(s: string): string { @@ -65,23 +56,23 @@ export default function (ag: Agenda) { if (!informationSchema) throw new Error("No information schema"); - let informationSchemaTableId = ""; + // let informationSchemaTableId = ""; const metrics: Partial[] = []; for (const metric of metricsToCreate) { if (metric.createForUser) { - informationSchema.databases.forEach((database) => { - database.schemas.forEach((schema) => { - schema.tables.forEach((table) => { - if ( - table.tableName === metric.event || - table.tableName === toSnakeCase(metric.event) - ) { - informationSchemaTableId = table.id; - } - }); - }); - }); + // informationSchema.databases.forEach((database) => { + // database.schemas.forEach((schema) => { + // schema.tables.forEach((table) => { + // if ( + // table.tableName === metric.event || + // table.tableName === toSnakeCase(metric.event) + // ) { + // informationSchemaTableId = table.id; + // } + // }); + // }); + // }); // const { // tableData, diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index a3aac48d5ea..5b4c39e5921 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -131,6 +131,18 @@ export type PastExperimentResult = { }[]; }; +export type AutoGeneratedMetric = { + event: string; + hasUserId: boolean; + createForUser: boolean; + displayName: string; + lastTrackedAt: Date | null; + count: number; + type?: MetricType; + valueClause?: string; + groupByClause?: string; +}; + export type MetricValueQueryResponseRow = { date: string; count: number; @@ -295,13 +307,9 @@ export interface SourceIntegrationInterface { runPastExperimentQuery(query: string): Promise; getEventsTrackedByDatasource?: ( schemaFormat: SchemaFormat - ) => Promise<{ event: string; hasUserId: boolean; createForUser: boolean }[]>; + ) => Promise; getAutoGeneratedMetricSqlQuery?( - metric: { - event: string; - hasUserId: boolean; - createForUser: boolean; - }, + metric: AutoGeneratedMetric, schemaFormat: SchemaFormat ): string; } From 5a161ec128f372fb7220e00bd8bfebbff8634762 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 26 May 2023 10:18:51 -0400 Subject: [PATCH 17/39] Includes a big refactor that simplifies things a lot - we're now getting all tracked events, and giving the user the option to create a binomial and/or count metric for each event. --- .../src/integrations/SqlIntegration.ts | 180 +++++------------- .../src/jobs/createAutomaticMetrics.ts | 130 ++++--------- packages/back-end/src/types/Integration.ts | 9 +- packages/back-end/types/datasource.d.ts | 1 + .../components/Settings/NewDataSourceForm.tsx | 146 ++++++++++++-- 5 files changed, 222 insertions(+), 244 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 6ea750dd4d3..453aaf7dc9c 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1309,15 +1309,25 @@ export default abstract class SqlIntegration getSchemaFormatConfig(schemaFormat: SchemaFormat): SchemaFormatConfig { switch (schemaFormat) { - // case "amplitude": TODO: Add support for amplitude later - // return { - // trackedEventTableName: "tracks", - // eventColumn: "event", - // timestampColumn: "server_upload_time", - // userIdColumn: "user_id", - // anonymousIdColumn: "amplitude_id", - // }; - // Segment & Rudderstack + case "amplitude": + return { + trackedEventTableName: "tracks", + eventColumn: "event", + timestampColumn: "server_upload_time", + userIdColumn: "user_id", + anonymousIdColumn: "amplitude_id", + }; + case "segment": { + return { + trackedEventTableName: "tracks", + eventColumn: "event", + timestampColumn: "received_at", + userIdColumn: "user_id", + anonymousIdColumn: "anonymous_id", + displayNameColumn: "event_text", + }; + } + // Rudderstack default: return { trackedEventTableName: "tracks", @@ -1329,70 +1339,27 @@ export default abstract class SqlIntegration } } - formatTrackedEventResults( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - results: any[], - schemaFormat: SchemaFormat - ): Omit[] { - function toTitleCase(input: string): string { - return input - .replace(/^[-_]*(.)/, (_, c) => c.toUpperCase()) - .replace(/[-_]+(.)/g, (_, c) => " " + c.toUpperCase()); - } - - const formattedResults: Omit< - AutoGeneratedMetric, - "type" | "valueClause" | "groupByClause" - >[] = []; - - results.forEach((result) => { - result.event; - result.createForUser = true; - result.lastTrackedAt = new Date(result.lastTrackedAt.value); - result.displayName = result.event; - result.count; - - if (schemaFormat === "segment") { - result.displayName = - result.event === "pages" ? "Paged Viewed" : toTitleCase(result.event); - } - formattedResults.push(result); - }); - - return formattedResults; - } + getAutoGeneratedMetricSqlQuery( + metric: AutoGeneratedMetric, + schemaFormat: SchemaFormat, + type: MetricType + ): string { + const { + timestampColumn, + userIdColumn, + anonymousIdColumn, + } = this.getSchemaFormatConfig(schemaFormat); - generateMetricsToCreate( - formattedResults: Omit< - AutoGeneratedMetric, - "type" | "valueClause" | "groupByClause" - >[], - timestampColumn: string, - userIdColumn: string, - anonymousIdColumn: string - ): AutoGeneratedMetric[] { - const metricsToCreate: AutoGeneratedMetric[] = []; - formattedResults.forEach((result) => { - const metric: AutoGeneratedMetric = { - event: result.event, - createForUser: true, - lastTrackedAt: result.lastTrackedAt, - hasUserId: !!result.hasUserId, - count: result.count, - displayName: result.displayName, - }; - metricsToCreate.push(metric); - if (result.event === "placed_order") { - const newMetric = cloneDeep(metric); - newMetric.displayName = "Average Order Value"; - newMetric.type = "revenue"; - newMetric.valueClause = "AVG(revenue)"; - newMetric.groupByClause = `GROUP BY ${userIdColumn}, ${anonymousIdColumn}, ${timestampColumn}`; - metricsToCreate.push(newMetric); - } - }); + const sqlQuery = ` + SELECT + ${ + metric.hasUserId ? `${userIdColumn}, ` : "" + }${anonymousIdColumn} as anonymous_id, ${timestampColumn} as timestamp + ${type === "count" && metric.createCountFromEvent ? `, 1 as value` : ""} + FROM ${this.generateTableName(metric.event)} + `; - return metricsToCreate; + return format(sqlQuery, this.getFormatDialect()); } async getEventsTrackedByDatasource( @@ -1402,12 +1369,11 @@ export default abstract class SqlIntegration trackedEventTableName, eventColumn, timestampColumn, - userIdColumn, - anonymousIdColumn, + displayNameColumn, } = this.getSchemaFormatConfig(schemaFormat); const currentDateTime = new Date(); - //MKTODO: Re-think how to name this so if we change the calculation, it doesn't make the variable name incorrect + //MKTODO: Re-think how to name & calculate this so if we change the calculation, it doesn't make the variable name incorrect const ThirtyDaysAgo = new Date( currentDateTime.valueOf() - 30 * 60 * 60 * 24 * 1000 ); @@ -1415,6 +1381,7 @@ export default abstract class SqlIntegration const sql = ` SELECT ${eventColumn} as event, + ${displayNameColumn ? displayNameColumn : eventColumn} as displayName, (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt @@ -1427,7 +1394,7 @@ export default abstract class SqlIntegration 10 )}' AND received_at > '${ThirtyDaysAgo.toISOString().slice(0, 10)}' AND event NOT IN ('experiment_viewed', 'experiment_started') - GROUP BY event, ${timestampColumn}`; + GROUP BY event, ${timestampColumn}, displayName`; const results = await this.runQuery(format(sql, this.getFormatDialect())); @@ -1435,6 +1402,7 @@ export default abstract class SqlIntegration const pageViewedSql = ` SELECT 'pages' as event, + "Page Viewed" as displayName, (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt @@ -1455,60 +1423,14 @@ export default abstract class SqlIntegration throw new Error(`No events found.`); } - const formattedResults: Omit< - AutoGeneratedMetric, - "type" | "valueClause" | "groupByClause" - >[] = this.formatTrackedEventResults(results, schemaFormat); - - return this.generateMetricsToCreate( - formattedResults, - userIdColumn, - timestampColumn, - anonymousIdColumn - ); - } - - getAutoGeneratedMetricValueColumn( - metricType: MetricType, - schemaFormat: SchemaFormat - ): string { - const countColumn = schemaFormat === "rudderstack" ? "value" : "count"; - - switch (metricType) { - case "count": - return `, ${countColumn} as value`; - case "revenue": - return ", revenue as value"; - default: - return ""; - } - } - - getAutoGeneratedMetricSqlQuery( - metric: AutoGeneratedMetric, - schemaFormat: SchemaFormat - ): string { - const { - timestampColumn, - userIdColumn, - anonymousIdColumn, - } = this.getSchemaFormatConfig(schemaFormat); - - const sqlQuery = ` - SELECT - ${ - metric.hasUserId ? `${userIdColumn}, ` : "" - }${anonymousIdColumn} as anonymous_id, ${timestampColumn} as timestamp - ${ - metric.type !== "binomial" && metric.valueClause - ? `, ${metric.valueClause} as value` - : "" - } - FROM ${this.generateTableName(metric.event)} - ${metric.groupByClause || ""} - `; - - return format(sqlQuery, this.getFormatDialect()); + return results.map((result) => { + result.createBinomialFromEvent = true; + result.createCountFromEvent = true; + result.lastTrackedAt = result.lastTrackedAt + ? new Date(result.lastTrackedAt) + : null; + return result; + }); } private getMetricQueryFormat(metric: MetricInterface) { diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 6e03d72e62a..6f29ac333bc 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -1,11 +1,11 @@ /* eslint-disable no-console */ import Agenda, { Job } from "agenda"; import uniqid from "uniqid"; +import { cloneDeep } from "lodash"; import { getDataSourceById } from "../models/DataSourceModel"; import { insertMetrics } from "../models/MetricModel"; import { MetricInterface } from "../../types/metric"; import { getSourceIntegrationObject } from "../services/datasource"; -import { getInformationSchemaById } from "../models/InformationSchemaModel"; import { logger } from "../util/logger"; import { AutoGeneratedMetric } from "../types/Integration"; @@ -17,10 +17,6 @@ type CreateAutomaticMetricsJob = Job<{ metricsToCreate: AutoGeneratedMetric[]; }>; -function toSnakeCase(s: string): string { - return s.replace(" ", "_").toLowerCase(); -} - let agenda: Agenda; export default function (ag: Agenda) { agenda = ag; @@ -35,6 +31,13 @@ export default function (ag: Agenda) { if (!datasource) throw new Error("No datasource"); + const schemaFormat = datasource.settings.schemaFormat || "custom"; + + if (!schemaFormat || schemaFormat === "custom") + throw new Error( + `Unable to automatically generate metrics for a ${schemaFormat} schema format.` + ); + const integration = getSourceIntegrationObject(datasource); if ( @@ -45,90 +48,41 @@ export default function (ag: Agenda) { "Auto generated metrics not supported for this data source" ); - const informationSchemaId = datasource.settings.informationSchemaId; - - if (!informationSchemaId) throw new Error("No information schema id"); - - const informationSchema = await getInformationSchemaById( - organization, - informationSchemaId - ); - - if (!informationSchema) throw new Error("No information schema"); - - // let informationSchemaTableId = ""; - const metrics: Partial[] = []; for (const metric of metricsToCreate) { - if (metric.createForUser) { - // informationSchema.databases.forEach((database) => { - // database.schemas.forEach((schema) => { - // schema.tables.forEach((table) => { - // if ( - // table.tableName === metric.event || - // table.tableName === toSnakeCase(metric.event) - // ) { - // informationSchemaTableId = table.id; - // } - // }); - // }); - // }); - - // const { - // tableData, - // databaseName, - // tableSchema, - // tableName, - // } = await fetchTableData( - // datasource, - // informationSchema, - // informationSchemaTableId - // ); - - // if (!tableData) throw new Error("No table data"); - - // const columns: Column[] = tableData?.map( - // (row: { column_name: string; data_type: string }) => { - // return { - // columnName: row.column_name, - // dataType: row.data_type, - // path: getPath(datasource.type, { - // tableCatalog: databaseName, - // tableSchema: tableSchema, - // tableName: tableName, - // columnName: row.column_name, - // }), - // }; - // } - // ); - - const metricType = metric.type; - - // let metricType: MetricType = "binomial"; - - // if (columns.length) { - // if (columns.some((column) => column.columnName === "revenue")) { - // metricType = "revenue"; - // } else if ( - // columns.some((column) => column.columnName === "count") - // ) { - // metricType = "count"; - // } - // } - const sqlQuery = integration.getAutoGeneratedMetricSqlQuery( + const baseMetric: Partial = { + organization, + datasource: datasourceId, + name: metric.displayName, + dateCreated: new Date(), + dateUpdated: new Date(), + }; + if (metric.createBinomialFromEvent) { + const metricToCreate: Partial = cloneDeep( + baseMetric + ); + metricToCreate.id = uniqid("met_"); + metricToCreate.type = "binomial"; + metricToCreate.sql = integration.getAutoGeneratedMetricSqlQuery( + metric, + schemaFormat, + "binomial" + ); + metrics.push(metricToCreate); + } + if (metric.createCountFromEvent) { + const metricToCreate: Partial = cloneDeep( + baseMetric + ); + metricToCreate.id = uniqid("met_"); + metricToCreate.type = "count"; + metricToCreate.name = `Number of ${metric.displayName}`; //TODO: This needs to be customized + metricToCreate.sql = integration.getAutoGeneratedMetricSqlQuery( metric, - integration.settings.schemaFormat || "custom" + schemaFormat, + "count" ); - metrics.push({ - id: uniqid("met_"), - organization, - datasource: datasourceId, - name: metric.displayName, - type: metricType, - sql: sqlQuery, - dateCreated: new Date(), - dateUpdated: new Date(), - }); + metrics.push(metricToCreate); } } await insertMetrics(metrics); @@ -145,11 +99,7 @@ export default function (ag: Agenda) { export async function queueCreateAutomaticMetrics( datasourceId: string, organization: string, - metricsToCreate: { - event: string; - hasUserId: boolean; - createForUser: boolean; - }[] + metricsToCreate: AutoGeneratedMetric[] ) { if (!datasourceId || !organization || !metricsToCreate) return; diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 5b4c39e5921..f56e5b086b1 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -134,13 +134,11 @@ export type PastExperimentResult = { export type AutoGeneratedMetric = { event: string; hasUserId: boolean; - createForUser: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; displayName: string; lastTrackedAt: Date | null; count: number; - type?: MetricType; - valueClause?: string; - groupByClause?: string; }; export type MetricValueQueryResponseRow = { @@ -310,6 +308,7 @@ export interface SourceIntegrationInterface { ) => Promise; getAutoGeneratedMetricSqlQuery?( metric: AutoGeneratedMetric, - schemaFormat: SchemaFormat + schemaFormat: SchemaFormat, + type: MetricType ): string; } diff --git a/packages/back-end/types/datasource.d.ts b/packages/back-end/types/datasource.d.ts index 54fbe41b95c..75861298fca 100644 --- a/packages/back-end/types/datasource.d.ts +++ b/packages/back-end/types/datasource.d.ts @@ -87,6 +87,7 @@ export interface SchemaFormatConfig { trackedEventTableName: string; eventColumn: string; timestampColumn: string; + displayNameColumn?: string; } export interface DataSourceProperties { diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 7eddfc8255e..4162ce2a776 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -69,7 +69,8 @@ const NewDataSourceForm: FC<{ { event: string; hasUserId: boolean; - createForUser: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; displayName: string; lastTrackedAt: Date; count: number; @@ -113,7 +114,8 @@ const NewDataSourceForm: FC<{ metricsToCreate: { event: string; hasUserId: boolean; - createForUser: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; }[]; }>({ defaultValues: { @@ -325,6 +327,16 @@ const NewDataSourceForm: FC<{ await updateSettings(); } if (isFinalStep) { + if (metricsToCreate.length > 0) { + track("Generating Auto Metrics For User", { + metricsToCreate, + source, + type: datasource.type, + dataSourceId, + schema: schema, + newDatasourceForm: true, + }); + } // @ts-expect-error TS(2345) If you come across this, please fix it!: Argument of type 'string | null' is not assignable... Remove this comment to see the full error message await onSuccess(newDataId); onCancel && onCancel(); @@ -560,7 +572,7 @@ const NewDataSourceForm: FC<{ ))} {datasourceSupportsAutoGeneratedMetrics && (
-

Metric Options

+

Generate Metrics Automatically

{metricsToCreate.length === 0 ? (
@@ -577,12 +589,20 @@ const NewDataSourceForm: FC<{ onClick={async () => { setAutoMetricError(""); try { + track("Generate Auto Metrics CTA Clicked", { + source, + type: datasource.type, + dataSourceId, + schema: schema, + newDatasourceForm: true, + }); const res = await apiCall<{ results: { event: string; displayName: string; hasUserId: boolean; - createForUser: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; lastTrackedAt: Date; count: number; }[]; @@ -591,11 +611,27 @@ const NewDataSourceForm: FC<{ method: "GET", }); if (res.message) { + track("Generate Auto Metrics Error", { + error: res.message, + source, + type: datasource.type, + dataSourceId, + schema: schema, + newDatasourceForm: true, + }); setAutoMetricError(res.message); return; } setMetricsToCreate(res.results); } catch (e) { + track("Generate Auto Metrics Error", { + error: e.message, + source, + type: datasource.type, + dataSourceId, + schema: schema, + newDatasourceForm: true, + }); setAutoMetricError(e.message); } }} @@ -608,25 +644,95 @@ const NewDataSourceForm: FC<{ ) : (

- These are the metrics we've found that we can generate - for you automatically. Once created, you can always edit - these if you need to. + {`These are the tracked events we found from ${ + schemasMap.get(schema)?.label || "" + } in your connected warehouse. We can use these events to automatically + generate the following metrics for you to give you a head + start on setting up your account.`}{" "} + + You can always edit and remove these metrics at anytime + after they're created. +

+ {metricsToCreate.length > 0 && ( +
+ + +
+ )} {metricsToCreate.map((metric, i) => { return ( -
- { - const newMetricsToCreate = cloneDeep( - metricsToCreate - ); - newMetricsToCreate[i].createForUser = value; - setMetricsToCreate(newMetricsToCreate); - }} - /> - +
+
+

Tracked Event: {metric.displayName}

+
+
+
+ { + const newMetricsToCreate = cloneDeep( + metricsToCreate + ); + newMetricsToCreate[ + i + ].createBinomialFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> + +
+
+ { + const newMetricsToCreate = cloneDeep( + metricsToCreate + ); + newMetricsToCreate[ + i + ].createCountFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> + +
+
); })} From c99eccc746aaf1b9e7dbab301cfe1f7b073eb894 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 26 May 2023 12:09:46 -0400 Subject: [PATCH 18/39] Improves the front end experience. --- .../src/integrations/SqlIntegration.ts | 11 +- packages/back-end/src/types/Integration.ts | 2 +- .../components/Settings/NewDataSourceForm.tsx | 115 +++++++++++------- 3 files changed, 77 insertions(+), 51 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 453aaf7dc9c..ebe8bb6410d 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1407,10 +1407,7 @@ export default abstract class SqlIntegration COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt FROM - ${this.generateTableName("pages")} - GROUP BY event, ${timestampColumn} - ORDER BY lastTrackedAt DESC - LIMIT 1`; + ${this.generateTableName("pages")}`; const pageViewedResults = await this.runQuery( format(pageViewedSql, this.getFormatDialect()) @@ -1426,9 +1423,9 @@ export default abstract class SqlIntegration return results.map((result) => { result.createBinomialFromEvent = true; result.createCountFromEvent = true; - result.lastTrackedAt = result.lastTrackedAt - ? new Date(result.lastTrackedAt) - : null; + result.lastTrackedAt = result.lastTrackedAt.value + ? new Date(result.lastTrackedAt.value) + : new Date(result.lastTrackedAt); return result; }); } diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index f56e5b086b1..5e07a4d615c 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -137,7 +137,7 @@ export type AutoGeneratedMetric = { createBinomialFromEvent: boolean; createCountFromEvent: boolean; displayName: string; - lastTrackedAt: Date | null; + lastTrackedAt: Date; count: number; }; diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 4162ce2a776..339ddad2b44 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -11,6 +11,7 @@ import { } from "back-end/types/datasource"; import { useForm } from "react-hook-form"; import { cloneDeep } from "lodash"; +import { ago } from "@/../shared/dates"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -28,6 +29,8 @@ import Modal from "../Modal"; import { GBCircleArrowLeft } from "../Icons"; import Button from "../Button"; import Toggle from "../Forms/Toggle"; +import { DocLink } from "../DocLink"; +import Tooltip from "../Tooltip/Tooltip"; import EventSourceList from "./EventSourceList"; import ConnectionSettings from "./ConnectionSettings"; @@ -572,14 +575,18 @@ const NewDataSourceForm: FC<{ ))} {datasourceSupportsAutoGeneratedMetrics && (
-

Generate Metrics Automatically

+

+ Generate Metrics Automatically + + New! + +

{metricsToCreate.length === 0 ? (
{`With ${ schemasMap.get(schema).label - }, we may be able to generate metrics for you - automatically,`}{" "} + }, we may be able to automatically generate metrics from your tracked events, `} saving you and your team valuable time. (It's Free) @@ -637,7 +644,7 @@ const NewDataSourceForm: FC<{ }} color="outline-primary" > - Generate Metrics + See What Metrics We Can Create
@@ -647,42 +654,51 @@ const NewDataSourceForm: FC<{ {`These are the tracked events we found from ${ schemasMap.get(schema)?.label || "" } in your connected warehouse. We can use these events to automatically - generate the following metrics for you to give you a head - start on setting up your account.`}{" "} - - You can always edit and remove these metrics at anytime - after they're created. - + generate the following metrics for you. And don't worry, you can always edit and remove these + metrics at anytime after they're created. `} + + Click here to learn more about GrowthBook Metrics. +

{metricsToCreate.length > 0 && ( -
- - +
+

+ Tracked Events{" "} + +

+
+ + +
)} {metricsToCreate.map((metric, i) => { @@ -691,8 +707,20 @@ const NewDataSourceForm: FC<{ key={`${metric}-${i}`} className="border rounded p-3 mb-1" > -
-

Tracked Event: {metric.displayName}

+
+
+

{metric.displayName}

+ + (Tracked {metric.count} time + {metric.count > 1 ? "s" : ""}) + +
+
+ Last tracked {ago(metric.lastTrackedAt)} +
@@ -710,7 +738,8 @@ const NewDataSourceForm: FC<{ }} />
@@ -728,8 +757,8 @@ const NewDataSourceForm: FC<{ }} />
From 33aa2a5fefc0e90e3b421dcc8dbf6195911d3702 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Sat, 27 May 2023 07:26:53 -0400 Subject: [PATCH 19/39] Introduces a method to build a count metric name that is pluralized, and adds polish to the front end. Still need to wire a few things on the front end up, mainly the SQL Preview. --- .../src/integrations/SqlIntegration.ts | 12 ++ .../src/jobs/createAutomaticMetrics.ts | 14 +- packages/back-end/src/types/Integration.ts | 3 + packages/back-end/src/util/metrics.ts | 16 ++ .../components/Settings/NewDataSourceForm.tsx | 165 +++++++++++++----- 5 files changed, 154 insertions(+), 56 deletions(-) create mode 100644 packages/back-end/src/util/metrics.ts diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index ebe8bb6410d..245f8428194 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -42,6 +42,7 @@ import { } from "../util/sql"; import { MetricRegressionAdjustmentStatus } from "../../types/report"; import { formatInformationSchema } from "../util/informationSchemas"; +import { getMetricPlural } from "../util/metrics"; export default abstract class SqlIntegration implements SourceIntegrationInterface { @@ -1426,6 +1427,17 @@ export default abstract class SqlIntegration result.lastTrackedAt = result.lastTrackedAt.value ? new Date(result.lastTrackedAt.value) : new Date(result.lastTrackedAt); + result.binomialSqlQuery = this.getAutoGeneratedMetricSqlQuery( + result, + schemaFormat, + "binomial" + ); + result.countSqlQuery = this.getAutoGeneratedMetricSqlQuery( + result, + schemaFormat, + "count" + ); + result.countDisplayName = getMetricPlural(result.displayName); return result; }); } diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutomaticMetrics.ts index 6f29ac333bc..429c2120f4e 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutomaticMetrics.ts @@ -63,11 +63,7 @@ export default function (ag: Agenda) { ); metricToCreate.id = uniqid("met_"); metricToCreate.type = "binomial"; - metricToCreate.sql = integration.getAutoGeneratedMetricSqlQuery( - metric, - schemaFormat, - "binomial" - ); + metricToCreate.sql = metric.binomialSqlQuery; metrics.push(metricToCreate); } if (metric.createCountFromEvent) { @@ -76,12 +72,8 @@ export default function (ag: Agenda) { ); metricToCreate.id = uniqid("met_"); metricToCreate.type = "count"; - metricToCreate.name = `Number of ${metric.displayName}`; //TODO: This needs to be customized - metricToCreate.sql = integration.getAutoGeneratedMetricSqlQuery( - metric, - schemaFormat, - "count" - ); + metricToCreate.name = metric.countDisplayName; + metricToCreate.sql = metric.countSqlQuery; metrics.push(metricToCreate); } } diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 5e07a4d615c..8a79421f4cd 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -139,6 +139,9 @@ export type AutoGeneratedMetric = { displayName: string; lastTrackedAt: Date; count: number; + binomialSqlQuery: string; + countSqlQuery: string; + countDisplayName: string; }; export type MetricValueQueryResponseRow = { diff --git a/packages/back-end/src/util/metrics.ts b/packages/back-end/src/util/metrics.ts new file mode 100644 index 00000000000..0c5dfb214e5 --- /dev/null +++ b/packages/back-end/src/util/metrics.ts @@ -0,0 +1,16 @@ +const eventsArray = [ + { name: "Signed Up", plural: "Count of Sign Ups" }, + { name: "Placed Order", plural: "Count of Orders" }, + { name: "Subscribed", plural: "Count of Subscriptions" }, + { name: "Page Viewed", plural: "Count of Page Views" }, +]; + +const events = new Map(); + +eventsArray.forEach((event) => { + events.set(event.name, event.plural); +}); + +export function getMetricPlural(metricName: string): string { + return events.get(metricName) || `Count of ${metricName}`; +} diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 339ddad2b44..017618173d4 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -12,6 +12,7 @@ import { import { useForm } from "react-hook-form"; import { cloneDeep } from "lodash"; import { ago } from "@/../shared/dates"; +import clsx from "clsx"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -60,6 +61,7 @@ const NewDataSourceForm: FC<{ mutateDefinitions, } = useDefinitions(); const [step, setStep] = useState(0); + const [showSqlPreview, setShowSqlPreview] = useState(false); const [schema, setSchema] = useState(""); const [dataSourceId, setDataSourceId] = useState( data?.id || null @@ -77,9 +79,14 @@ const NewDataSourceForm: FC<{ displayName: string; lastTrackedAt: Date; count: number; + binomialSqlQuery: string; + countSqlQuery: string; + countDisplayName: string; }[] >([]); + console.log("metricsToCreate", metricsToCreate); + const permissions = usePermissions(); const [datasource, setDatasource] = useState< @@ -526,6 +533,17 @@ const NewDataSourceForm: FC<{ } else { stepContents = (
+ {showSqlPreview && ( + setShowSqlPreview(false)} + size="md" + > + SQL PREVIEW + + )}
(`/datasource/${dataSourceId}/auto-metrics`, { @@ -642,7 +663,8 @@ const NewDataSourceForm: FC<{ setAutoMetricError(e.message); } }} - color="outline-primary" + color="warning" + className="font-weight-bold" > See What Metrics We Can Create @@ -661,7 +683,7 @@ const NewDataSourceForm: FC<{

{metricsToCreate.length > 0 && ( -
+

Tracked Events{" "} { return ( -
+

{metric.displayName}

@@ -714,52 +733,108 @@ const NewDataSourceForm: FC<{ className="pl-2 font-italic" body="Limited to the last 7 days." > - (Tracked {metric.count} time - {metric.count > 1 ? "s" : ""}) + (Count: {metric.count})
- Last tracked {ago(metric.lastTrackedAt)} + Last seen {ago(metric.lastTrackedAt)}
-
- { - const newMetricsToCreate = cloneDeep( - metricsToCreate - ); - newMetricsToCreate[ - i - ].createBinomialFromEvent = value; - setMetricsToCreate(newMetricsToCreate); - }} - /> - +
+
+ { + const newMetricsToCreate = cloneDeep( + metricsToCreate + ); + newMetricsToCreate[ + i + ].createBinomialFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
+
+

+ Metric Name: {metric.displayName}{" "} +

+

+ Type:{" "} + + binomial + +

+
+
+ +
-
- { - const newMetricsToCreate = cloneDeep( - metricsToCreate - ); - newMetricsToCreate[ - i - ].createCountFromEvent = value; - setMetricsToCreate(newMetricsToCreate); - }} - /> - +
+
+ { + const newMetricsToCreate = cloneDeep( + metricsToCreate + ); + newMetricsToCreate[ + i + ].createCountFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
+
+

+ Metric Name: {metric.countDisplayName}{" "} +

+

+ Type:{" "} + + count + +

+
+
+ +
From 7d275d5fc3aaf54d08afe34333af5f46dfbbdbcc Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Sun, 28 May 2023 07:32:31 -0400 Subject: [PATCH 20/39] Improved the UI a bit to include the SQL Preview functionality, and built out the pluralization map further, inspired from Segment's article on e-comm tracking suggestions. --- .../src/integrations/SqlIntegration.ts | 4 +- .../back-end/src/util/autoGeneratedMetrics.ts | 53 ++++++ packages/back-end/src/util/metrics.ts | 16 -- .../front-end/components/SQLInputField.tsx | 64 +++---- .../components/Settings/AutoMetricCard.tsx | 163 ++++++++++++++++++ .../components/Settings/NewDataSourceForm.tsx | 143 ++------------- 6 files changed, 265 insertions(+), 178 deletions(-) create mode 100644 packages/back-end/src/util/autoGeneratedMetrics.ts delete mode 100644 packages/back-end/src/util/metrics.ts create mode 100644 packages/front-end/components/Settings/AutoMetricCard.tsx diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 245f8428194..9a0c4c9879b 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -42,7 +42,7 @@ import { } from "../util/sql"; import { MetricRegressionAdjustmentStatus } from "../../types/report"; import { formatInformationSchema } from "../util/informationSchemas"; -import { getMetricPlural } from "../util/metrics"; +import { getPluralizedMetricName } from "../util/autoGeneratedMetrics"; export default abstract class SqlIntegration implements SourceIntegrationInterface { @@ -1437,7 +1437,7 @@ export default abstract class SqlIntegration schemaFormat, "count" ); - result.countDisplayName = getMetricPlural(result.displayName); + result.countDisplayName = getPluralizedMetricName(result.displayName); return result; }); } diff --git a/packages/back-end/src/util/autoGeneratedMetrics.ts b/packages/back-end/src/util/autoGeneratedMetrics.ts new file mode 100644 index 00000000000..17d18f81998 --- /dev/null +++ b/packages/back-end/src/util/autoGeneratedMetrics.ts @@ -0,0 +1,53 @@ +const eventsArray = [ + { name: "Signed Up", plural: "Sign Ups" }, + { name: "Placed Order", plural: "Orders Placed" }, + { name: "Subscribed", plural: "Subscriptions" }, + { name: "Page Viewed", plural: "Page Views" }, + { name: "Products Searched", plural: "Products Searches" }, + { name: "Product List Viewed", plural: "Count of Product List Views" }, + { name: "Product List Filtered", plural: "Count of Product List Filters" }, + { name: "Promotion Viewed", plural: "Promotion Views" }, + { name: "Promotion Clicked", plural: "Promotion Clicks" }, + { name: "Product Clicked", plural: "Product Clicks" }, + { name: "Product Viewed", plural: "Product Views" }, + { name: "Product Added", plural: "Product Additions" }, + { name: "Product Removed", plural: "Products Removals" }, + { name: "Cart Viewed", plural: "Carts Views" }, + { name: "Checkout Started", plural: "Checkouts Started" }, + { name: "Checkout Step Viewed", plural: "Checkout Step Views" }, + { name: "Checkout Step Completed", plural: "Checkout Step Completes" }, + { name: "Payment Info Entered", plural: "Count of Payment Info Entered" }, + { name: "Order Completed", plural: "Count of Completed Orders" }, + { name: "Order Updated", plural: "Count of Orders Updates" }, + { name: "Order Refunded", plural: "Count of Orders Refunds" }, + { name: "Order Cancelled", plural: "Count of Order Cancelletions" }, + { name: "Coupon Entered", plural: "Count of Coupons Entered" }, + { name: "Coupon Applied", plural: "Count of Coupons Applied" }, + { name: "Coupon Denied", plural: "Count of Coupons Denied" }, + { name: "Coupon Removed", plural: "Count of Coupons Removed" }, + { + name: "Product Added to Wishlist", + plural: "Count of Product Adds to Wishlist", + }, + { + name: "Product Removed from Wishlist", + plural: "Count of Product Removals from Wishlist", + }, + { + name: "Wishlist Product Added to Cart", + plural: "Count of Wishlist Product Added to Cart", + }, + { name: "Product Shared", plural: "Product Shares" }, + { name: "Cart Shared", plural: "Cart Shares" }, + { name: "Product Reviewed", plural: "Product Reviews" }, +]; + +const events = new Map(); + +eventsArray.forEach((event) => { + events.set(event.name, event.plural); +}); + +export function getPluralizedMetricName(metricName: string): string { + return events.get(metricName) || `Count of ${metricName}`; +} diff --git a/packages/back-end/src/util/metrics.ts b/packages/back-end/src/util/metrics.ts deleted file mode 100644 index 0c5dfb214e5..00000000000 --- a/packages/back-end/src/util/metrics.ts +++ /dev/null @@ -1,16 +0,0 @@ -const eventsArray = [ - { name: "Signed Up", plural: "Count of Sign Ups" }, - { name: "Placed Order", plural: "Count of Orders" }, - { name: "Subscribed", plural: "Count of Subscriptions" }, - { name: "Page Viewed", plural: "Count of Page Views" }, -]; - -const events = new Map(); - -eventsArray.forEach((event) => { - events.set(event.name, event.plural); -}); - -export function getMetricPlural(metricName: string): string { - return events.get(metricName) || `Count of ${metricName}`; -} diff --git a/packages/front-end/components/SQLInputField.tsx b/packages/front-end/components/SQLInputField.tsx index 204e3e8d635..d8007c54e84 100644 --- a/packages/front-end/components/SQLInputField.tsx +++ b/packages/front-end/components/SQLInputField.tsx @@ -31,6 +31,8 @@ type Props = { queryType: "segment" | "dimension" | "metric" | "experiment-assignment"; className?: string; setCursorData?: (data: CursorData) => void; + showHeader?: boolean; + showTestButton?: boolean; }; export default function SQLInputField({ @@ -45,6 +47,8 @@ export default function SQLInputField({ queryType, className, setCursorData, + showHeader = true, + showTestButton = true, }: Props) { const [ testQueryResults, @@ -168,41 +172,43 @@ export default function SQLInputField({ return (
- + {showHeader && }
-
- - {queryType === "experiment-assignment" ? ( -
- - - -
- ) : null} -
+ {showTestButton && ( +
+ + {queryType === "experiment-assignment" ? ( +
+ + + +
+ ) : null} +
+ )} {showPreview ? ( ) : ( diff --git a/packages/front-end/components/Settings/AutoMetricCard.tsx b/packages/front-end/components/Settings/AutoMetricCard.tsx new file mode 100644 index 00000000000..50c3be9ec00 --- /dev/null +++ b/packages/front-end/components/Settings/AutoMetricCard.tsx @@ -0,0 +1,163 @@ +import clsx from "clsx"; +import { ago } from "@/../shared/dates"; +import { cloneDeep } from "lodash"; +import { useState } from "react"; +import Tooltip from "../Tooltip/Tooltip"; +import Toggle from "../Forms/Toggle"; +import Button from "../Button"; +import SQLInputField from "../SQLInputField"; + +type Props = { + metric: any; + setMetricsToCreate: any; + metricsToCreate: any; + dataSourceId: string; + form: any; + i: number; +}; + +export default function AutoMetricCard({ + metric, + setMetricsToCreate, + metricsToCreate, + dataSourceId, + form, + i, +}: Props) { + const [showBinomialSqlPreview, setShowBinomialSqlPreview] = useState(false); + const [showCountSqlPreview, setShowCountSqlPreview] = useState(false); + + return ( +
+
+
+

{metric.displayName}

+ + (Count: {metric.count}) + +
+
Last seen {ago(metric.lastTrackedAt)}
+
+ +
+
+
+
+ { + const newMetricsToCreate = cloneDeep(metricsToCreate); + newMetricsToCreate[i].createBinomialFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
+
+

Metric Name: {metric.displayName}

+

+ Type:{" "} + + binomial + +

+
+
+ +
+
+
+ {showBinomialSqlPreview && ( + + )} +
+
+
+
+
+ { + const newMetricsToCreate = cloneDeep(metricsToCreate); + newMetricsToCreate[i].createCountFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
+
+

Metric Name: {metric.countDisplayName}

+

+ Type:{" "} + + count + +

+
+
+ +
+
+
+ {showCountSqlPreview && ( + + )} +
+
+
+
+ ); +} diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 017618173d4..f790212c07b 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -10,9 +10,6 @@ import { DataSourceSettings, } from "back-end/types/datasource"; import { useForm } from "react-hook-form"; -import { cloneDeep } from "lodash"; -import { ago } from "@/../shared/dates"; -import clsx from "clsx"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -29,11 +26,11 @@ import Field from "../Forms/Field"; import Modal from "../Modal"; import { GBCircleArrowLeft } from "../Icons"; import Button from "../Button"; -import Toggle from "../Forms/Toggle"; import { DocLink } from "../DocLink"; import Tooltip from "../Tooltip/Tooltip"; import EventSourceList from "./EventSourceList"; import ConnectionSettings from "./ConnectionSettings"; +import AutoMetricCard from "./AutoMetricCard"; const NewDataSourceForm: FC<{ data: Partial; @@ -61,7 +58,8 @@ const NewDataSourceForm: FC<{ mutateDefinitions, } = useDefinitions(); const [step, setStep] = useState(0); - const [showSqlPreview, setShowSqlPreview] = useState(false); + // const [showSqlPreview, setShowSqlPreview] = + // useState < { name: string, showPreview: boolean } >[]([]); const [schema, setSchema] = useState(""); const [dataSourceId, setDataSourceId] = useState( data?.id || null @@ -85,8 +83,6 @@ const NewDataSourceForm: FC<{ }[] >([]); - console.log("metricsToCreate", metricsToCreate); - const permissions = usePermissions(); const [datasource, setDatasource] = useState< @@ -533,17 +529,6 @@ const NewDataSourceForm: FC<{ } else { stepContents = (
- {showSqlPreview && ( - setShowSqlPreview(false)} - size="md" - > - SQL PREVIEW - - )} From 652eb758c3b3087cd3a341e85507d79addf14958 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 09:34:37 -0400 Subject: [PATCH 21/39] Switched the front end experience over to what Luke suggested - left the previous implementation in, but commented out, and I will make a follow-up commit to remove it, but I just want a snapshot in case we want to revert back to it. --- .../components/Settings/AutoMetricCard.tsx | 243 ++++++++---------- .../components/Settings/NewDataSourceForm.tsx | 129 ++++++++-- 2 files changed, 215 insertions(+), 157 deletions(-) diff --git a/packages/front-end/components/Settings/AutoMetricCard.tsx b/packages/front-end/components/Settings/AutoMetricCard.tsx index 50c3be9ec00..a6fe7b4fb74 100644 --- a/packages/front-end/components/Settings/AutoMetricCard.tsx +++ b/packages/front-end/components/Settings/AutoMetricCard.tsx @@ -1,4 +1,3 @@ -import clsx from "clsx"; import { ago } from "@/../shared/dates"; import { cloneDeep } from "lodash"; import { useState } from "react"; @@ -8,10 +7,46 @@ import Button from "../Button"; import SQLInputField from "../SQLInputField"; type Props = { - metric: any; - setMetricsToCreate: any; - metricsToCreate: any; + metric: { + event: string; + hasUserId: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; + displayName: string; + lastTrackedAt: Date; + count: number; + binomialSqlQuery: string; + countSqlQuery: string; + countDisplayName: string; + }; + setMetricsToCreate: ( + metrics: { + event: string; + hasUserId: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; + displayName: string; + lastTrackedAt: Date; + count: number; + binomialSqlQuery: string; + countSqlQuery: string; + countDisplayName: string; + }[] + ) => void; + metricsToCreate: { + event: string; + hasUserId: boolean; + createBinomialFromEvent: boolean; + createCountFromEvent: boolean; + displayName: string; + lastTrackedAt: Date; + count: number; + binomialSqlQuery: string; + countSqlQuery: string; + countDisplayName: string; + }[]; dataSourceId: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any form: any; i: number; }; @@ -24,140 +59,84 @@ export default function AutoMetricCard({ form, i, }: Props) { - const [showBinomialSqlPreview, setShowBinomialSqlPreview] = useState(false); - const [showCountSqlPreview, setShowCountSqlPreview] = useState(false); + const [showSqlPreview, setShowSqlPreview] = useState(""); + + const handleSqlPreview = async (sql: string) => { + if (!showSqlPreview || showSqlPreview !== sql) { + setShowSqlPreview(sql); + } else { + setShowSqlPreview(""); + } + }; return ( -
-
-
-

{metric.displayName}

+ <> + + {metric.displayName} + - (Count: {metric.count}) + {metric.count} -
-
Last seen {ago(metric.lastTrackedAt)}
-
- -
-
-
-
- { - const newMetricsToCreate = cloneDeep(metricsToCreate); - newMetricsToCreate[i].createBinomialFromEvent = value; - setMetricsToCreate(newMetricsToCreate); - }} - /> -
-
-

Metric Name: {metric.displayName}

-

- Type:{" "} - - binomial - -

-
-
- -
-
-
- {showBinomialSqlPreview && ( - - )} -
-
-
-
-
- { - const newMetricsToCreate = cloneDeep(metricsToCreate); - newMetricsToCreate[i].createCountFromEvent = value; - setMetricsToCreate(newMetricsToCreate); - }} - /> -
-
-

Metric Name: {metric.countDisplayName}

-

- Type:{" "} - - count - -

-
-
- -
+ + +
+ { + const newMetricsToCreate = cloneDeep(metricsToCreate); + newMetricsToCreate[i].createBinomialFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
-
- {showCountSqlPreview && ( - - )} + + +
+ { + const newMetricsToCreate = cloneDeep(metricsToCreate); + newMetricsToCreate[i].createCountFromEvent = value; + setMetricsToCreate(newMetricsToCreate); + }} + /> +
-
-
-
+ + + {showSqlPreview && ( + + + + + + )} + ); } diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index f790212c07b..aae0fc6ffa4 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -668,16 +668,8 @@ const NewDataSourceForm: FC<{

{metricsToCreate.length > 0 && ( -
-

- Tracked Events{" "} - -

-
+ <> +
-
+ + + + + + + + + + {metricsToCreate.map((metric, i) => { + return ( + + // <> + // + // + // + // + // + // + // {} + // + // SQL PREVIEW GOES HERE + // + // + ); + })} +
Metric NameCount + + Create Binomial Metric + + + {" "} + + Create Count Metric + +
{metric.displayName} + // + // {metric.count} + // + // + //
+ // { + // const newMetricsToCreate = cloneDeep( + // metricsToCreate + // ); + // newMetricsToCreate[ + // i + // ].createBinomialFromEvent = value; + // setMetricsToCreate(newMetricsToCreate); + // }} + // /> + // + //
+ // 		+ //
+ // { + // const newMetricsToCreate = cloneDeep( + // metricsToCreate + // ); + // newMetricsToCreate[ + // i + // ].createCountFromEvent = value; + // setMetricsToCreate(newMetricsToCreate); + // }} + // /> + // + //
+ //
+ )} - {metricsToCreate.map((metric, i) => { - return ( - - ); - })}
)}
From 10367b32635a744abe608b5900070542c6484647 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 09:35:11 -0400 Subject: [PATCH 22/39] Removed commented out code. --- .../components/Settings/NewDataSourceForm.tsx | 69 ------------------- 1 file changed, 69 deletions(-) diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index aae0fc6ffa4..8b5d092cad1 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -726,75 +726,6 @@ const NewDataSourceForm: FC<{ dataSourceId={dataSourceId || ""} metricsToCreate={metricsToCreate} /> - // <> - // - // {metric.displayName} - // - // - // {metric.count} - // - // - // - //
- // { - // const newMetricsToCreate = cloneDeep( - // metricsToCreate - // ); - // newMetricsToCreate[ - // i - // ].createBinomialFromEvent = value; - // setMetricsToCreate(newMetricsToCreate); - // }} - // /> - // - //
- // - // - //
- // { - // const newMetricsToCreate = cloneDeep( - // metricsToCreate - // ); - // newMetricsToCreate[ - // i - // ].createCountFromEvent = value; - // setMetricsToCreate(newMetricsToCreate); - // }} - // /> - // - //
- // - // - // {} - // - // SQL PREVIEW GOES HERE - // - // ); })} From 9785c2dc275b97d3b563b213078a53e0b8d7f509 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 09:41:44 -0400 Subject: [PATCH 23/39] Fixed a type issue --- packages/back-end/src/controllers/datasources.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 74f807b2b8d..2caacade39d 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -421,13 +421,7 @@ export async function putDataSource( params: DataSourceParams; settings: DataSourceSettings; projects?: string[]; - metricsToCreate?: { - event: string; - hasUserId: boolean; - createForUser: boolean; - lastTrackedAt: Date; - count: number; - }[]; + metricsToCreate?: AutoGeneratedMetric[]; }, { id: string } >, From 5f90e093f4e62a2fd058cddb78088820d7547d6e Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 12:28:43 -0400 Subject: [PATCH 24/39] First round of self PR review adjustments. --- packages/back-end/src/app.ts | 4 ++-- .../back-end/src/controllers/datasources.ts | 11 +++++---- packages/back-end/src/init/queue.ts | 4 ++-- ...trics.ts => createAutoGeneratedMetrics.ts} | 23 ++++++++---------- packages/back-end/src/types/Integration.ts | 4 ++-- .../front-end/components/SQLInputField.tsx | 4 +--- .../components/Settings/AutoMetricCard.tsx | 24 ++++++++++--------- .../components/Settings/NewDataSourceForm.tsx | 4 +--- 8 files changed, 38 insertions(+), 40 deletions(-) rename packages/back-end/src/jobs/{createAutomaticMetrics.ts => createAutoGeneratedMetrics.ts} (81%) diff --git a/packages/back-end/src/app.ts b/packages/back-end/src/app.ts index 962efbf103c..75abb40e983 100644 --- a/packages/back-end/src/app.ts +++ b/packages/back-end/src/app.ts @@ -482,9 +482,9 @@ app.get("/datasource/:id", datasourcesController.getDataSource); app.post("/datasources", datasourcesController.postDataSources); app.put("/datasource/:id", datasourcesController.putDataSource); app.delete("/datasource/:id", datasourcesController.deleteDataSource); -app.get( +app.post( "/datasource/:id/auto-metrics", - datasourcesController.getAutoGeneratedMetricsToCreate + datasourcesController.postAutoGeneratedMetricsToCreate ); // Information Schemas diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index 2caacade39d..c6665b8d8cc 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -45,7 +45,7 @@ import { import { EventAuditUserForResponseLocals } from "../events/event-types"; import { deleteInformationSchemaById } from "../models/InformationSchemaModel"; import { deleteInformationSchemaTablesByInformationSchemaId } from "../models/InformationSchemaTablesModel"; -import { queueCreateAutomaticMetrics } from "../jobs/createAutomaticMetrics"; +import { queueCreateAutoGeneratedMetrics } from "../jobs/createAutoGeneratedMetrics"; import { AutoGeneratedMetric } from "../types/Integration"; export async function postSampleData( @@ -466,7 +466,11 @@ export async function putDataSource( } if (metricsToCreate?.length) { - await queueCreateAutomaticMetrics(datasource.id, org.id, metricsToCreate); + await queueCreateAutoGeneratedMetrics( + datasource.id, + org.id, + metricsToCreate + ); } try { @@ -617,7 +621,7 @@ export async function testLimitedQuery( }); } -export async function getAutoGeneratedMetricsToCreate( +export async function postAutoGeneratedMetricsToCreate( req: AuthRequest, res: Response ) { @@ -641,7 +645,6 @@ export async function getAutoGeneratedMetricsToCreate( const integration = getSourceIntegrationObject(dataSourceObj); if ( - !integration.getEventsTrackedByDatasource || !integration.settings.schemaFormat || !integration.getSourceProperties().supportsAutoGeneratedMetrics ) { diff --git a/packages/back-end/src/init/queue.ts b/packages/back-end/src/init/queue.ts index 3fc227980f7..44e0b7f9c22 100644 --- a/packages/back-end/src/init/queue.ts +++ b/packages/back-end/src/init/queue.ts @@ -6,7 +6,7 @@ import addMetricUpdateJob from "../jobs/updateMetrics"; import addProxyUpdateJob from "../jobs/proxyUpdate"; import createInformationSchemaJob from "../jobs/createInformationSchema"; import updateInformationSchemaJob from "../jobs/updateInformationSchema"; -import createAutomaticMetrics from "../jobs/createAutomaticMetrics"; +import createAutoGeneratedMetrics from "../jobs/createAutoGeneratedMetrics"; import { CRON_ENABLED } from "../util/secrets"; import { getAgendaInstance } from "../services/queueing"; import updateStaleInformationSchemaTable from "../jobs/updateStaleInformationSchemaTable"; @@ -24,7 +24,7 @@ export async function queueInit() { addProxyUpdateJob(agenda); createInformationSchemaJob(agenda); updateInformationSchemaJob(agenda); - createAutomaticMetrics(agenda); + createAutoGeneratedMetrics(agenda); updateStaleInformationSchemaTable(agenda); await agenda.start(); diff --git a/packages/back-end/src/jobs/createAutomaticMetrics.ts b/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts similarity index 81% rename from packages/back-end/src/jobs/createAutomaticMetrics.ts rename to packages/back-end/src/jobs/createAutoGeneratedMetrics.ts index 429c2120f4e..9024843c9d0 100644 --- a/packages/back-end/src/jobs/createAutomaticMetrics.ts +++ b/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts @@ -9,9 +9,9 @@ import { getSourceIntegrationObject } from "../services/datasource"; import { logger } from "../util/logger"; import { AutoGeneratedMetric } from "../types/Integration"; -const CREATE_AUTOMATIC_METRICS_JOB_NAME = "createAutomaticMetrics"; +const CREATE_AUTOGENERATED_METRICS_JOB_NAME = "createAutoGeneratedMetrics"; -type CreateAutomaticMetricsJob = Job<{ +type CreateAutoGeneratedMetricsJob = Job<{ organization: string; datasourceId: string; metricsToCreate: AutoGeneratedMetric[]; @@ -22,8 +22,8 @@ export default function (ag: Agenda) { agenda = ag; agenda.define( - CREATE_AUTOMATIC_METRICS_JOB_NAME, - async (job: CreateAutomaticMetricsJob) => { + CREATE_AUTOGENERATED_METRICS_JOB_NAME, + async (job: CreateAutoGeneratedMetricsJob) => { const { datasourceId, organization, metricsToCreate } = job.attrs.data; try { @@ -33,17 +33,14 @@ export default function (ag: Agenda) { const schemaFormat = datasource.settings.schemaFormat || "custom"; - if (!schemaFormat || schemaFormat === "custom") + if (schemaFormat === "custom") throw new Error( - `Unable to automatically generate metrics for a ${schemaFormat} schema format.` + `Unable to automatically generate metrics for a custom schema format.` ); const integration = getSourceIntegrationObject(datasource); - if ( - !integration.getAutoGeneratedMetricSqlQuery || - !integration.getSourceProperties().supportsAutoGeneratedMetrics - ) + if (!integration.getSourceProperties().supportsAutoGeneratedMetrics) throw new Error( "Auto generated metrics not supported for this data source" ); @@ -88,18 +85,18 @@ export default function (ag: Agenda) { ); } -export async function queueCreateAutomaticMetrics( +export async function queueCreateAutoGeneratedMetrics( datasourceId: string, organization: string, metricsToCreate: AutoGeneratedMetric[] ) { if (!datasourceId || !organization || !metricsToCreate) return; - const job = agenda.create(CREATE_AUTOMATIC_METRICS_JOB_NAME, { + const job = agenda.create(CREATE_AUTOGENERATED_METRICS_JOB_NAME, { organization, datasourceId, metricsToCreate, - }) as CreateAutomaticMetricsJob; + }) as CreateAutoGeneratedMetricsJob; job.unique({ datasourceId, organization, metricsToCreate }); job.schedule(new Date()); await job.save(); diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 8a79421f4cd..56f24168c84 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -306,10 +306,10 @@ export interface SourceIntegrationInterface { query: string ): Promise; runPastExperimentQuery(query: string): Promise; - getEventsTrackedByDatasource?: ( + getEventsTrackedByDatasource: ( schemaFormat: SchemaFormat ) => Promise; - getAutoGeneratedMetricSqlQuery?( + getAutoGeneratedMetricSqlQuery( metric: AutoGeneratedMetric, schemaFormat: SchemaFormat, type: MetricType diff --git a/packages/front-end/components/SQLInputField.tsx b/packages/front-end/components/SQLInputField.tsx index d8007c54e84..bb9812fd569 100644 --- a/packages/front-end/components/SQLInputField.tsx +++ b/packages/front-end/components/SQLInputField.tsx @@ -31,7 +31,6 @@ type Props = { queryType: "segment" | "dimension" | "metric" | "experiment-assignment"; className?: string; setCursorData?: (data: CursorData) => void; - showHeader?: boolean; showTestButton?: boolean; }; @@ -47,7 +46,6 @@ export default function SQLInputField({ queryType, className, setCursorData, - showHeader = true, showTestButton = true, }: Props) { const [ @@ -172,7 +170,7 @@ export default function SQLInputField({ return (
- {showHeader && } +
(""); + const [sqlPreview, setSqlPreview] = useState(""); const handleSqlPreview = async (sql: string) => { - if (!showSqlPreview || showSqlPreview !== sql) { - setShowSqlPreview(sql); + if (!sqlPreview || sqlPreview !== sql) { + setSqlPreview(sql); } else { - setShowSqlPreview(""); + setSqlPreview(""); } }; return ( - <> - +
+ {metric.displayName} handleSqlPreview(metric.binomialSqlQuery)} > - {showSqlPreview && showSqlPreview === metric.binomialSqlQuery + {sqlPreview && sqlPreview === metric.binomialSqlQuery ? "Hide SQL" : "Preview SQL"} @@ -117,26 +117,28 @@ export default function AutoMetricCard({ color="link" onClick={async () => handleSqlPreview(metric.countSqlQuery)} > - {showSqlPreview && showSqlPreview === metric.countSqlQuery + {sqlPreview && sqlPreview === metric.countSqlQuery ? "Hide SQL" : "Preview SQL"}
- {showSqlPreview && ( + {sqlPreview && ( )} - +
); } diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 8b5d092cad1..67561d72e02 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -58,8 +58,6 @@ const NewDataSourceForm: FC<{ mutateDefinitions, } = useDefinitions(); const [step, setStep] = useState(0); - // const [showSqlPreview, setShowSqlPreview] = - // useState < { name: string, showPreview: boolean } >[]([]); const [schema, setSchema] = useState(""); const [dataSourceId, setDataSourceId] = useState( data?.id || null @@ -621,7 +619,7 @@ const NewDataSourceForm: FC<{ }[]; message?: string; }>(`/datasource/${dataSourceId}/auto-metrics`, { - method: "GET", + method: "POST", }); if (res.message) { track("Generate Auto Metrics Error", { From 9687da0797a45291bcce04f410eefd480fdfa067 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 15:09:05 -0400 Subject: [PATCH 25/39] Adds back in an optional type within the Integration.ts type file, and updates the individual SqlIntegration classes. --- .../back-end/src/controllers/datasources.ts | 3 +- .../back-end/src/integrations/ClickHouse.ts | 6 ++- .../back-end/src/integrations/Postgres.ts | 6 ++- packages/back-end/src/integrations/Presto.ts | 5 +- .../back-end/src/integrations/Redshift.ts | 24 +++++----- .../back-end/src/integrations/Snowflake.ts | 17 ++++--- .../src/integrations/SqlIntegration.ts | 33 ++++++++----- packages/back-end/src/types/Integration.ts | 4 +- .../components/Settings/AutoMetricCard.tsx | 48 +++---------------- .../components/Settings/NewDataSourceForm.tsx | 31 ++---------- 10 files changed, 67 insertions(+), 110 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index c6665b8d8cc..a3dff14efd0 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ import { Response } from "express"; import uniqid from "uniqid"; import { @@ -645,6 +644,7 @@ export async function postAutoGeneratedMetricsToCreate( const integration = getSourceIntegrationObject(dataSourceObj); if ( + !integration.getEventsTrackedByDatasource || !integration.settings.schemaFormat || !integration.getSourceProperties().supportsAutoGeneratedMetrics ) { @@ -656,7 +656,6 @@ export async function postAutoGeneratedMetricsToCreate( } try { - // Get the list of events tracked by this datasource - we can create metrics from these events. const results: AutoGeneratedMetric[] = await integration.getEventsTrackedByDatasource( integration.settings.schemaFormat ); diff --git a/packages/back-end/src/integrations/ClickHouse.ts b/packages/back-end/src/integrations/ClickHouse.ts index c8754351fb8..2427370e0ab 100644 --- a/packages/back-end/src/integrations/ClickHouse.ts +++ b/packages/back-end/src/integrations/ClickHouse.ts @@ -91,12 +91,14 @@ export default class ClickHouse extends SqlIntegration { ); return `table_schema IN ('${this.params.database}')`; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + generateTableName(tableName?: string): string { if (!this.params.database) { throw new MissingDatasourceParamsError( "No database provided. To automatically generate metrics, you must provide a database." ); } - return `${this.params.database}.${trackedEventTableName}`; + return tableName + ? `${this.params.database}.${tableName}` + : "information_schema.columns"; } } diff --git a/packages/back-end/src/integrations/Postgres.ts b/packages/back-end/src/integrations/Postgres.ts index 5b8f5ed2c4c..781d985c43c 100644 --- a/packages/back-end/src/integrations/Postgres.ts +++ b/packages/back-end/src/integrations/Postgres.ts @@ -38,11 +38,13 @@ export default class Postgres extends SqlIntegration { getInformationSchemaWhereClause(): string { return "table_schema NOT IN ('pg_catalog', 'information_schema', 'pg_toast')"; } - getAutoGeneratedMetricFromClause(trackedEventTableName: string): string { + generateTableName(tableName?: string): string { if (!this.params.defaultSchema) throw new Error( "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." ); - return `${this.params.defaultSchema}.${trackedEventTableName}`; + return tableName + ? `${this.params.defaultSchema}.${tableName}` + : "information_schema.columns"; } } diff --git a/packages/back-end/src/integrations/Presto.ts b/packages/back-end/src/integrations/Presto.ts index 2a931d2f508..0dd3cef55db 100644 --- a/packages/back-end/src/integrations/Presto.ts +++ b/packages/back-end/src/integrations/Presto.ts @@ -109,12 +109,13 @@ export default class Presto extends SqlIntegration { getInformationSchemaTableFromClause(databaseName: string): string { return `${databaseName}.information_schema.columns`; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars generateTableName(tableName?: string) { if (!this.params.catalog) throw new MissingDatasourceParamsError( "To view the information schema for a Presto data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings." ); - return `${this.params.catalog}.information_schema.columns`; + return tableName + ? `${this.params.catalog}.${tableName}` + : `${this.params.catalog}.information_schema.columns`; } } diff --git a/packages/back-end/src/integrations/Redshift.ts b/packages/back-end/src/integrations/Redshift.ts index 1f52d9ea4f0..195f05b962c 100644 --- a/packages/back-end/src/integrations/Redshift.ts +++ b/packages/back-end/src/integrations/Redshift.ts @@ -39,19 +39,17 @@ export default class Redshift extends SqlIntegration { return "SVV_COLUMNS"; } generateTableName(tableName?: string): string { - if (tableName) { - if (!this.params.database) { - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default database." - ); - } - if (!this.params.defaultSchema) - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default catalog." - ); - return `${this.params.database}.${this.params.defaultSchema}.${tableName}`; - } else { - return "SVV_COLUMNS"; + if (!tableName) return "SVV_COLUMNS"; + + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); } + if (!this.params.defaultSchema) + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default catalog." + ); + return `${this.params.database}.${this.params.defaultSchema}.${tableName}`; } } diff --git a/packages/back-end/src/integrations/Snowflake.ts b/packages/back-end/src/integrations/Snowflake.ts index 1944120fb7c..eb9680eff31 100644 --- a/packages/back-end/src/integrations/Snowflake.ts +++ b/packages/back-end/src/integrations/Snowflake.ts @@ -47,14 +47,13 @@ export default class Snowflake extends SqlIntegration { "No database provided. In order to get the information schema, you must provide a database." ); - if (tableName) { - if (!this.params.schema) - throw new MissingDatasourceParamsError( - "No schema provided. To automatically generate metrics, you must provide a schema." - ); - return `${this.params.database}.${this.params.schema}.${tableName}`; - } else { - return `${this.params.database}.information_schema.columns`; - } + if (!tableName) return `${this.params.database}.information_schema.columns`; + + if (!this.params.schema) + throw new MissingDatasourceParamsError( + "No schema provided. To automatically generate metrics, you must provide a schema." + ); + + return `${this.params.database}.${this.params.schema}.${tableName}`; } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 9a0c4c9879b..8c7b12eb2da 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -83,7 +83,7 @@ export default abstract class SqlIntegration activationDimension: true, pastExperiments: true, supportsInformationSchema: true, - supportsAutoGeneratedMetrics: this.isAutoGeneratingMetricSupported(), + supportsAutoGeneratedMetrics: this.isAutoGeneratingMetricsSupported(), }; } @@ -92,7 +92,7 @@ export default abstract class SqlIntegration return true; } - isAutoGeneratingMetricSupported(): boolean { + isAutoGeneratingMetricsSupported(): boolean { const supportedEventTrackers: SchemaFormat[] = ["segment", "rudderstack"]; if ( @@ -1374,9 +1374,8 @@ export default abstract class SqlIntegration } = this.getSchemaFormatConfig(schemaFormat); const currentDateTime = new Date(); - //MKTODO: Re-think how to name & calculate this so if we change the calculation, it doesn't make the variable name incorrect - const ThirtyDaysAgo = new Date( - currentDateTime.valueOf() - 30 * 60 * 60 * 24 * 1000 + const SevenDaysAgo = new Date( + currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 ); const sql = ` @@ -1390,10 +1389,10 @@ export default abstract class SqlIntegration ${this.generateTableName(trackedEventTableName)} WHERE received_at < '${currentDateTime .toISOString() - .slice( - 0, - 10 - )}' AND received_at > '${ThirtyDaysAgo.toISOString().slice(0, 10)}' + .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( + 0, + 10 + )}' AND event NOT IN ('experiment_viewed', 'experiment_started') GROUP BY event, ${timestampColumn}, displayName`; @@ -1408,13 +1407,25 @@ export default abstract class SqlIntegration COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt FROM - ${this.generateTableName("pages")}`; + ${this.generateTableName("pages")} + WHERE received_at < '${currentDateTime + .toISOString() + .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( + 0, + 10 + )}'`; const pageViewedResults = await this.runQuery( format(pageViewedSql, this.getFormatDialect()) ); - pageViewedResults.forEach((result) => results.push(result)); + console.log("pageViewedResults", pageViewedResults); + + pageViewedResults.forEach((result) => { + if (result.count > 0) { + results.push(result); + } + }); } if (!results) { diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index 56f24168c84..8a79421f4cd 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -306,10 +306,10 @@ export interface SourceIntegrationInterface { query: string ): Promise; runPastExperimentQuery(query: string): Promise; - getEventsTrackedByDatasource: ( + getEventsTrackedByDatasource?: ( schemaFormat: SchemaFormat ) => Promise; - getAutoGeneratedMetricSqlQuery( + getAutoGeneratedMetricSqlQuery?( metric: AutoGeneratedMetric, schemaFormat: SchemaFormat, type: MetricType diff --git a/packages/front-end/components/Settings/AutoMetricCard.tsx b/packages/front-end/components/Settings/AutoMetricCard.tsx index 0a62c379e6c..9efbc07bea6 100644 --- a/packages/front-end/components/Settings/AutoMetricCard.tsx +++ b/packages/front-end/components/Settings/AutoMetricCard.tsx @@ -1,50 +1,16 @@ import { ago } from "@/../shared/dates"; import { cloneDeep } from "lodash"; import { useState } from "react"; +import { AutoGeneratedMetric } from "@/../back-end/src/types/Integration"; import Tooltip from "../Tooltip/Tooltip"; import Toggle from "../Forms/Toggle"; import Button from "../Button"; import SQLInputField from "../SQLInputField"; type Props = { - metric: { - event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; - displayName: string; - lastTrackedAt: Date; - count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; - }; - setMetricsToCreate: ( - metrics: { - event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; - displayName: string; - lastTrackedAt: Date; - count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; - }[] - ) => void; - metricsToCreate: { - event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; - displayName: string; - lastTrackedAt: Date; - count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; - }[]; + metric: AutoGeneratedMetric; + setMetricsToCreate: (metrics: AutoGeneratedMetric[]) => void; + metricsToCreate: AutoGeneratedMetric[]; dataSourceId: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any form: any; @@ -70,8 +36,8 @@ export default function AutoMetricCard({ }; return ( -
- + <> + {metric.displayName} )} -
+ ); } diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 67561d72e02..7cb779c58a1 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -10,6 +10,7 @@ import { DataSourceSettings, } from "back-end/types/datasource"; import { useForm } from "react-hook-form"; +import { AutoGeneratedMetric } from "back-end/src/types/Integration"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -66,20 +67,9 @@ const NewDataSourceForm: FC<{ const [possibleTypes, setPossibleTypes] = useState( dataSourceConnections.map((d) => d.type) ); - const [metricsToCreate, setMetricsToCreate] = useState< - { - event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; - displayName: string; - lastTrackedAt: Date; - count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; - }[] - >([]); + const [metricsToCreate, setMetricsToCreate] = useState( + [] + ); const permissions = usePermissions(); @@ -605,18 +595,7 @@ const NewDataSourceForm: FC<{ newDatasourceForm: true, }); const res = await apiCall<{ - results: { - event: string; - displayName: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; - lastTrackedAt: Date; - count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; - }[]; + results: AutoGeneratedMetric[]; message?: string; }>(`/datasource/${dataSourceId}/auto-metrics`, { method: "POST", From b7342819d99c9504ff5fa25884ff4cc71888e955 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Thu, 1 Jun 2023 15:32:25 -0400 Subject: [PATCH 26/39] Removes console log --- packages/back-end/src/integrations/SqlIntegration.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 8c7b12eb2da..59578a26d9d 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1419,8 +1419,6 @@ export default abstract class SqlIntegration format(pageViewedSql, this.getFormatDialect()) ); - console.log("pageViewedResults", pageViewedResults); - pageViewedResults.forEach((result) => { if (result.count > 0) { results.push(result); From 351cb7da7060d496418f210989966234b4625c23 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 2 Jun 2023 07:02:25 -0400 Subject: [PATCH 27/39] Adds support for screens tables for Segment & Rudderstack --- .../src/integrations/SqlIntegration.ts | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 59578a26d9d..ce0efdf610a 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1310,15 +1310,8 @@ export default abstract class SqlIntegration getSchemaFormatConfig(schemaFormat: SchemaFormat): SchemaFormatConfig { switch (schemaFormat) { - case "amplitude": - return { - trackedEventTableName: "tracks", - eventColumn: "event", - timestampColumn: "server_upload_time", - userIdColumn: "user_id", - anonymousIdColumn: "amplitude_id", - }; - case "segment": { + // Segment & Rudderstack + default: { return { trackedEventTableName: "tracks", eventColumn: "event", @@ -1326,17 +1319,10 @@ export default abstract class SqlIntegration userIdColumn: "user_id", anonymousIdColumn: "anonymous_id", displayNameColumn: "event_text", + includesPagesTable: true, + includesScreensTable: true, }; } - // Rudderstack - default: - return { - trackedEventTableName: "tracks", - eventColumn: "event", - timestampColumn: "received_at", - userIdColumn: "user_id", - anonymousIdColumn: "anonymous_id", - }; } } @@ -1371,11 +1357,13 @@ export default abstract class SqlIntegration eventColumn, timestampColumn, displayNameColumn, + includesPagesTable, + includesScreensTable, } = this.getSchemaFormatConfig(schemaFormat); const currentDateTime = new Date(); const SevenDaysAgo = new Date( - currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 + currentDateTime.valueOf() - 35 * 60 * 60 * 24 * 1000 ); const sql = ` @@ -1398,7 +1386,7 @@ export default abstract class SqlIntegration const results = await this.runQuery(format(sql, this.getFormatDialect())); - if (schemaFormat === "segment") { + if (includesPagesTable) { const pageViewedSql = ` SELECT 'pages' as event, @@ -1426,6 +1414,34 @@ export default abstract class SqlIntegration }); } + if (includesScreensTable) { + const screenViewedSql = ` + SELECT + 'screens' as event, + "Screen Viewed" as displayName, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName("pages")} + WHERE received_at < '${currentDateTime + .toISOString() + .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( + 0, + 10 + )}'`; + + const screenViewedResults = await this.runQuery( + format(screenViewedSql, this.getFormatDialect()) + ); + + screenViewedResults.forEach((result) => { + if (result.count > 0) { + results.push(result); + } + }); + } + if (!results) { throw new Error(`No events found.`); } From 1e565e9f54c64f8145753b87ad36ee69cbf5884d Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 2 Jun 2023 08:48:48 -0400 Subject: [PATCH 28/39] Cleans up the logic for the various data warehouses supported by Segment & Rudderstack and improves error handling. --- .../back-end/src/controllers/datasources.ts | 4 +- packages/back-end/src/integrations/Athena.ts | 18 ++--- packages/back-end/src/integrations/Mssql.ts | 15 ++-- .../back-end/src/integrations/Postgres.ts | 7 +- .../src/integrations/SqlIntegration.ts | 78 +++++++++---------- 5 files changed, 62 insertions(+), 60 deletions(-) diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index a3dff14efd0..cba41566a05 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -677,7 +677,9 @@ export async function postAutoGeneratedMetricsToCreate( res.status(400).json({ status: 400, results: [], - message: e.message, + message: + "We were unable to identify any metrics to generate for you automatically: " + + e.message, }); return; } diff --git a/packages/back-end/src/integrations/Athena.ts b/packages/back-end/src/integrations/Athena.ts index 1cf43e22946..f14b6205a5a 100644 --- a/packages/back-end/src/integrations/Athena.ts +++ b/packages/back-end/src/integrations/Athena.ts @@ -57,16 +57,16 @@ export default class Athena extends SqlIntegration { throw new MissingDatasourceParamsError( "To view the information schema for an Athena data source, you must define a default catalog. Please add a default catalog by editing the datasource's connection settings." ); - if (tableName) { - // MKTODO: Test if the database is necessary here - if (!this.params.database) { - throw new MissingDatasourceParamsError( - "To automatically generate metrics for an Athena data source, you must define a default database." - ); - } - return `${this.params.database}.${this.params.catalog}.${tableName}`; - } else { + + if (!tableName) { return `${this.params.catalog}.information_schema.columns`; } + + if (!this.params.database) { + throw new MissingDatasourceParamsError( + "To automatically generate metrics for an Athena data source, you must define a default database." + ); + } + return `${this.params.database}.${this.params.catalog}.${tableName}`; } } diff --git a/packages/back-end/src/integrations/Mssql.ts b/packages/back-end/src/integrations/Mssql.ts index 9112ae7301b..125e7398fd3 100644 --- a/packages/back-end/src/integrations/Mssql.ts +++ b/packages/back-end/src/integrations/Mssql.ts @@ -75,15 +75,14 @@ export default class Mssql extends SqlIntegration { throw new MissingDatasourceParamsError( "To view the information schema for a MS Sql dataset, you must define a default database. Please add a default database by editing the datasource's connection settings." ); - if (tableName) { - //MK TODO: Validate if the defaultSchema is necessary here - if (!this.params.defaultSchema) - throw new Error( - "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." - ); - return `${this.params.defaultSchema}.${tableName}`; - } else { + + if (!tableName) { return `${this.params.database}.information_schema.columns`; } + if (!this.params.defaultSchema) + throw new Error( + "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." + ); + return `${this.params.defaultSchema}.${tableName}`; } } diff --git a/packages/back-end/src/integrations/Postgres.ts b/packages/back-end/src/integrations/Postgres.ts index 781d985c43c..8a761bd5058 100644 --- a/packages/back-end/src/integrations/Postgres.ts +++ b/packages/back-end/src/integrations/Postgres.ts @@ -39,12 +39,13 @@ export default class Postgres extends SqlIntegration { return "table_schema NOT IN ('pg_catalog', 'information_schema', 'pg_toast')"; } generateTableName(tableName?: string): string { + if (!tableName) { + return "information_schema.columns"; + } if (!this.params.defaultSchema) throw new Error( "No default schema provided. To automatically generate metrics, you must provide a default schema. This should be the schema where your tracked events are stored." ); - return tableName - ? `${this.params.defaultSchema}.${tableName}` - : "information_schema.columns"; + return `${this.params.defaultSchema}.${tableName}`; } } diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index ce0efdf610a..363b4d02781 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -1363,42 +1363,42 @@ export default abstract class SqlIntegration const currentDateTime = new Date(); const SevenDaysAgo = new Date( - currentDateTime.valueOf() - 35 * 60 * 60 * 24 * 1000 + currentDateTime.valueOf() - 7 * 60 * 60 * 24 * 1000 ); const sql = ` - SELECT - ${eventColumn} as event, - ${displayNameColumn ? displayNameColumn : eventColumn} as displayName, - (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, - COUNT (*) as count, - MAX(${timestampColumn}) as lastTrackedAt - FROM - ${this.generateTableName(trackedEventTableName)} - WHERE received_at < '${currentDateTime - .toISOString() - .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( - 0, - 10 - )}' - AND event NOT IN ('experiment_viewed', 'experiment_started') - GROUP BY event, ${timestampColumn}, displayName`; + SELECT + ${eventColumn} as event, + ${displayNameColumn ? displayNameColumn : eventColumn} as displayName, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName(trackedEventTableName)} + WHERE received_at < '${currentDateTime + .toISOString() + .slice( + 0, + 10 + )}' AND received_at > '${SevenDaysAgo.toISOString().slice(0, 10)}' + AND event NOT IN ('experiment_viewed', 'experiment_started') + GROUP BY event, ${timestampColumn}, displayName`; const results = await this.runQuery(format(sql, this.getFormatDialect())); if (includesPagesTable) { const pageViewedSql = ` - SELECT - 'pages' as event, - "Page Viewed" as displayName, - (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, - COUNT (*) as count, - MAX(${timestampColumn}) as lastTrackedAt - FROM - ${this.generateTableName("pages")} - WHERE received_at < '${currentDateTime - .toISOString() - .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( + SELECT + 'pages' as event, + "Page Viewed" as displayName, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName("pages")} + WHERE received_at < '${currentDateTime + .toISOString() + .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( 0, 10 )}'`; @@ -1416,17 +1416,17 @@ export default abstract class SqlIntegration if (includesScreensTable) { const screenViewedSql = ` - SELECT - 'screens' as event, - "Screen Viewed" as displayName, - (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, - COUNT (*) as count, - MAX(${timestampColumn}) as lastTrackedAt - FROM - ${this.generateTableName("pages")} - WHERE received_at < '${currentDateTime - .toISOString() - .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( + SELECT + 'screens' as event, + "Screen Viewed" as displayName, + (CASE WHEN COUNT(user_id) > 0 THEN 1 ELSE 0 END) as hasUserId, + COUNT (*) as count, + MAX(${timestampColumn}) as lastTrackedAt + FROM + ${this.generateTableName("pages")} + WHERE received_at < '${currentDateTime + .toISOString() + .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( 0, 10 )}'`; From aea2c57de284bdf4e1c01a2fb293515a35ec127e Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Fri, 2 Jun 2023 08:57:42 -0400 Subject: [PATCH 29/39] Adds types for includesScreensTable and includesPagesTable that were not commited in last push --- packages/back-end/types/datasource.d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/back-end/types/datasource.d.ts b/packages/back-end/types/datasource.d.ts index 75861298fca..b2da526ffcb 100644 --- a/packages/back-end/types/datasource.d.ts +++ b/packages/back-end/types/datasource.d.ts @@ -88,6 +88,8 @@ export interface SchemaFormatConfig { eventColumn: string; timestampColumn: string; displayNameColumn?: string; + includesPagesTable: boolean; + includesScreensTable: boolean; } export interface DataSourceProperties { From 09d426298058f89a48cfbaa91f24f959161011a8 Mon Sep 17 00:00:00 2001 From: Michael Knowlton Date: Tue, 6 Jun 2023 11:31:18 -0400 Subject: [PATCH 30/39] Addresses almost all of Jeremy's feedback - updates the data structure we were using, removes the pluralization logic, and added a few todos for future iterations. --- .../back-end/src/controllers/datasources.ts | 17 ++- .../src/integrations/SqlIntegration.ts | 125 ++++++++++++------ .../src/jobs/createAutoGeneratedMetrics.ts | 63 ++++----- packages/back-end/src/models/MetricModel.ts | 14 +- packages/back-end/src/types/Integration.ts | 22 +-- .../back-end/src/util/autoGeneratedMetrics.ts | 53 -------- packages/back-end/types/datasource.d.ts | 2 + .../components/Settings/AutoMetricCard.tsx | 54 ++++---- .../components/Settings/NewDataSourceForm.tsx | 99 +++++++++----- 9 files changed, 250 insertions(+), 199 deletions(-) delete mode 100644 packages/back-end/src/util/autoGeneratedMetrics.ts diff --git a/packages/back-end/src/controllers/datasources.ts b/packages/back-end/src/controllers/datasources.ts index b5795eb4219..39f701c135b 100644 --- a/packages/back-end/src/controllers/datasources.ts +++ b/packages/back-end/src/controllers/datasources.ts @@ -43,7 +43,8 @@ import { EventAuditUserForResponseLocals } from "../events/event-types"; import { deleteInformationSchemaById } from "../models/InformationSchemaModel"; import { deleteInformationSchemaTablesByInformationSchemaId } from "../models/InformationSchemaTablesModel"; import { queueCreateAutoGeneratedMetrics } from "../jobs/createAutoGeneratedMetrics"; -import { AutoGeneratedMetric } from "../types/Integration"; +import { TrackedEventData } from "../types/Integration"; +import { MetricType } from "../../types/metric"; export async function postSampleData( req: AuthRequest, @@ -422,7 +423,7 @@ export async function putDataSource( params: DataSourceParams; settings: DataSourceSettings; projects?: string[]; - metricsToCreate?: AutoGeneratedMetric[]; + metricsToCreate?: { name: string; type: MetricType; sql: string }[]; }, { id: string } >, @@ -466,6 +467,8 @@ export async function putDataSource( return; } + console.log("metricsToCreate in the controller", metricsToCreate); + if (metricsToCreate?.length) { await queueCreateAutoGeneratedMetrics( datasource.id, @@ -658,14 +661,14 @@ export async function postAutoGeneratedMetricsToCreate( } try { - const results: AutoGeneratedMetric[] = await integration.getEventsTrackedByDatasource( + const trackedEvents: TrackedEventData[] = await integration.getEventsTrackedByDatasource( integration.settings.schemaFormat ); - if (!results.length) { + if (!trackedEvents.length) { return res.status(200).json({ status: 200, - results, + trackedEvents, message: "We were unable to identify any metrics to generate automatically for you.", }); @@ -673,12 +676,12 @@ export async function postAutoGeneratedMetricsToCreate( return res.status(200).json({ status: 200, - results, + trackedEvents, }); } catch (e) { res.status(400).json({ status: 400, - results: [], + trackedEvents: [], message: "We were unable to identify any metrics to generate for you automatically: " + e.message, diff --git a/packages/back-end/src/integrations/SqlIntegration.ts b/packages/back-end/src/integrations/SqlIntegration.ts index 655a65506b9..48f9b2b52c4 100644 --- a/packages/back-end/src/integrations/SqlIntegration.ts +++ b/packages/back-end/src/integrations/SqlIntegration.ts @@ -24,7 +24,6 @@ import { MetricAggregationType, InformationSchema, RawInformationSchema, - AutoGeneratedMetric, } from "../types/Integration"; import { DimensionInterface } from "../../types/dimension"; import { @@ -39,7 +38,6 @@ import { FormatDialect, } from "../util/sql"; import { formatInformationSchema } from "../util/informationSchemas"; -import { getPluralizedMetricName } from "../util/autoGeneratedMetrics"; import { ExperimentSnapshotSettings } from "../../types/experiment-snapshot"; export default abstract class SqlIntegration @@ -1290,7 +1288,8 @@ export default abstract class SqlIntegration } getAutoGeneratedMetricSqlQuery( - metric: AutoGeneratedMetric, + event: string, + hasUserId: boolean, schemaFormat: SchemaFormat, type: MetricType ): string { @@ -1303,18 +1302,73 @@ export default abstract class SqlIntegration const sqlQuery = ` SELECT ${ - metric.hasUserId ? `${userIdColumn}, ` : "" + hasUserId ? `${userIdColumn}, ` : "" }${anonymousIdColumn} as anonymous_id, ${timestampColumn} as timestamp - ${type === "count" && metric.createCountFromEvent ? `, 1 as value` : ""} - FROM ${this.generateTableName(metric.event)} + ${type === "count" ? `, 1 as value` : ""} + FROM ${this.generateTableName(event)} `; return format(sqlQuery, this.getFormatDialect()); } + getMetricsToCreate( + result: { + event: string; + displayName: string; + hasUserId: boolean; + count: number; + lastTrackedAt: Date; + }, + schemaFormat: SchemaFormat + ): { + name: string; + type: MetricType; + sql: string; + }[] { + const metricsToCreate: { + name: string; + type: MetricType; + sql: string; + }[] = []; + + //TODO Build some logic where based on the event, we determine what metrics to create (by default, we create binomial and count) for every event + metricsToCreate.push({ + name: result.displayName, + type: "binomial", + sql: this.getAutoGeneratedMetricSqlQuery( + result.event, + result.hasUserId, + schemaFormat, + "binomial" + ), + }); + + metricsToCreate.push({ + name: `Count of ${result.displayName}`, + type: "count", + sql: this.getAutoGeneratedMetricSqlQuery( + result.event, + result.hasUserId, + schemaFormat, + "count" + ), + }); + + return metricsToCreate; + } + async getEventsTrackedByDatasource( schemaFormat: SchemaFormat - ): Promise { + ): Promise< + { + event: string; + displayName: string; + count: number; + hasUserId: boolean; + lastTrackedAt: Date; + metricsToCreate: { name: string; sql: string; type: MetricType }[]; + }[] + > { const { trackedEventTableName, eventColumn, @@ -1366,15 +1420,19 @@ export default abstract class SqlIntegration 10 )}'`; - const pageViewedResults = await this.runQuery( - format(pageViewedSql, this.getFormatDialect()) - ); + try { + const pageViewedResults = await this.runQuery( + format(pageViewedSql, this.getFormatDialect()) + ); - pageViewedResults.forEach((result) => { - if (result.count > 0) { - results.push(result); - } - }); + pageViewedResults.forEach((result) => { + if (result.count > 0) { + results.push(result); + } + }); + } catch (e) { + // This happens when the table doesn't exists - this is optional, so just ignoring + } } if (includesScreensTable) { @@ -1386,7 +1444,7 @@ export default abstract class SqlIntegration COUNT (*) as count, MAX(${timestampColumn}) as lastTrackedAt FROM - ${this.generateTableName("pages")} + ${this.generateTableName("screens")} WHERE received_at < '${currentDateTime .toISOString() .slice(0, 10)}' AND received_at > '${SevenDaysAgo.toISOString().slice( @@ -1394,15 +1452,19 @@ export default abstract class SqlIntegration 10 )}'`; - const screenViewedResults = await this.runQuery( - format(screenViewedSql, this.getFormatDialect()) - ); + try { + const screenViewedResults = await this.runQuery( + format(screenViewedSql, this.getFormatDialect()) + ); - screenViewedResults.forEach((result) => { - if (result.count > 0) { - results.push(result); - } - }); + screenViewedResults.forEach((result) => { + if (result.count > 0) { + results.push(result); + } + }); + } catch (e) { + // This happens when the table doesn't exists - this is optional, so just ignoring + } } if (!results) { @@ -1410,22 +1472,11 @@ export default abstract class SqlIntegration } return results.map((result) => { - result.createBinomialFromEvent = true; - result.createCountFromEvent = true; + // Normalize the lastTrackedAt field - BigQuery stores it as an object result.lastTrackedAt = result.lastTrackedAt.value ? new Date(result.lastTrackedAt.value) : new Date(result.lastTrackedAt); - result.binomialSqlQuery = this.getAutoGeneratedMetricSqlQuery( - result, - schemaFormat, - "binomial" - ); - result.countSqlQuery = this.getAutoGeneratedMetricSqlQuery( - result, - schemaFormat, - "count" - ); - result.countDisplayName = getPluralizedMetricName(result.displayName); + result.metricsToCreate = this.getMetricsToCreate(result, schemaFormat); return result; }); } diff --git a/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts b/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts index 9024843c9d0..f204098f287 100644 --- a/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts +++ b/packages/back-end/src/jobs/createAutoGeneratedMetrics.ts @@ -1,7 +1,6 @@ /* eslint-disable no-console */ import Agenda, { Job } from "agenda"; import uniqid from "uniqid"; -import { cloneDeep } from "lodash"; import { getDataSourceById } from "../models/DataSourceModel"; import { insertMetrics } from "../models/MetricModel"; import { MetricInterface } from "../../types/metric"; @@ -14,7 +13,17 @@ const CREATE_AUTOGENERATED_METRICS_JOB_NAME = "createAutoGeneratedMetrics"; type CreateAutoGeneratedMetricsJob = Job<{ organization: string; datasourceId: string; - metricsToCreate: AutoGeneratedMetric[]; + metricsToCreate: Pick< + MetricInterface, + | "name" + | "type" + | "sql" + | "id" + | "organization" + | "datasource" + | "dateCreated" + | "dateUpdated" + >[]; }>; let agenda: Agenda; @@ -45,35 +54,27 @@ export default function (ag: Agenda) { "Auto generated metrics not supported for this data source" ); - const metrics: Partial[] = []; - for (const metric of metricsToCreate) { - const baseMetric: Partial = { - organization, - datasource: datasourceId, - name: metric.displayName, - dateCreated: new Date(), - dateUpdated: new Date(), - }; - if (metric.createBinomialFromEvent) { - const metricToCreate: Partial = cloneDeep( - baseMetric - ); - metricToCreate.id = uniqid("met_"); - metricToCreate.type = "binomial"; - metricToCreate.sql = metric.binomialSqlQuery; - metrics.push(metricToCreate); - } - if (metric.createCountFromEvent) { - const metricToCreate: Partial = cloneDeep( - baseMetric - ); - metricToCreate.id = uniqid("met_"); - metricToCreate.type = "count"; - metricToCreate.name = metric.countDisplayName; - metricToCreate.sql = metric.countSqlQuery; - metrics.push(metricToCreate); - } - } + const metrics: Pick< + MetricInterface, + | "name" + | "type" + | "sql" + | "id" + | "organization" + | "datasource" + | "dateCreated" + | "dateUpdated" + >[] = []; + + metricsToCreate.forEach((metric) => { + metric.id = uniqid("met_"); + metric.organization = organization; + metric.datasource = datasourceId; + metric.dateCreated = new Date(); + metric.dateUpdated = new Date(); + metrics.push(metric); + }); + await insertMetrics(metrics); } catch (e) { logger.error( diff --git a/packages/back-end/src/models/MetricModel.ts b/packages/back-end/src/models/MetricModel.ts index 6cd4a34ff69..96cc67f1c23 100644 --- a/packages/back-end/src/models/MetricModel.ts +++ b/packages/back-end/src/models/MetricModel.ts @@ -108,7 +108,19 @@ export async function insertMetric(metric: Partial) { return toInterface(await MetricModel.create(metric)); } -export async function insertMetrics(metrics: Partial[]) { +export async function insertMetrics( + metrics: Pick< + MetricInterface, + | "name" + | "type" + | "sql" + | "id" + | "organization" + | "datasource" + | "dateCreated" + | "dateUpdated" + >[] +) { if (usingFileConfig()) { throw new Error("Cannot add. Metrics managed by config.yml"); } diff --git a/packages/back-end/src/types/Integration.ts b/packages/back-end/src/types/Integration.ts index c4ff41f1f65..8bfeb60b1dc 100644 --- a/packages/back-end/src/types/Integration.ts +++ b/packages/back-end/src/types/Integration.ts @@ -127,17 +127,18 @@ export type PastExperimentResult = { }[]; }; -export type AutoGeneratedMetric = { +export type TrackedEventData = { event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; displayName: string; - lastTrackedAt: Date; count: number; - binomialSqlQuery: string; - countSqlQuery: string; - countDisplayName: string; + hasUserId: boolean; + lastTrackedAt: Date; + metricsToCreate: { + name: string; + sql: string; + type: MetricType; + shouldCreate?: boolean; + }[]; }; export type MetricValueQueryResponseRow = { @@ -302,9 +303,10 @@ export interface SourceIntegrationInterface { runPastExperimentQuery(query: string): Promise; getEventsTrackedByDatasource?: ( schemaFormat: SchemaFormat - ) => Promise; + ) => Promise; getAutoGeneratedMetricSqlQuery?( - metric: AutoGeneratedMetric, + event: string, + hasUserId: boolean, schemaFormat: SchemaFormat, type: MetricType ): string; diff --git a/packages/back-end/src/util/autoGeneratedMetrics.ts b/packages/back-end/src/util/autoGeneratedMetrics.ts deleted file mode 100644 index 17d18f81998..00000000000 --- a/packages/back-end/src/util/autoGeneratedMetrics.ts +++ /dev/null @@ -1,53 +0,0 @@ -const eventsArray = [ - { name: "Signed Up", plural: "Sign Ups" }, - { name: "Placed Order", plural: "Orders Placed" }, - { name: "Subscribed", plural: "Subscriptions" }, - { name: "Page Viewed", plural: "Page Views" }, - { name: "Products Searched", plural: "Products Searches" }, - { name: "Product List Viewed", plural: "Count of Product List Views" }, - { name: "Product List Filtered", plural: "Count of Product List Filters" }, - { name: "Promotion Viewed", plural: "Promotion Views" }, - { name: "Promotion Clicked", plural: "Promotion Clicks" }, - { name: "Product Clicked", plural: "Product Clicks" }, - { name: "Product Viewed", plural: "Product Views" }, - { name: "Product Added", plural: "Product Additions" }, - { name: "Product Removed", plural: "Products Removals" }, - { name: "Cart Viewed", plural: "Carts Views" }, - { name: "Checkout Started", plural: "Checkouts Started" }, - { name: "Checkout Step Viewed", plural: "Checkout Step Views" }, - { name: "Checkout Step Completed", plural: "Checkout Step Completes" }, - { name: "Payment Info Entered", plural: "Count of Payment Info Entered" }, - { name: "Order Completed", plural: "Count of Completed Orders" }, - { name: "Order Updated", plural: "Count of Orders Updates" }, - { name: "Order Refunded", plural: "Count of Orders Refunds" }, - { name: "Order Cancelled", plural: "Count of Order Cancelletions" }, - { name: "Coupon Entered", plural: "Count of Coupons Entered" }, - { name: "Coupon Applied", plural: "Count of Coupons Applied" }, - { name: "Coupon Denied", plural: "Count of Coupons Denied" }, - { name: "Coupon Removed", plural: "Count of Coupons Removed" }, - { - name: "Product Added to Wishlist", - plural: "Count of Product Adds to Wishlist", - }, - { - name: "Product Removed from Wishlist", - plural: "Count of Product Removals from Wishlist", - }, - { - name: "Wishlist Product Added to Cart", - plural: "Count of Wishlist Product Added to Cart", - }, - { name: "Product Shared", plural: "Product Shares" }, - { name: "Cart Shared", plural: "Cart Shares" }, - { name: "Product Reviewed", plural: "Product Reviews" }, -]; - -const events = new Map(); - -eventsArray.forEach((event) => { - events.set(event.name, event.plural); -}); - -export function getPluralizedMetricName(metricName: string): string { - return events.get(metricName) || `Count of ${metricName}`; -} diff --git a/packages/back-end/types/datasource.d.ts b/packages/back-end/types/datasource.d.ts index b2da526ffcb..7a297d94b9b 100644 --- a/packages/back-end/types/datasource.d.ts +++ b/packages/back-end/types/datasource.d.ts @@ -88,6 +88,8 @@ export interface SchemaFormatConfig { eventColumn: string; timestampColumn: string; displayNameColumn?: string; + //TODO: The includesPagesTable & includesScreensTable are only used for Segment & Rudderstack. Other event trackers might have their own unique properties, + // so we'll probably want to build a method that maps unique properties to event tracker type in the future. includesPagesTable: boolean; includesScreensTable: boolean; } diff --git a/packages/front-end/components/Settings/AutoMetricCard.tsx b/packages/front-end/components/Settings/AutoMetricCard.tsx index 9efbc07bea6..531443e9168 100644 --- a/packages/front-end/components/Settings/AutoMetricCard.tsx +++ b/packages/front-end/components/Settings/AutoMetricCard.tsx @@ -1,16 +1,16 @@ import { ago } from "@/../shared/dates"; import { cloneDeep } from "lodash"; import { useState } from "react"; -import { AutoGeneratedMetric } from "@/../back-end/src/types/Integration"; +import { TrackedEventData } from "@/../back-end/src/types/Integration"; import Tooltip from "../Tooltip/Tooltip"; import Toggle from "../Forms/Toggle"; import Button from "../Button"; import SQLInputField from "../SQLInputField"; type Props = { - metric: AutoGeneratedMetric; - setMetricsToCreate: (metrics: AutoGeneratedMetric[]) => void; - metricsToCreate: AutoGeneratedMetric[]; + event: TrackedEventData; + setTrackedEvents: (events: TrackedEventData[]) => void; + trackedEvents: TrackedEventData[]; dataSourceId: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any form: any; @@ -18,9 +18,9 @@ type Props = { }; export default function AutoMetricCard({ - metric, - setMetricsToCreate, - metricsToCreate, + event, + setTrackedEvents, + trackedEvents, dataSourceId, form, i, @@ -37,32 +37,34 @@ export default function AutoMetricCard({ return ( <> - - {metric.displayName} + + {event.displayName} - {metric.count} + {event.count}
{ - const newMetricsToCreate = cloneDeep(metricsToCreate); - newMetricsToCreate[i].createBinomialFromEvent = value; - setMetricsToCreate(newMetricsToCreate); + const updatedTrackedEvents = cloneDeep(trackedEvents); + updatedTrackedEvents[i].metricsToCreate[0].shouldCreate = value; + setTrackedEvents(updatedTrackedEvents); }} /> @@ -71,19 +73,21 @@ export default function AutoMetricCard({
{ - const newMetricsToCreate = cloneDeep(metricsToCreate); - newMetricsToCreate[i].createCountFromEvent = value; - setMetricsToCreate(newMetricsToCreate); + const updatedTrackedEvents = cloneDeep(trackedEvents); + updatedTrackedEvents[i].metricsToCreate[1].shouldCreate = value; + setTrackedEvents(updatedTrackedEvents); }} /> diff --git a/packages/front-end/components/Settings/NewDataSourceForm.tsx b/packages/front-end/components/Settings/NewDataSourceForm.tsx index 7cb779c58a1..879c69abab7 100644 --- a/packages/front-end/components/Settings/NewDataSourceForm.tsx +++ b/packages/front-end/components/Settings/NewDataSourceForm.tsx @@ -10,7 +10,9 @@ import { DataSourceSettings, } from "back-end/types/datasource"; import { useForm } from "react-hook-form"; -import { AutoGeneratedMetric } from "back-end/src/types/Integration"; +import cloneDeep from "lodash/cloneDeep"; +import { MetricType } from "@/../back-end/types/metric"; +import { TrackedEventData } from "@/../back-end/src/types/Integration"; import { useAuth } from "@/services/auth"; import track from "@/services/track"; import { getInitialSettings } from "@/services/datasources"; @@ -67,9 +69,7 @@ const NewDataSourceForm: FC<{ const [possibleTypes, setPossibleTypes] = useState( dataSourceConnections.map((d) => d.type) ); - const [metricsToCreate, setMetricsToCreate] = useState( - [] - ); + const [trackedEvents, setTrackedEvents] = useState([]); const permissions = usePermissions(); @@ -106,10 +106,9 @@ const NewDataSourceForm: FC<{ const form = useForm<{ settings: DataSourceSettings | undefined; metricsToCreate: { - event: string; - hasUserId: boolean; - createBinomialFromEvent: boolean; - createCountFromEvent: boolean; + name: string; + sql: string; + type: MetricType; }[]; }>({ defaultValues: { @@ -137,8 +136,24 @@ const NewDataSourceForm: FC<{ }, [source]); useEffect(() => { - form.setValue("metricsToCreate", metricsToCreate); - }, [form, metricsToCreate]); + const updatedMetricsToCreate: { + name: string; + sql: string; + type: MetricType; + }[] = []; + trackedEvents.forEach((event: TrackedEventData) => { + event.metricsToCreate.forEach((metric) => { + if (metric.shouldCreate) { + updatedMetricsToCreate.push({ + name: metric.name, + type: metric.type, + sql: metric.sql, + }); + } + }); + }); + form.setValue("metricsToCreate", updatedMetricsToCreate); + }, [form, trackedEvents]); const { apiCall } = useAuth(); @@ -234,7 +249,7 @@ const NewDataSourceForm: FC<{ const newVal = { ...datasource, settings, - metricsToCreate, + metricsToCreate: form.watch("metricsToCreate"), }; setDatasource(newVal as Partial); await apiCall<{ status: number; message: string }>( @@ -321,9 +336,9 @@ const NewDataSourceForm: FC<{ await updateSettings(); } if (isFinalStep) { - if (metricsToCreate.length > 0) { + if (trackedEvents.length > 0) { track("Generating Auto Metrics For User", { - metricsToCreate, + metricsToCreate: form.watch("metricsToCreate"), source, type: datasource.type, dataSourceId, @@ -572,7 +587,7 @@ const NewDataSourceForm: FC<{ New!

- {metricsToCreate.length === 0 ? ( + {trackedEvents.length === 0 ? (
{`With ${ @@ -595,7 +610,7 @@ const NewDataSourceForm: FC<{ newDatasourceForm: true, }); const res = await apiCall<{ - results: AutoGeneratedMetric[]; + trackedEvents: TrackedEventData[]; message?: string; }>(`/datasource/${dataSourceId}/auto-metrics`, { method: "POST", @@ -612,7 +627,15 @@ const NewDataSourceForm: FC<{ setAutoMetricError(res.message); return; } - setMetricsToCreate(res.results); + // Before we setMetricsToCreate, we need to add a "shouldCreate" boolean property to each metric + res.trackedEvents.forEach( + (event: TrackedEventData) => { + event.metricsToCreate.forEach((metric) => { + metric.shouldCreate = true; + }); + } + ); + setTrackedEvents(res.trackedEvents); } catch (e) { track("Generate Auto Metrics Error", { error: e.message, @@ -644,18 +667,21 @@ const NewDataSourceForm: FC<{ Click here to learn more about GrowthBook Metrics.

- {metricsToCreate.length > 0 && ( + {trackedEvents.length > 0 && ( <>
-
- - -
- { - const updatedTrackedEvents = cloneDeep(trackedEvents); - updatedTrackedEvents[i].metricsToCreate[1].shouldCreate = value; - setTrackedEvents(updatedTrackedEvents); - }} - /> - -
- +
+ { + const updatedTrackedEvents = cloneDeep(trackedEvents); + updatedTrackedEvents[i].metricsToCreate[ + j + ].shouldCreate = value; + setTrackedEvents(updatedTrackedEvents); + }} + /> + +
+ + ); + })} {sqlPreview && ( Date: Thu, 8 Jun 2023 15:32:30 -0400 Subject: [PATCH 39/39] refactors the AutoMetricCard to look for specific metric types to ensure the column order remains correct. --- .../components/Settings/AutoMetricCard.tsx | 96 +++++++++++++------ 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/packages/front-end/components/Settings/AutoMetricCard.tsx b/packages/front-end/components/Settings/AutoMetricCard.tsx index 494ba55e758..6eecc66630e 100644 --- a/packages/front-end/components/Settings/AutoMetricCard.tsx +++ b/packages/front-end/components/Settings/AutoMetricCard.tsx @@ -38,6 +38,14 @@ export default function AutoMetricCard({ const selected = sqlPreview && event.metricsToCreate.findIndex((s) => s.sql === sqlPreview); + const binmomialIndex = event.metricsToCreate.findIndex( + (metric) => metric.type === "binomial" + ); + + const countIndex = event.metricsToCreate.findIndex( + (metric) => metric.type === "count" + ); + return ( <> @@ -50,34 +58,66 @@ export default function AutoMetricCard({ {event.count} - {event.metricsToCreate.map((metric, j) => { - return ( - -
- { - const updatedTrackedEvents = cloneDeep(trackedEvents); - updatedTrackedEvents[i].metricsToCreate[ - j - ].shouldCreate = value; - setTrackedEvents(updatedTrackedEvents); - }} - /> - -
- - ); - })} + {event.metricsToCreate[binmomialIndex]?.sql ? ( + +
+ { + const updatedTrackedEvents = cloneDeep(trackedEvents); + updatedTrackedEvents[i].metricsToCreate[ + binmomialIndex + ].shouldCreate = value; + setTrackedEvents(updatedTrackedEvents); + }} + /> + +
+ + ) : ( + +
-
+ + )} + {event.metricsToCreate[countIndex]?.sql ? ( + +
+ { + const updatedTrackedEvents = cloneDeep(trackedEvents); + updatedTrackedEvents[i].metricsToCreate[ + countIndex + ].shouldCreate = value; + setTrackedEvents(updatedTrackedEvents); + }} + /> + +
+ + ) : ( + +
-
+ + )} {sqlPreview && (