internal/event_logs: optimize ping cte

internal/database/event_logs.go

@@ -1574,27 +1574,11 @@ func (l *eventLogStore) aggregatedSearchEvents(ctx context.Context, queryString
 	return events, nil
 }
 
-// List of events that don't meet the criteria of "active" usage of Cody.
-var nonActiveCodyEvents = []string{
-	"CodyVSCodeExtension:CodySavedLogin:executed",
-	"web:codyChat:tryOnPublicCode",
-	"web:codyEditorWidget:viewed",
-	"web:codyChat:pageViewed",
-	"CodyConfigurationPageViewed",
-	"ClickedOnTryCodySearchCTA",
-	"TryCodyWebOnboardingDisplayed",
-	"AboutGetCodyPopover",
-	"TryCodyWeb",
-	"CodySurveyToastViewed",
-	"SiteAdminCodyPageViewed",
-	"CodyUninstalled",
-	"SpeakToACodyEngineerCTA",
-}
-
 var aggregatedCodyUsageEventsQuery = `
-WITH events AS (
+WITH
+events AS (
   SELECT
-    name AS key,
+    name,
     ` + aggregatedUserIDQueryFragment + ` AS user_id,
     ` + makeDateTruncExpression("month", "timestamp") + ` as month,
     ` + makeDateTruncExpression("week", "timestamp") + ` as week,
@@ -1604,36 +1588,11 @@ WITH events AS (
     ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day
   FROM event_logs
   WHERE
-    timestamp >= ` + makeDateTruncExpression("month", "%s::timestamp") + `
-    AND lower(name) like '%%cody%%'
-    AND name not like '%%CTA%%'
-    AND name not like '%%Cta%%'
-    AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `'))
-),
-code_generation_keys AS (
-  SELECT * FROM unnest(ARRAY[
-    'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',
-    'CodyVSCodeExtension:recipe:improve-variable-names:executed',
-    'CodyVSCodeExtension:recipe:replace:executed',
-    'CodyVSCodeExtension:recipe:generate-docstring:executed',
-    'CodyVSCodeExtension:recipe:generate-unit-test:executed',
-    'CodyVSCodeExtension:recipe:rewrite-functional:executed',
-    'CodyVSCodeExtension:recipe:code-refactor:executed',
-    'CodyVSCodeExtension:recipe:fixup:executed',
-	'CodyVSCodeExtension:recipe:translate-to-language:executed'
-  ]) AS key
-),
-explanation_keys AS (
-  SELECT * FROM unnest(ARRAY[
-    'CodyVSCodeExtension:recipe:explain-code-high-level:executed',
-    'CodyVSCodeExtension:recipe:explain-code-detailed:executed',
-    'CodyVSCodeExtension:recipe:find-code-smells:executed',
-    'CodyVSCodeExtension:recipe:git-history:executed',
-    'CodyVSCodeExtension:recipe:rate-code:executed'
-  ]) AS key
+  	timestamp >= %s::timestamp - '1 month'::interval
+    AND iscodyactiveevent(name)
 )
 SELECT
-  key,
+  name,
   current_month,
   current_week,
   current_day,
@@ -1643,25 +1602,23 @@ SELECT
   COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month,
   COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week,
   COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day,
-  SUM(case when month = current_month and key in
-  	(SELECT * FROM code_generation_keys)
+  SUM(case when month = current_month and iscodygenerationevent(name)
   	then 1 else 0 end) as code_generation_month,
-  SUM(case when week = current_week and key in
-  	(SELECT * FROM explanation_keys)
+  SUM(case when week = current_week and iscodygenerationevent(name)
 	then 1 else 0 end) as code_generation_week,
-  SUM(case when day = current_day and key in (SELECT * FROM code_generation_keys)
+  SUM(case when day = current_day and iscodygenerationevent(name)
 	then 1 else 0 end) as code_generation_day,
-  SUM(case when month = current_month and key in (SELECT * FROM explanation_keys)
+  SUM(case when month = current_month and iscodyexplanationevent(name)
 	then 1 else 0 end) as explanation_month,
-  SUM(case when week = current_week and key in (SELECT * FROM explanation_keys)
+  SUM(case when week = current_week and iscodyexplanationevent(name)
 	then 1 else 0 end) as explanation_week,
-  SUM(case when day = current_day and key in (SELECT * FROM explanation_keys)
+  SUM(case when day = current_day and iscodyexplanationevent(name)
 	then 1 else 0 end) as explanation_day,
 	0 as invalid_month,
 	0 as invalid_week,
 	0 as invalid_day
 FROM events
-GROUP BY key, current_month, current_week, current_day
+GROUP BY name, current_month, current_week, current_day
 `
 
 var searchLatencyEventNames = []string{

internal/database/event_logs_test.go

@@ -1317,7 +1317,7 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) {
 
 	codyEventNames := []string{"CodyVSCodeExtension:recipe:rewrite-to-functional:executed",
 		"CodyVSCodeExtension:recipe:explain-code-high-level:executed"}
-	users := []uint32{1, 2}
+	users := []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
 
 	days := []time.Time{
 		now,                          // Today
@@ -1332,7 +1332,7 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) {
 	for _, user := range users {
 		for _, name := range codyEventNames {
 			for _, day := range days {
-				for i := 0; i < 25; i++ {
+				for i := 0; i < 250; i++ {
 					e := &Event{
 						UserID: user,
 						Name:   name,
@@ -1361,35 +1361,46 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) {
 
 	expectedEvents := []types.CodyAggregatedEvent{
 		{
-			Name:               "CodyVSCodeExtension:recipe:explain-code-high-level:executed",
-			Month:              time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC),
-			Week:               now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5),
-			Day:                now.Truncate(time.Hour * 24),
-			TotalMonth:         200,
-			TotalWeek:          150,
-			TotalDay:           50,
-			UniquesMonth:       2,
-			UniquesWeek:        2,
-			UniquesDay:         2,
-			CodeGenerationWeek: 150,
-			CodeGenerationDay:  0,
-			ExplanationMonth:   200,
-			ExplanationWeek:    150,
-			ExplanationDay:     50,
+			Name:                "CodyVSCodeExtension:recipe:explain-code-high-level:executed",
+			Month:               time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC),
+			Week:                now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5),
+			Day:                 now.Truncate(time.Hour * 24),
+			TotalMonth:          20000,
+			TotalWeek:           15000,
+			TotalDay:            5000,
+			UniquesMonth:        20,
+			UniquesWeek:         20,
+			UniquesDay:          20,
+			CodeGenerationMonth: 0,
+			CodeGenerationWeek:  0,
+			CodeGenerationDay:   0,
+			ExplanationMonth:    20000,
+			ExplanationWeek:     15000,
+			ExplanationDay:      5000,
+			InvalidMonth:        0,
+			InvalidWeek:         0,
+			InvalidDay:          0,
 		},
 		{
 			Name:                "CodyVSCodeExtension:recipe:rewrite-to-functional:executed",
 			Month:               time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC),
 			Week:                now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5),
 			Day:                 now.Truncate(time.Hour * 24),
-			TotalMonth:          200,
-			TotalWeek:           150,
-			TotalDay:            50,
-			UniquesMonth:        2,
-			UniquesWeek:         2,
-			UniquesDay:          2,
-			CodeGenerationMonth: 200,
-			CodeGenerationDay:   50,
+			TotalMonth:          20000,
+			TotalWeek:           15000,
+			TotalDay:            5000,
+			UniquesMonth:        20,
+			UniquesWeek:         20,
+			UniquesDay:          20,
+			CodeGenerationMonth: 20000,
+			CodeGenerationWeek:  15000,
+			CodeGenerationDay:   5000,
+			ExplanationMonth:    0,
+			ExplanationWeek:     0,
+			ExplanationDay:      0,
+			InvalidMonth:        0,
+			InvalidWeek:         0,
+			InvalidDay:          0,
 		},
 	}
 

internal/database/schema.json

@@ -138,6 +138,18 @@
       "Name": "invalidate_session_for_userid_on_password_change",
       "Definition": "CREATE OR REPLACE FUNCTION public.invalidate_session_for_userid_on_password_change()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\n    BEGIN\n        IF OLD.passwd != NEW.passwd THEN\n            NEW.invalidated_sessions_at = now() + (1 * interval '1 second');\n            RETURN NEW;\n        END IF;\n    RETURN NEW;\n    END;\n$function$\n"
     },
+    {
+      "Name": "iscodyactiveevent",
+      "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n  RETURN\n    (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n    AND name NOT IN (\n        '%completion:started%',\n        '%completion:suggested%',\n        '%cta%',\n        '%Cta%',\n        'CodyVSCodeExtension:CodySavedLogin:executed',\n        'web:codyChat:tryOnPublicCode',\n        'web:codyEditorWidget:viewed',\n        'web:codyChat:pageViewed',\n        'CodyConfigurationPageViewed',\n        'ClickedOnTryCodySearchCTA',\n        'TryCodyWebOnboardingDisplayed',\n        'AboutGetCodyPopover',\n        'TryCodyWeb',\n        'CodySurveyToastViewed',\n        'SiteAdminCodyPageViewed',\n        'CodyUninstalled',\n        'SpeakToACodyEngineerCTA'\n    );\nEND;\n$function$\n"
+    },
+    {
+      "Name": "iscodyexplanationevent",
+      "Definition": "CREATE OR REPLACE FUNCTION public.iscodyexplanationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n  RETURN name = ANY(ARRAY[\n    'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n    'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n    'CodyVSCodeExtension:recipe:find-code-smells:executed',\n    'CodyVSCodeExtension:recipe:git-history:executed',\n    'CodyVSCodeExtension:recipe:rate-code:executed'\n  ]);\nEND;\n$function$\n"
+    },
+    {
+      "Name": "iscodygenerationevent",
+      "Definition": "CREATE OR REPLACE FUNCTION public.iscodygenerationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n  RETURN name = ANY(ARRAY[\n    'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n    'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n    'CodyVSCodeExtension:recipe:replace:executed',\n    'CodyVSCodeExtension:recipe:generate-docstring:executed',\n    'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n    'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n    'CodyVSCodeExtension:recipe:code-refactor:executed',\n    'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n  ]);\nEND;\n$function$\n"
+    },
     {
       "Name": "merge_audit_log_transitions",
       "Definition": "CREATE OR REPLACE FUNCTION public.merge_audit_log_transitions(internal hstore, arrayhstore hstore[])\n RETURNS hstore\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\n    DECLARE\n        trans hstore;\n    BEGIN\n      FOREACH trans IN ARRAY arrayhstore\n      LOOP\n          internal := internal || hstore(trans-\u003e'column', trans-\u003e'new');\n      END LOOP;\n\n      RETURN internal;\n    END;\n$function$\n"
@@ -10595,6 +10607,36 @@
           "ConstraintType": "",
           "ConstraintDefinition": ""
         },
+        {
+          "Name": "event_logs_name_is_cody_active_event",
+          "IsPrimaryKey": false,
+          "IsUnique": false,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name))",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
+        },
+        {
+          "Name": "event_logs_name_is_cody_explanation_event",
+          "IsPrimaryKey": false,
+          "IsUnique": false,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name))",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
+        },
+        {
+          "Name": "event_logs_name_is_cody_generation_event",
+          "IsPrimaryKey": false,
+          "IsUnique": false,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name))",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
+        },
         {
           "Name": "event_logs_name_timestamp",
           "IsPrimaryKey": false,

internal/database/schema.md

@@ -1379,6 +1379,9 @@ Referenced by:
 Indexes:
     "event_logs_pkey" PRIMARY KEY, btree (id)
     "event_logs_anonymous_user_id" btree (anonymous_user_id)
+    "event_logs_name_is_cody_active_event" btree (iscodyactiveevent(name))
+    "event_logs_name_is_cody_explanation_event" btree (iscodyexplanationevent(name))
+    "event_logs_name_is_cody_generation_event" btree (iscodygenerationevent(name))
     "event_logs_name_timestamp" btree (name, "timestamp" DESC)
     "event_logs_source" btree (source)
     "event_logs_timestamp" btree ("timestamp")

migrations/BUILD.bazel

@@ -1114,6 +1114,9 @@ go_library(
         "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql",
         "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml",
         "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql",
+        "frontend/1691043630_event_logs_indexing/down.sql",
+        "frontend/1691043630_event_logs_indexing/metadata.yaml",
+        "frontend/1691043630_event_logs_indexing/up.sql",
     ],
     importpath = "github.com/sourcegraph/sourcegraph/migrations",
     visibility = ["//visibility:public"],

migrations/frontend/1691043630_event_logs_indexing/down.sql

@@ -0,0 +1,11 @@
+DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event;
+
+DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event;
+
+DROP INDEX IF EXISTS event_logs_name_is_cody_active_event;
+
+DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text);
+
+DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text);
+
+DROP FUNCTION IF EXISTS isCodyActiveEvent(name text);

migrations/frontend/1691043630_event_logs_indexing/metadata.yaml

@@ -0,0 +1,2 @@
+name: event_logs_indexing
+parents: [1690460411, 1690323910]

migrations/frontend/1691043630_event_logs_indexing/up.sql

@@ -0,0 +1,65 @@
+CREATE OR REPLACE FUNCTION iscodyactiveevent(name text) RETURNS boolean
+    LANGUAGE plpgsql IMMUTABLE
+    AS $$
+BEGIN
+  RETURN
+    (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')
+    AND name NOT IN (
+        '%completion:started%',
+        '%completion:suggested%',
+        '%cta%',
+        '%Cta%',
+        'CodyVSCodeExtension:CodySavedLogin:executed',
+        'web:codyChat:tryOnPublicCode',
+        'web:codyEditorWidget:viewed',
+        'web:codyChat:pageViewed',
+        'CodyConfigurationPageViewed',
+        'ClickedOnTryCodySearchCTA',
+        'TryCodyWebOnboardingDisplayed',
+        'AboutGetCodyPopover',
+        'TryCodyWeb',
+        'CodySurveyToastViewed',
+        'SiteAdminCodyPageViewed',
+        'CodyUninstalled',
+        'SpeakToACodyEngineerCTA'
+    );
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION iscodyexplanationevent(name text) RETURNS boolean
+    LANGUAGE plpgsql IMMUTABLE
+    AS $$
+BEGIN
+  RETURN name = ANY(ARRAY[
+    'CodyVSCodeExtension:recipe:explain-code-high-level:executed',
+    'CodyVSCodeExtension:recipe:explain-code-detailed:executed',
+    'CodyVSCodeExtension:recipe:find-code-smells:executed',
+    'CodyVSCodeExtension:recipe:git-history:executed',
+    'CodyVSCodeExtension:recipe:rate-code:executed'
+  ]);
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION iscodygenerationevent(name text) RETURNS boolean
+    LANGUAGE plpgsql IMMUTABLE
+    AS $$
+BEGIN
+  RETURN name = ANY(ARRAY[
+    'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',
+    'CodyVSCodeExtension:recipe:improve-variable-names:executed',
+    'CodyVSCodeExtension:recipe:replace:executed',
+    'CodyVSCodeExtension:recipe:generate-docstring:executed',
+    'CodyVSCodeExtension:recipe:generate-unit-test:executed',
+    'CodyVSCodeExtension:recipe:rewrite-functional:executed',
+    'CodyVSCodeExtension:recipe:code-refactor:executed',
+    'CodyVSCodeExtension:recipe:fixup:executed',
+	'CodyVSCodeExtension:recipe:translate-to-language:executed'
+  ]);
+END;
+$$;
+
+CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name));
+
+CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name));
+
+CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name));