From 406baf822cc9b2e6b5643897429c3810cc93d779 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:33:55 -0700 Subject: [PATCH 01/77] internal/event_logs: optimize ping cte The aggregatedCodyUsageEventsQuery is timing out for dotcom. These changes are cte optimization steps. --- internal/database/event_logs.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index cc0229b67d93..a485ed470ee0 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1592,7 +1592,16 @@ var nonActiveCodyEvents = []string{ } var aggregatedCodyUsageEventsQuery = ` -WITH events AS ( +WITH +sub_cohort AS ( + SELECT * + FROM event_logs + WHERE + timestamp >= %s::timestamp - '1 month'::interval + AND name not like '%%completion:started%%' + AND name not like '%%completion:suggested%%' +), +events AS ( SELECT name AS key, ` + aggregatedUserIDQueryFragment + ` AS user_id, @@ -1604,8 +1613,7 @@ WITH events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - timestamp >= ` + makeDateTruncExpression("month", "%s::timestamp") + ` - AND lower(name) like '%%cody%%' + lower(name) like '%%cody%%' AND name not like '%%CTA%%' AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) From 383bc92a348290dc8d53f223ecd7c875b56febcc Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 2 Aug 2023 23:25:45 -0700 Subject: [PATCH 02/77] internal/event_logs: event_logs indexing Add indexes to event_logs table to increase query performance. --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 3 +++ .../frontend/1691043630_event_logs_indexing/metadata.yaml | 2 ++ migrations/frontend/1691043630_event_logs_indexing/up.sql | 3 +++ 3 files changed, 8 insertions(+) create mode 100644 migrations/frontend/1691043630_event_logs_indexing/down.sql create mode 100644 migrations/frontend/1691043630_event_logs_indexing/metadata.yaml create mode 100644 migrations/frontend/1691043630_event_logs_indexing/up.sql diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql new file mode 100644 index 000000000000..4beed28c7fc8 --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -0,0 +1,3 @@ +DROP INDEX event_logs_name; + +DROP INDEX event_logs_name; diff --git a/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml b/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml new file mode 100644 index 000000000000..0c01d8abf90c --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml @@ -0,0 +1,2 @@ +name: event_logs_indexing +parents: [1690460411, 1690323910] diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql new file mode 100644 index 000000000000..bf55394693af --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -0,0 +1,3 @@ +CREATE INDEX event_logs_name ON event_logs (name); + +CREATE INDEX event_logs_timestamp ON event_logs (timestamp); From 33e60da9b7f3a0548461cad5fad87477b211848c Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:26:50 -0700 Subject: [PATCH 03/77] internal/event_logs: sg lint --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 4 ++-- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 4beed28c7fc8..4ad6ae8b3597 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,3 @@ -DROP INDEX event_logs_name; +DROP INDEX IF EXISTS event_logs_name; -DROP INDEX event_logs_name; +DROP INDEX IF EXISTS event_logs_name; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index bf55394693af..d7915c8d69c5 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ -CREATE INDEX event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); -CREATE INDEX event_logs_timestamp ON event_logs (timestamp); +CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs (timestamp); From 5ef4c9eefd390194d0f257488be9923da25a3b17 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:11:22 -0700 Subject: [PATCH 04/77] internal/event_logs: sg lint fix --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 4ad6ae8b3597..12c4ac06d459 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,3 @@ DROP INDEX IF EXISTS event_logs_name; -DROP INDEX IF EXISTS event_logs_name; +DROP INDEX IF EXISTS event_logs_timestamp; From 47965ec20ca2a8cd1aba35bf1736d00ae61cd293 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:14:36 -0700 Subject: [PATCH 05/77] internal/event_logs: bazel configure --- dev/linters/bodyclose/BUILD.bazel | 5 +---- dev/linters/depguard/BUILD.bazel | 1 - dev/linters/exportloopref/BUILD.bazel | 5 +---- dev/linters/gocheckcompilerdirectives/BUILD.bazel | 5 +---- dev/linters/unparam/BUILD.bazel | 1 - migrations/BUILD.bazel | 3 +++ 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/dev/linters/bodyclose/BUILD.bazel b/dev/linters/bodyclose/BUILD.bazel index 93228f234266..9d6c52923f38 100644 --- a/dev/linters/bodyclose/BUILD.bazel +++ b/dev/linters/bodyclose/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["bodyclose.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/bodyclose", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_github_timakin_bodyclose//passes/bodyclose:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/depguard/BUILD.bazel b/dev/linters/depguard/BUILD.bazel index 7b5fca9789e0..6cfb17e89c98 100644 --- a/dev/linters/depguard/BUILD.bazel +++ b/dev/linters/depguard/BUILD.bazel @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", - "@com_github_openpeedeep_depguard_v2//:go_default_library", "@org_golang_x_tools//go/analysis", ], ) diff --git a/dev/linters/exportloopref/BUILD.bazel b/dev/linters/exportloopref/BUILD.bazel index a49fce597026..cfab09d40786 100644 --- a/dev/linters/exportloopref/BUILD.bazel +++ b/dev/linters/exportloopref/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["exportloopref.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/exportloopref", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_github_kyoh86_exportloopref//:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/gocheckcompilerdirectives/BUILD.bazel b/dev/linters/gocheckcompilerdirectives/BUILD.bazel index 0544019392fb..0650219f5263 100644 --- a/dev/linters/gocheckcompilerdirectives/BUILD.bazel +++ b/dev/linters/gocheckcompilerdirectives/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["gocheckcompilerdirectives.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/gocheckcompilerdirectives", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_4d63_gocheckcompilerdirectives//checkcompilerdirectives:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/unparam/BUILD.bazel b/dev/linters/unparam/BUILD.bazel index 39aaebe7497f..becc863acebd 100644 --- a/dev/linters/unparam/BUILD.bazel +++ b/dev/linters/unparam/BUILD.bazel @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", - "@cc_mvdan_unparam//check:go_default_library", "@org_golang_x_tools//go/analysis", "@org_golang_x_tools//go/analysis/passes/buildssa", "@org_golang_x_tools//go/packages", diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 3cbf784adc96..86bcadae3fff 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,6 +1114,9 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", + "frontend/1691043630_event_logs_indexing/down.sql", + "frontend/1691043630_event_logs_indexing/metadata.yaml", + "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From 97ce6608654b32532bcc88bc4404bde9fbd787a0 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:22:22 -0700 Subject: [PATCH 06/77] Revert "internal/event_logs: bazel configure" This reverts commit 47965ec20ca2a8cd1aba35bf1736d00ae61cd293. --- dev/linters/bodyclose/BUILD.bazel | 5 ++++- dev/linters/depguard/BUILD.bazel | 1 + dev/linters/exportloopref/BUILD.bazel | 5 ++++- dev/linters/gocheckcompilerdirectives/BUILD.bazel | 5 ++++- dev/linters/unparam/BUILD.bazel | 1 + migrations/BUILD.bazel | 3 --- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dev/linters/bodyclose/BUILD.bazel b/dev/linters/bodyclose/BUILD.bazel index 9d6c52923f38..93228f234266 100644 --- a/dev/linters/bodyclose/BUILD.bazel +++ b/dev/linters/bodyclose/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["bodyclose.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/bodyclose", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_github_timakin_bodyclose//passes/bodyclose:go_default_library", + ], ) diff --git a/dev/linters/depguard/BUILD.bazel b/dev/linters/depguard/BUILD.bazel index 6cfb17e89c98..7b5fca9789e0 100644 --- a/dev/linters/depguard/BUILD.bazel +++ b/dev/linters/depguard/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", + "@com_github_openpeedeep_depguard_v2//:go_default_library", "@org_golang_x_tools//go/analysis", ], ) diff --git a/dev/linters/exportloopref/BUILD.bazel b/dev/linters/exportloopref/BUILD.bazel index cfab09d40786..a49fce597026 100644 --- a/dev/linters/exportloopref/BUILD.bazel +++ b/dev/linters/exportloopref/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["exportloopref.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/exportloopref", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_github_kyoh86_exportloopref//:go_default_library", + ], ) diff --git a/dev/linters/gocheckcompilerdirectives/BUILD.bazel b/dev/linters/gocheckcompilerdirectives/BUILD.bazel index 0650219f5263..0544019392fb 100644 --- a/dev/linters/gocheckcompilerdirectives/BUILD.bazel +++ b/dev/linters/gocheckcompilerdirectives/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["gocheckcompilerdirectives.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/gocheckcompilerdirectives", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_4d63_gocheckcompilerdirectives//checkcompilerdirectives:go_default_library", + ], ) diff --git a/dev/linters/unparam/BUILD.bazel b/dev/linters/unparam/BUILD.bazel index becc863acebd..39aaebe7497f 100644 --- a/dev/linters/unparam/BUILD.bazel +++ b/dev/linters/unparam/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", + "@cc_mvdan_unparam//check:go_default_library", "@org_golang_x_tools//go/analysis", "@org_golang_x_tools//go/analysis/passes/buildssa", "@org_golang_x_tools//go/packages", diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 86bcadae3fff..3cbf784adc96 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,9 +1114,6 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", - "frontend/1691043630_event_logs_indexing/down.sql", - "frontend/1691043630_event_logs_indexing/metadata.yaml", - "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From 09ea293856d5028c7cb8c4bd0aaf7de10b67c8ae Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:58:01 -0700 Subject: [PATCH 07/77] internal/event_logs: schema definition update --- internal/database/schema.json | 10 ++++++++++ internal/database/schema.md | 1 + migrations/frontend/squashed.sql | 2 ++ 3 files changed, 13 insertions(+) diff --git a/internal/database/schema.json b/internal/database/schema.json index 6684115b5a16..fbf06372c8af 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -10595,6 +10595,16 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING btree (name)", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 0d939903eded..a885939ee210 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name" btree (name) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index c0d8590259ae..a2b665f73099 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,6 +5866,8 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); +CREATE INDEX event_logs_name ON event_logs USING btree (name); + CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); CREATE INDEX event_logs_source ON event_logs USING btree (source); From ad282b710ebc5ec417c1661e42f0fa78b3c385ad Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:00:45 -0700 Subject: [PATCH 08/77] internal/event_logs: schema definition update --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index fbf06372c8af..2f45caddfbf5 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -28567,4 +28567,4 @@ "Definition": " SELECT changeset_specs.id AS changeset_spec_id,\n COALESCE(changesets.id, (0)::bigint) AS changeset_id,\n changeset_specs.repo_id,\n changeset_specs.batch_spec_id,\n repo.name AS repo_name,\n COALESCE((changesets.metadata -\u003e\u003e 'Title'::text), (changesets.metadata -\u003e\u003e 'title'::text)) AS changeset_name,\n changesets.external_state,\n changesets.publication_state,\n changesets.reconciler_state,\n changesets.computed_state\n FROM ((changeset_specs\n LEFT JOIN changesets ON (((changesets.repo_id = changeset_specs.repo_id) AND (changesets.external_id = changeset_specs.external_id))))\n JOIN repo ON ((changeset_specs.repo_id = repo.id)))\n WHERE ((changeset_specs.external_id IS NOT NULL) AND (repo.deleted_at IS NULL));" } ] -} \ No newline at end of file +} diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index a2b665f73099..d01a6b464a56 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file +SELECT pg_catalog.setval('roles_id_seq', 3, true); From 0ea51e2d499ce9db7458878590d895737f6a821b Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:06:55 -0700 Subject: [PATCH 09/77] internal/event_logs: bazel update --- migrations/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 3cbf784adc96..86bcadae3fff 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,6 +1114,9 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", + "frontend/1691043630_event_logs_indexing/down.sql", + "frontend/1691043630_event_logs_indexing/metadata.yaml", + "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From 85386837e94065a9b901391fdae0f492e235a6fb Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:37:23 -0700 Subject: [PATCH 10/77] internal/event_logs: manual lint fixes From 78c9018fbb8b30515cd19a609ea62e3b8e9e70b5 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:38:58 -0700 Subject: [PATCH 11/77] internal/event_logs: manual lint fixes #2 From d0bb130f2d393c65904624fde309576741469f86 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:54:28 -0700 Subject: [PATCH 12/77] internal/event_logs: revert lint breaking commit --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index 2f45caddfbf5..fbf06372c8af 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -28567,4 +28567,4 @@ "Definition": " SELECT changeset_specs.id AS changeset_spec_id,\n COALESCE(changesets.id, (0)::bigint) AS changeset_id,\n changeset_specs.repo_id,\n changeset_specs.batch_spec_id,\n repo.name AS repo_name,\n COALESCE((changesets.metadata -\u003e\u003e 'Title'::text), (changesets.metadata -\u003e\u003e 'title'::text)) AS changeset_name,\n changesets.external_state,\n changesets.publication_state,\n changesets.reconciler_state,\n changesets.computed_state\n FROM ((changeset_specs\n LEFT JOIN changesets ON (((changesets.repo_id = changeset_specs.repo_id) AND (changesets.external_id = changeset_specs.external_id))))\n JOIN repo ON ((changeset_specs.repo_id = repo.id)))\n WHERE ((changeset_specs.external_id IS NOT NULL) AND (repo.deleted_at IS NULL));" } ] -} +} \ No newline at end of file diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index d01a6b464a56..a2b665f73099 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); +SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file From ebd34c45a31684f860dfc95463e4573ffec85058 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:01:59 -0700 Subject: [PATCH 13/77] internal/event_logs: backcompat test drift fix --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d7915c8d69c5..f34215b0efed 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); -CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs (timestamp); +CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From 98a9beecdf3c85b33a4dbb46648acd0f524596fa Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:38:07 -0700 Subject: [PATCH 14/77] internal/event_logs: update name index to lower --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index f34215b0efed..07f55ef5df70 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (lower(name)); CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From c7b5ac1036b1181905f8e9a280b2622c9b0a4daf Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:40:34 -0700 Subject: [PATCH 15/77] internal/event_logs: event log indexs --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 2 ++ migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 12c4ac06d459..b4694a825916 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,5 @@ DROP INDEX IF EXISTS event_logs_name; +DROP INDEX IF EXISTS event_logs_name_lower; + DROP INDEX IF EXISTS event_logs_timestamp; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 07f55ef5df70..18aff6f7e33e 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,5 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (lower(name)); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); + +CREATE INDEX IF NOT EXISTS event_logs_name_lower ON event_logs (lower(name)); CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From adc6f06d284cc2dd24e0e2c79e562e92bde44196 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:54:56 -0700 Subject: [PATCH 16/77] internal/event_logs: remove lower index It's computationally expensive to have an index on both the column and the lower(column) simuiltaneously. We need the index on the column so we're going to take the computational hit in the where clause. There is already timestamp, removing redundant migration. --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 18aff6f7e33e..7268d19f0685 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,5 +1 @@ CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); - -CREATE INDEX IF NOT EXISTS event_logs_name_lower ON event_logs (lower(name)); - -CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From f5b43f965b716d7061cf90c6305915f3c7b5fa70 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:02:28 -0700 Subject: [PATCH 17/77] Update down.sql --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index b4694a825916..52bc712a47ab 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,5 +1 @@ DROP INDEX IF EXISTS event_logs_name; - -DROP INDEX IF EXISTS event_logs_name_lower; - -DROP INDEX IF EXISTS event_logs_timestamp; From bc8be7b582d4a7ad291d81b605e5d837719cd0fd Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:35:07 -0700 Subject: [PATCH 18/77] Update event_logs.go --- internal/database/event_logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index a485ed470ee0..25dceb6f9bed 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1613,7 +1613,7 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - lower(name) like '%%cody%%' + (name like '%%cody%%' or name like '%%Cody%%') AND name not like '%%CTA%%' AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) From 631df5b90ef296efa3d6ec301690788c62d2dea0 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:58:57 -0700 Subject: [PATCH 19/77] internal/event_logs: add GIN index A GIN index should improve performance of text searches. --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- migrations/frontend/squashed.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 7268d19f0685..d22d4a985dec 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1 +1 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index a2b665f73099..010ab6c5249e 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,7 +5866,7 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); -CREATE INDEX event_logs_name ON event_logs USING btree (name); +CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file +SELECT pg_catalog.setval('roles_id_seq', 3, true); From 494a4484156e15f80e06694fbeee7126d15bc916 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Mon, 7 Aug 2023 16:41:41 -0700 Subject: [PATCH 20/77] internal/event_logs: add GIN index Additional schema and migration infrastructure. --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index fbf06372c8af..1eada8719b31 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -10601,7 +10601,7 @@ "IsUnique": false, "IsExclusion": false, "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING btree (name)", + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", "ConstraintType": "", "ConstraintDefinition": "" }, diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 010ab6c5249e..df928458e8d0 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,7 +5866,7 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); -CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); +SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file From 3012be901f09c7729fe1a3b1a6e626b6fdd14d36 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:57:09 -0700 Subject: [PATCH 21/77] internal/event_logs: query efficiency changes It's more efficient to perform case-insensitive comparisons using ilike instead of like. --- internal/database/event_logs.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index 25dceb6f9bed..e5a5102e2ae7 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1598,8 +1598,8 @@ sub_cohort AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND name not like '%%completion:started%%' - AND name not like '%%completion:suggested%%' + AND name not ilike '%%completion:started%%' + AND name not ilike '%%completion:suggested%%' ), events AS ( SELECT @@ -1613,9 +1613,8 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - (name like '%%cody%%' or name like '%%Cody%%') - AND name not like '%%CTA%%' - AND name not like '%%Cta%%' + name ilike '%%cody%%' + AND name not ilike '%%cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) ), code_generation_keys AS ( From 487f44b3b16d60ee5c2ac40328db11c2cf699dfd Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:58:27 -0700 Subject: [PATCH 22/77] internal/event_logs: sql logic and lint fixes --- internal/database/event_logs.go | 16 ++++++---------- internal/database/schema.md | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index e5a5102e2ae7..e927b274f1d3 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1593,14 +1593,6 @@ var nonActiveCodyEvents = []string{ var aggregatedCodyUsageEventsQuery = ` WITH -sub_cohort AS ( - SELECT * - FROM event_logs - WHERE - timestamp >= %s::timestamp - '1 month'::interval - AND name not ilike '%%completion:started%%' - AND name not ilike '%%completion:suggested%%' -), events AS ( SELECT name AS key, @@ -1613,8 +1605,12 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - name ilike '%%cody%%' - AND name not ilike '%%cta%%' + timestamp >= %s::timestamp - '1 month'::interval + AND name not ilike '%%completion:started%%' + AND name not ilike '%%completion:suggested%%' + AND name ilike '%%cody%%' + AND name not like '%%cta%%' + AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) ), code_generation_keys AS ( diff --git a/internal/database/schema.md b/internal/database/schema.md index a885939ee210..559e47adbd3a 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,7 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) - "event_logs_name" btree (name) + "event_logs_name" gin (name gin_trgm_ops) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") From 474e3013417bcf5a926886694d851f3f628cccf9 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:38:38 -0700 Subject: [PATCH 23/77] internal/event_logs: add lookup fuctions Add lookup functions to categorize and filter event logs and aid performance. --- internal/database/schema.json | 42 ++++++++++++ internal/database/schema.md | 3 + .../1691043630_event_logs_indexing/down.sql | 12 ++++ .../1691043630_event_logs_indexing/up.sql | 63 ++++++++++++++++++ migrations/frontend/squashed.sql | 66 ++++++++++++++++++- 5 files changed, 185 insertions(+), 1 deletion(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index 1eada8719b31..3a5f5a5b4749 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -201,6 +201,18 @@ { "Name": "versions_insert_row_trigger", "Definition": "CREATE OR REPLACE FUNCTION public.versions_insert_row_trigger()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\nBEGIN\n NEW.first_version = NEW.version;\n RETURN NEW;\nEND $function$\n" + }, + { + "Name": "isCodyGenerationEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyGenerationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" + }, + { + "Name": "isCodyExplanationEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyExplanationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" + }, + { + "Name": "isCodyActiveEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyActiveEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" } ], "Sequences": [ @@ -10605,6 +10617,36 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name_is_cody_explanation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_generation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_active_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 559e47adbd3a..34a739ed2053 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1380,6 +1380,9 @@ Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) "event_logs_name" gin (name gin_trgm_ops) + "event_logs_name_is_cody_explanation_event" isCodyExplanationEvent(name) + "event_logs_name_is_cody_generation_event" isCodyGenerationEvent(name) + "event_logs_name_is_cody_active_event" isCodyActiveEvent(name) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 52bc712a47ab..57ec8a573978 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1 +1,13 @@ +DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text) + +DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text) + +DROP FUNCTION IF EXISTS isCodyActiveEvent(name text) + DROP INDEX IF EXISTS event_logs_name; + +DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; + +DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event; + +DROP INDEX IF EXISTS event_logs_name_is_cody_active_event; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d22d4a985dec..db39785d2b62 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1 +1,64 @@ +CREATE FUNCTION isCodyGenerationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyExplanationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyActiveEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index df928458e8d0..85e972179c4a 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -865,7 +865,65 @@ CREATE FUNCTION versions_insert_row_trigger() RETURNS trigger BEGIN NEW.first_version = NEW.version; RETURN NEW; -END $$; +END +$$; + +CREATE FUNCTION isCodyGenerationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyExplanationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyActiveEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$ LANGUAGE plpgsql IMMUTABLE; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, @@ -5868,6 +5926,12 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); + +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); + +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); + CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); CREATE INDEX event_logs_source ON event_logs USING btree (source); From a62bc06a0f099a6d57dc4ac12bc10e6ad6c995e3 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:05:33 -0700 Subject: [PATCH 24/77] internal/event_logs: refactor cody usage Simplify the aggregatedCodyUsageEvents query to utilize added indexes to improve performance. --- internal/database/event_logs.go | 66 +++++---------------------------- internal/database/schema.json | 52 -------------------------- internal/database/schema.md | 5 --- 3 files changed, 10 insertions(+), 113 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index e927b274f1d3..fa5a0899931b 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1574,28 +1574,11 @@ func (l *eventLogStore) aggregatedSearchEvents(ctx context.Context, queryString return events, nil } -// List of events that don't meet the criteria of "active" usage of Cody. -var nonActiveCodyEvents = []string{ - "CodyVSCodeExtension:CodySavedLogin:executed", - "web:codyChat:tryOnPublicCode", - "web:codyEditorWidget:viewed", - "web:codyChat:pageViewed", - "CodyConfigurationPageViewed", - "ClickedOnTryCodySearchCTA", - "TryCodyWebOnboardingDisplayed", - "AboutGetCodyPopover", - "TryCodyWeb", - "CodySurveyToastViewed", - "SiteAdminCodyPageViewed", - "CodyUninstalled", - "SpeakToACodyEngineerCTA", -} - var aggregatedCodyUsageEventsQuery = ` WITH events AS ( SELECT - name AS key, + name, ` + aggregatedUserIDQueryFragment + ` AS user_id, ` + makeDateTruncExpression("month", "timestamp") + ` as month, ` + makeDateTruncExpression("week", "timestamp") + ` as week, @@ -1606,37 +1589,10 @@ events AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND name not ilike '%%completion:started%%' - AND name not ilike '%%completion:suggested%%' - AND name ilike '%%cody%%' - AND name not like '%%cta%%' - AND name not like '%%Cta%%' - AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) -), -code_generation_keys AS ( - SELECT * FROM unnest(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]) AS key -), -explanation_keys AS ( - SELECT * FROM unnest(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]) AS key + AND isCodyActiveEvent(name) ) SELECT - key, + name, current_month, current_week, current_day, @@ -1646,25 +1602,23 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and key in - (SELECT * FROM code_generation_keys) + SUM(case when month = current_month and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and key in - (SELECT * FROM explanation_keys) + SUM(case when week = current_week and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and key in (SELECT * FROM code_generation_keys) + SUM(case when day = current_day and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and key in (SELECT * FROM explanation_keys) + SUM(case when month = current_month and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and key in (SELECT * FROM explanation_keys) + SUM(case when week = current_week and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and key in (SELECT * FROM explanation_keys) + SUM(case when day = current_day and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, 0 as invalid_day FROM events -GROUP BY key, current_month, current_week, current_day +GROUP BY name, current_month, current_week, current_day ` var searchLatencyEventNames = []string{ diff --git a/internal/database/schema.json b/internal/database/schema.json index 3a5f5a5b4749..6684115b5a16 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -201,18 +201,6 @@ { "Name": "versions_insert_row_trigger", "Definition": "CREATE OR REPLACE FUNCTION public.versions_insert_row_trigger()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\nBEGIN\n NEW.first_version = NEW.version;\n RETURN NEW;\nEND $function$\n" - }, - { - "Name": "isCodyGenerationEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyGenerationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" - }, - { - "Name": "isCodyExplanationEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyExplanationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" - }, - { - "Name": "isCodyActiveEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyActiveEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" } ], "Sequences": [ @@ -10607,46 +10595,6 @@ "ConstraintType": "", "ConstraintDefinition": "" }, - { - "Name": "event_logs_name", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_explanation_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_generation_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_active_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 34a739ed2053..fa7792bfc8b0 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,11 +1379,6 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) - "event_logs_name" gin (name gin_trgm_ops) - "event_logs_name_is_cody_explanation_event" isCodyExplanationEvent(name) - "event_logs_name_is_cody_generation_event" isCodyGenerationEvent(name) - "event_logs_name_is_cody_active_event" isCodyActiveEvent(name) - "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") "event_logs_timestamp_at_utc" btree (date(timezone('UTC'::text, "timestamp"))) From e0fe148fc2271add8966480d52a473fc579a2fe9 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:06:55 -0700 Subject: [PATCH 25/77] internal/event_logs: revert deletion --- internal/database/schema.md | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/database/schema.md b/internal/database/schema.md index fa7792bfc8b0..0d939903eded 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") "event_logs_timestamp_at_utc" btree (date(timezone('UTC'::text, "timestamp"))) From 89dd5f1e0a88135054d07fcdd0812fc4e7e743a0 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:24:55 -0700 Subject: [PATCH 26/77] internal/event_logs: lint fixes --- .../1691043630_event_logs_indexing/up.sql | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index db39785d2b62..607aa2de166a 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,5 +1,6 @@ -CREATE FUNCTION isCodyGenerationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', @@ -13,10 +14,11 @@ BEGIN 'CodyVSCodeExtension:recipe:translate-to-language:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyExplanationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', @@ -28,8 +30,9 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; -CREATE FUNCTION isCodyActiveEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') @@ -57,8 +60,8 @@ $$ LANGUAGE plpgsql IMMUTABLE; CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); From 30b4c34ff6078fa1e104963b7f16352962e19f64 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:41:12 -0700 Subject: [PATCH 27/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 85e972179c4a..fb30387acbaf 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5926,11 +5926,11 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); From 1d4105af6d0570f293cbf201f55e211d5b292214 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 03:49:23 -0700 Subject: [PATCH 28/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index fb30387acbaf..6d7289ecb8ed 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -868,8 +868,9 @@ BEGIN END $$; -CREATE FUNCTION isCodyGenerationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', @@ -883,10 +884,11 @@ BEGIN 'CodyVSCodeExtension:recipe:translate-to-language:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyExplanationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', @@ -896,10 +898,11 @@ BEGIN 'CodyVSCodeExtension:recipe:rate-code:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyActiveEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean +LANGUAGE plpgsql IMMUTABLE +AS $$ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') @@ -923,7 +926,7 @@ BEGIN 'CodyUninstalled', 'SpeakToACodyEngineerCTA'])); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, From ec866edcdcd92b4ec68fbbab7d61a44555f7665f Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:03:43 -0700 Subject: [PATCH 29/77] internal/event_logs: lint fixes --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 607aa2de166a..57e55d35f111 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -28,7 +28,7 @@ BEGIN 'CodyVSCodeExtension:recipe:rate-code:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE @@ -56,7 +56,7 @@ BEGIN 'CodyUninstalled', 'SpeakToACodyEngineerCTA'])); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); From a1b64d14655f1dac4b301f2d71e4da4cdeda0506 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:32:36 -0700 Subject: [PATCH 30/77] internal/event_logs: lint fixes --- internal/database/event_logs.go | 14 ++-- internal/database/schema.json | 52 +++++++++++++ internal/database/schema.md | 4 + migrations/frontend/squashed.sql | 126 +++++++++++++++---------------- 4 files changed, 126 insertions(+), 70 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index fa5a0899931b..1e7aab2015c5 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1589,7 +1589,7 @@ events AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND isCodyActiveEvent(name) + AND iscodyactiveevent(name) ) SELECT name, @@ -1602,17 +1602,17 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and isCodeGenerationEvent(name) + SUM(case when month = current_month and iscodegenerationevent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and isCodeGenerationEvent(name) + SUM(case when week = current_week and iscodegenerationevent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and isCodeGenerationEvent(name) + SUM(case when day = current_day and iscodegenerationevent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and isCodeExplanationEvent(name) + SUM(case when month = current_month and iscodeexplanationevent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and isCodeExplanationEvent(name) + SUM(case when week = current_week and iscodeexplanationevent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and isCodeExplanationEvent(name) + SUM(case when day = current_day and iscodeexplanationevent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, diff --git a/internal/database/schema.json b/internal/database/schema.json index 6684115b5a16..b59e884a0315 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -138,6 +138,18 @@ "Name": "invalidate_session_for_userid_on_password_change", "Definition": "CREATE OR REPLACE FUNCTION public.invalidate_session_for_userid_on_password_change()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\n BEGIN\n IF OLD.passwd != NEW.passwd THEN\n NEW.invalidated_sessions_at = now() + (1 * interval '1 second');\n RETURN NEW;\n END IF;\n RETURN NEW;\n END;\n$function$\n" }, + { + "Name": "iscodyactiveevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$function$\n" + }, + { + "Name": "iscodyexplanationevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyexplanationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$function$\n" + }, + { + "Name": "iscodygenerationevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodygenerationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$function$\n" + }, { "Name": "merge_audit_log_transitions", "Definition": "CREATE OR REPLACE FUNCTION public.merge_audit_log_transitions(internal hstore, arrayhstore hstore[])\n RETURNS hstore\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\n DECLARE\n trans hstore;\n BEGIN\n FOREACH trans IN ARRAY arrayhstore\n LOOP\n internal := internal || hstore(trans-\u003e'column', trans-\u003e'new');\n END LOOP;\n\n RETURN internal;\n END;\n$function$\n" @@ -10595,6 +10607,46 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_active_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_explanation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_generation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 0d939903eded..ab1d86852af8 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,10 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name" gin (name gin_trgm_ops) + "event_logs_name_is_cody_active_event" btree (iscodyactiveevent(name)) + "event_logs_name_is_cody_explanation_event" btree (iscodyexplanationevent(name)) + "event_logs_name_is_cody_generation_event" btree (iscodygenerationevent(name)) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 6d7289ecb8ed..450a3f91c3f6 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -451,6 +451,66 @@ CREATE FUNCTION invalidate_session_for_userid_on_password_change() RETURNS trigg END; $$; +CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$; + +CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$; + +CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$; + CREATE FUNCTION merge_audit_log_transitions(internal hstore, arrayhstore hstore[]) RETURNS hstore LANGUAGE plpgsql IMMUTABLE AS $$ @@ -868,66 +928,6 @@ BEGIN END $$; -CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean -LANGUAGE plpgsql IMMUTABLE -AS $$ -BEGIN - RETURN - (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); -END; -$$; - CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, STYPE = hstore, @@ -5929,11 +5929,11 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); From 45810ac6b4e066f9f71ec33cb377e7814db55897 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:47:25 -0700 Subject: [PATCH 31/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 450a3f91c3f6..0a652e92e0af 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -925,8 +925,7 @@ CREATE FUNCTION versions_insert_row_trigger() RETURNS trigger BEGIN NEW.first_version = NEW.version; RETURN NEW; -END -$$; +END $$; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, From ae252153706ed2601933e2473e2ad78097260d84 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 05:14:39 -0700 Subject: [PATCH 32/77] internal/event_logs: typo fixes --- internal/database/event_logs.go | 12 ++-- .../1691043630_event_logs_indexing/up.sql | 72 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index 1e7aab2015c5..42c4ac1bb40d 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1602,17 +1602,17 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and iscodegenerationevent(name) + SUM(case when month = current_month and iscodygenerationevent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and iscodegenerationevent(name) + SUM(case when week = current_week and iscodygenerationevent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and iscodegenerationevent(name) + SUM(case when day = current_day and iscodygenerationevent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and iscodeexplanationevent(name) + SUM(case when month = current_month and iscodyexplanationevent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and iscodeexplanationevent(name) + SUM(case when week = current_week and iscodyexplanationevent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and iscodeexplanationevent(name) + SUM(case when day = current_day and iscodyexplanationevent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 57e55d35f111..3a8855d4bf84 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,36 +1,4 @@ -CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean +CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +26,42 @@ BEGIN END; $$; +CREATE FUNCTION IF NOT EXISTS iscodyexplanationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$; + +CREATE FUNCTION IF NOT EXISTS iscodygenerationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$; + CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 83f70622c2b315ea0ad9059c816d4a1cee23eeef Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 10:32:17 -0700 Subject: [PATCH 33/77] internal/event_logs: bazel test fixes --- internal/database/schema.json | 2 +- .../1691043630_event_logs_indexing/up.sql | 37 ++++++++++--------- migrations/frontend/squashed.sql | 37 ++++++++++--------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index b59e884a0315..f66fa5c8992f 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -140,7 +140,7 @@ }, { "Name": "iscodyactiveevent", - "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$function$\n" + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n ((\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'])));\nEND;\n$function$\n" }, { "Name": "iscodyexplanationevent", diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 3a8855d4bf84..33692d6a2fb5 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -5,24 +5,25 @@ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); + (( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA']) + )); END; $$; diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 0a652e92e0af..1153abb91ce3 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -458,24 +458,25 @@ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); + (( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA']) + )); END; $$; From 1e3333a0eace6f3c0391536157bc820c4ab23933 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:15:09 -0700 Subject: [PATCH 34/77] internal/event_logs: bazel test fixes --- .../1691043630_event_logs_indexing/up.sql | 39 +++++++++---------- migrations/frontend/squashed.sql | 39 +++++++++---------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 33692d6a2fb5..d28a06b9f93c 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -4,26 +4,25 @@ CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - (( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA']) - )); + AND name NOT IN ( + '%completion:started%', + '%completion:suggested%', + '%cta%', + '%Cta%', + 'CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA' + ); END; $$; diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 1153abb91ce3..66bae0427356 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -457,26 +457,25 @@ CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - (( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA']) - )); + AND name NOT IN ( + '%completion:started%', + '%completion:suggested%', + '%cta%', + '%Cta%', + 'CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA' + ); END; $$; From 2e94fe74b3ae9f5b59a1ef01681238f9e1f24ed3 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:57:43 -0700 Subject: [PATCH 35/77] internal/event_logs: backward compatibility fixes --- .../1691043630_event_logs_indexing/down.sql | 12 ++++++------ .../frontend/1691043630_event_logs_indexing/up.sql | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 57ec8a573978..66b5d2622430 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,9 +1,3 @@ -DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text) - -DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text) - -DROP FUNCTION IF EXISTS isCodyActiveEvent(name text) - DROP INDEX IF EXISTS event_logs_name; DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; @@ -11,3 +5,9 @@ DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event; DROP INDEX IF EXISTS event_logs_name_is_cody_active_event; + +DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text); + +DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text); + +DROP FUNCTION IF EXISTS isCodyActiveEvent(name text); diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d28a06b9f93c..574c3f3980c7 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,4 +1,4 @@ -CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean +CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -26,7 +26,7 @@ BEGIN END; $$; -CREATE FUNCTION IF NOT EXISTS iscodyexplanationevent(name text) RETURNS boolean +CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -40,7 +40,7 @@ BEGIN END; $$; -CREATE FUNCTION IF NOT EXISTS iscodygenerationevent(name text) RETURNS boolean +CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +58,10 @@ BEGIN END; $$; -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 0836a0b666e46cca4b0c25c94c2aef02236d7f9f Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:06:40 -0700 Subject: [PATCH 36/77] internal/event_logs: lint format fix --- internal/database/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index f66fa5c8992f..c20cd2632228 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -140,7 +140,7 @@ }, { "Name": "iscodyactiveevent", - "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n ((\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'])));\nEND;\n$function$\n" + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND name NOT IN (\n '%completion:started%',\n '%completion:suggested%',\n '%cta%',\n '%Cta%',\n 'CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'\n );\nEND;\n$function$\n" }, { "Name": "iscodyexplanationevent", From e582b18b5d12a24481829c2ab3176dbf62a6855f Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:47:28 -0700 Subject: [PATCH 37/77] internal/event_logs: make functions idempotent --- .../frontend/1691043630_event_logs_indexing/up.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 574c3f3980c7..16bd496e23ea 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,4 +1,4 @@ -CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -26,7 +26,7 @@ BEGIN END; $$; -CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodyexplanationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -40,7 +40,7 @@ BEGIN END; $$; -CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodygenerationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +58,10 @@ BEGIN END; $$; -CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 0be3cb5f58c4b2aa7f240f6521b0faf7aeb6d009 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:33:55 -0700 Subject: [PATCH 38/77] internal/event_logs: optimize ping cte The aggregatedCodyUsageEventsQuery is timing out for dotcom. These changes are cte optimization steps. --- internal/database/event_logs.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index cc0229b67d93..a485ed470ee0 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1592,7 +1592,16 @@ var nonActiveCodyEvents = []string{ } var aggregatedCodyUsageEventsQuery = ` -WITH events AS ( +WITH +sub_cohort AS ( + SELECT * + FROM event_logs + WHERE + timestamp >= %s::timestamp - '1 month'::interval + AND name not like '%%completion:started%%' + AND name not like '%%completion:suggested%%' +), +events AS ( SELECT name AS key, ` + aggregatedUserIDQueryFragment + ` AS user_id, @@ -1604,8 +1613,7 @@ WITH events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - timestamp >= ` + makeDateTruncExpression("month", "%s::timestamp") + ` - AND lower(name) like '%%cody%%' + lower(name) like '%%cody%%' AND name not like '%%CTA%%' AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) From 0f4a5d2b1acace50438ced84477720b245c0b429 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 2 Aug 2023 23:25:45 -0700 Subject: [PATCH 39/77] internal/event_logs: event_logs indexing Add indexes to event_logs table to increase query performance. --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 3 +++ .../frontend/1691043630_event_logs_indexing/metadata.yaml | 2 ++ migrations/frontend/1691043630_event_logs_indexing/up.sql | 3 +++ 3 files changed, 8 insertions(+) create mode 100644 migrations/frontend/1691043630_event_logs_indexing/down.sql create mode 100644 migrations/frontend/1691043630_event_logs_indexing/metadata.yaml create mode 100644 migrations/frontend/1691043630_event_logs_indexing/up.sql diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql new file mode 100644 index 000000000000..4beed28c7fc8 --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -0,0 +1,3 @@ +DROP INDEX event_logs_name; + +DROP INDEX event_logs_name; diff --git a/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml b/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml new file mode 100644 index 000000000000..0c01d8abf90c --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/metadata.yaml @@ -0,0 +1,2 @@ +name: event_logs_indexing +parents: [1690460411, 1690323910] diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql new file mode 100644 index 000000000000..bf55394693af --- /dev/null +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -0,0 +1,3 @@ +CREATE INDEX event_logs_name ON event_logs (name); + +CREATE INDEX event_logs_timestamp ON event_logs (timestamp); From 1aa531831ef8014dedd2d600f3e4304916800bae Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:26:50 -0700 Subject: [PATCH 40/77] internal/event_logs: sg lint --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 4 ++-- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 4beed28c7fc8..4ad6ae8b3597 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,3 @@ -DROP INDEX event_logs_name; +DROP INDEX IF EXISTS event_logs_name; -DROP INDEX event_logs_name; +DROP INDEX IF EXISTS event_logs_name; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index bf55394693af..d7915c8d69c5 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ -CREATE INDEX event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); -CREATE INDEX event_logs_timestamp ON event_logs (timestamp); +CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs (timestamp); From fed4db6ea32527ec3c34cbb6dc44f0485c8e9de6 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:11:22 -0700 Subject: [PATCH 41/77] internal/event_logs: sg lint fix --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 4ad6ae8b3597..12c4ac06d459 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,3 @@ DROP INDEX IF EXISTS event_logs_name; -DROP INDEX IF EXISTS event_logs_name; +DROP INDEX IF EXISTS event_logs_timestamp; From de432536827fb7ded038259536a4b9c848156e2d Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:14:36 -0700 Subject: [PATCH 42/77] internal/event_logs: bazel configure --- dev/linters/bodyclose/BUILD.bazel | 5 +---- dev/linters/depguard/BUILD.bazel | 1 - dev/linters/exportloopref/BUILD.bazel | 5 +---- dev/linters/gocheckcompilerdirectives/BUILD.bazel | 5 +---- dev/linters/unparam/BUILD.bazel | 1 - migrations/BUILD.bazel | 3 +++ 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/dev/linters/bodyclose/BUILD.bazel b/dev/linters/bodyclose/BUILD.bazel index 93228f234266..9d6c52923f38 100644 --- a/dev/linters/bodyclose/BUILD.bazel +++ b/dev/linters/bodyclose/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["bodyclose.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/bodyclose", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_github_timakin_bodyclose//passes/bodyclose:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/depguard/BUILD.bazel b/dev/linters/depguard/BUILD.bazel index 7b5fca9789e0..6cfb17e89c98 100644 --- a/dev/linters/depguard/BUILD.bazel +++ b/dev/linters/depguard/BUILD.bazel @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", - "@com_github_openpeedeep_depguard_v2//:go_default_library", "@org_golang_x_tools//go/analysis", ], ) diff --git a/dev/linters/exportloopref/BUILD.bazel b/dev/linters/exportloopref/BUILD.bazel index a49fce597026..cfab09d40786 100644 --- a/dev/linters/exportloopref/BUILD.bazel +++ b/dev/linters/exportloopref/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["exportloopref.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/exportloopref", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_github_kyoh86_exportloopref//:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/gocheckcompilerdirectives/BUILD.bazel b/dev/linters/gocheckcompilerdirectives/BUILD.bazel index 0544019392fb..0650219f5263 100644 --- a/dev/linters/gocheckcompilerdirectives/BUILD.bazel +++ b/dev/linters/gocheckcompilerdirectives/BUILD.bazel @@ -5,8 +5,5 @@ go_library( srcs = ["gocheckcompilerdirectives.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/gocheckcompilerdirectives", visibility = ["//visibility:public"], - deps = [ - "//dev/linters/nolint", - "@com_4d63_gocheckcompilerdirectives//checkcompilerdirectives:go_default_library", - ], + deps = ["//dev/linters/nolint"], ) diff --git a/dev/linters/unparam/BUILD.bazel b/dev/linters/unparam/BUILD.bazel index 39aaebe7497f..becc863acebd 100644 --- a/dev/linters/unparam/BUILD.bazel +++ b/dev/linters/unparam/BUILD.bazel @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", - "@cc_mvdan_unparam//check:go_default_library", "@org_golang_x_tools//go/analysis", "@org_golang_x_tools//go/analysis/passes/buildssa", "@org_golang_x_tools//go/packages", diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 3cbf784adc96..86bcadae3fff 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,6 +1114,9 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", + "frontend/1691043630_event_logs_indexing/down.sql", + "frontend/1691043630_event_logs_indexing/metadata.yaml", + "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From fe2528b29a95bb282122f785dcc73d801014ecb3 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:22:22 -0700 Subject: [PATCH 43/77] Revert "internal/event_logs: bazel configure" This reverts commit 47965ec20ca2a8cd1aba35bf1736d00ae61cd293. --- dev/linters/bodyclose/BUILD.bazel | 5 ++++- dev/linters/depguard/BUILD.bazel | 1 + dev/linters/exportloopref/BUILD.bazel | 5 ++++- dev/linters/gocheckcompilerdirectives/BUILD.bazel | 5 ++++- dev/linters/unparam/BUILD.bazel | 1 + migrations/BUILD.bazel | 3 --- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dev/linters/bodyclose/BUILD.bazel b/dev/linters/bodyclose/BUILD.bazel index 9d6c52923f38..93228f234266 100644 --- a/dev/linters/bodyclose/BUILD.bazel +++ b/dev/linters/bodyclose/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["bodyclose.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/bodyclose", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_github_timakin_bodyclose//passes/bodyclose:go_default_library", + ], ) diff --git a/dev/linters/depguard/BUILD.bazel b/dev/linters/depguard/BUILD.bazel index 6cfb17e89c98..7b5fca9789e0 100644 --- a/dev/linters/depguard/BUILD.bazel +++ b/dev/linters/depguard/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", + "@com_github_openpeedeep_depguard_v2//:go_default_library", "@org_golang_x_tools//go/analysis", ], ) diff --git a/dev/linters/exportloopref/BUILD.bazel b/dev/linters/exportloopref/BUILD.bazel index cfab09d40786..a49fce597026 100644 --- a/dev/linters/exportloopref/BUILD.bazel +++ b/dev/linters/exportloopref/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["exportloopref.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/exportloopref", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_github_kyoh86_exportloopref//:go_default_library", + ], ) diff --git a/dev/linters/gocheckcompilerdirectives/BUILD.bazel b/dev/linters/gocheckcompilerdirectives/BUILD.bazel index 0650219f5263..0544019392fb 100644 --- a/dev/linters/gocheckcompilerdirectives/BUILD.bazel +++ b/dev/linters/gocheckcompilerdirectives/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["gocheckcompilerdirectives.go"], importpath = "github.com/sourcegraph/sourcegraph/dev/linters/gocheckcompilerdirectives", visibility = ["//visibility:public"], - deps = ["//dev/linters/nolint"], + deps = [ + "//dev/linters/nolint", + "@com_4d63_gocheckcompilerdirectives//checkcompilerdirectives:go_default_library", + ], ) diff --git a/dev/linters/unparam/BUILD.bazel b/dev/linters/unparam/BUILD.bazel index becc863acebd..39aaebe7497f 100644 --- a/dev/linters/unparam/BUILD.bazel +++ b/dev/linters/unparam/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dev/linters/nolint", + "@cc_mvdan_unparam//check:go_default_library", "@org_golang_x_tools//go/analysis", "@org_golang_x_tools//go/analysis/passes/buildssa", "@org_golang_x_tools//go/packages", diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 86bcadae3fff..3cbf784adc96 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,9 +1114,6 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", - "frontend/1691043630_event_logs_indexing/down.sql", - "frontend/1691043630_event_logs_indexing/metadata.yaml", - "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From 708eb7e974495c176f8d834ebee5ac6de0c06a02 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:58:01 -0700 Subject: [PATCH 44/77] internal/event_logs: schema definition update --- internal/database/schema.json | 10 ++++++++++ internal/database/schema.md | 1 + migrations/frontend/squashed.sql | 2 ++ 3 files changed, 13 insertions(+) diff --git a/internal/database/schema.json b/internal/database/schema.json index 6684115b5a16..fbf06372c8af 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -10595,6 +10595,16 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING btree (name)", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 0d939903eded..a885939ee210 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name" btree (name) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index c0d8590259ae..a2b665f73099 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,6 +5866,8 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); +CREATE INDEX event_logs_name ON event_logs USING btree (name); + CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); CREATE INDEX event_logs_source ON event_logs USING btree (source); From a1049b43998e6e1d05698535f0c4d2c5f74d7be1 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:00:45 -0700 Subject: [PATCH 45/77] internal/event_logs: schema definition update --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index fbf06372c8af..2f45caddfbf5 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -28567,4 +28567,4 @@ "Definition": " SELECT changeset_specs.id AS changeset_spec_id,\n COALESCE(changesets.id, (0)::bigint) AS changeset_id,\n changeset_specs.repo_id,\n changeset_specs.batch_spec_id,\n repo.name AS repo_name,\n COALESCE((changesets.metadata -\u003e\u003e 'Title'::text), (changesets.metadata -\u003e\u003e 'title'::text)) AS changeset_name,\n changesets.external_state,\n changesets.publication_state,\n changesets.reconciler_state,\n changesets.computed_state\n FROM ((changeset_specs\n LEFT JOIN changesets ON (((changesets.repo_id = changeset_specs.repo_id) AND (changesets.external_id = changeset_specs.external_id))))\n JOIN repo ON ((changeset_specs.repo_id = repo.id)))\n WHERE ((changeset_specs.external_id IS NOT NULL) AND (repo.deleted_at IS NULL));" } ] -} \ No newline at end of file +} diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index a2b665f73099..d01a6b464a56 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file +SELECT pg_catalog.setval('roles_id_seq', 3, true); From 74dd6720a563e228f8d188becaaf1fc95a03255c Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:06:55 -0700 Subject: [PATCH 46/77] internal/event_logs: bazel update --- migrations/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 3cbf784adc96..86bcadae3fff 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1114,6 +1114,9 @@ go_library( "frontend/1690323910_add_chunks_excluded_embeddings_stats/down.sql", "frontend/1690323910_add_chunks_excluded_embeddings_stats/metadata.yaml", "frontend/1690323910_add_chunks_excluded_embeddings_stats/up.sql", + "frontend/1691043630_event_logs_indexing/down.sql", + "frontend/1691043630_event_logs_indexing/metadata.yaml", + "frontend/1691043630_event_logs_indexing/up.sql", ], importpath = "github.com/sourcegraph/sourcegraph/migrations", visibility = ["//visibility:public"], From fb46682ca2329396df9aa8a49a9641f862f98353 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:37:23 -0700 Subject: [PATCH 47/77] internal/event_logs: manual lint fixes From d16451edaa001663bbf5ae6ac4d3962ae953815a Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:38:58 -0700 Subject: [PATCH 48/77] internal/event_logs: manual lint fixes #2 From ed8e9a494ecc026b73ac09c7837dc8ced8272b77 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:54:28 -0700 Subject: [PATCH 49/77] internal/event_logs: revert lint breaking commit --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index 2f45caddfbf5..fbf06372c8af 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -28567,4 +28567,4 @@ "Definition": " SELECT changeset_specs.id AS changeset_spec_id,\n COALESCE(changesets.id, (0)::bigint) AS changeset_id,\n changeset_specs.repo_id,\n changeset_specs.batch_spec_id,\n repo.name AS repo_name,\n COALESCE((changesets.metadata -\u003e\u003e 'Title'::text), (changesets.metadata -\u003e\u003e 'title'::text)) AS changeset_name,\n changesets.external_state,\n changesets.publication_state,\n changesets.reconciler_state,\n changesets.computed_state\n FROM ((changeset_specs\n LEFT JOIN changesets ON (((changesets.repo_id = changeset_specs.repo_id) AND (changesets.external_id = changeset_specs.external_id))))\n JOIN repo ON ((changeset_specs.repo_id = repo.id)))\n WHERE ((changeset_specs.external_id IS NOT NULL) AND (repo.deleted_at IS NULL));" } ] -} +} \ No newline at end of file diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index d01a6b464a56..a2b665f73099 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); +SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file From 423b1e6b01c87ce5d9d907684dacc82f95f2ba8e Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:01:59 -0700 Subject: [PATCH 50/77] internal/event_logs: backcompat test drift fix --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d7915c8d69c5..f34215b0efed 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); -CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs (timestamp); +CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From 2c5e61d9eeefcd6ea73e6aad6b0b2b60d36b96fc Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:38:07 -0700 Subject: [PATCH 51/77] internal/event_logs: update name index to lower --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index f34215b0efed..07f55ef5df70 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,3 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (lower(name)); CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From 127af7366000e263e1013549727b68de310fae71 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:40:34 -0700 Subject: [PATCH 52/77] internal/event_logs: event log indexs --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 2 ++ migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 12c4ac06d459..b4694a825916 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,3 +1,5 @@ DROP INDEX IF EXISTS event_logs_name; +DROP INDEX IF EXISTS event_logs_name_lower; + DROP INDEX IF EXISTS event_logs_timestamp; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 07f55ef5df70..18aff6f7e33e 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,3 +1,5 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (lower(name)); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); + +CREATE INDEX IF NOT EXISTS event_logs_name_lower ON event_logs (lower(name)); CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From d3c1db2c3b68184e9216f061510e6b7ae3e75a04 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:54:56 -0700 Subject: [PATCH 53/77] internal/event_logs: remove lower index It's computationally expensive to have an index on both the column and the lower(column) simuiltaneously. We need the index on the column so we're going to take the computational hit in the where clause. There is already timestamp, removing redundant migration. --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 18aff6f7e33e..7268d19f0685 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,5 +1 @@ CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); - -CREATE INDEX IF NOT EXISTS event_logs_name_lower ON event_logs (lower(name)); - -CREATE INDEX IF NOT EXISTS event_logs_timestamp ON event_logs USING btree ("timestamp"); From d40d1bfb9ba8f2e1e4d0058af0fb998d543ab49f Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:02:28 -0700 Subject: [PATCH 54/77] Update down.sql --- migrations/frontend/1691043630_event_logs_indexing/down.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index b4694a825916..52bc712a47ab 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,5 +1 @@ DROP INDEX IF EXISTS event_logs_name; - -DROP INDEX IF EXISTS event_logs_name_lower; - -DROP INDEX IF EXISTS event_logs_timestamp; From 0d52fbba773c049d9751c9c481322f33d32442d8 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:35:07 -0700 Subject: [PATCH 55/77] Update event_logs.go --- internal/database/event_logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index a485ed470ee0..25dceb6f9bed 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1613,7 +1613,7 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - lower(name) like '%%cody%%' + (name like '%%cody%%' or name like '%%Cody%%') AND name not like '%%CTA%%' AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) From 2855d83d9c25124d031970c0e063ed5346f857e2 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:58:57 -0700 Subject: [PATCH 56/77] internal/event_logs: add GIN index A GIN index should improve performance of text searches. --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 2 +- migrations/frontend/squashed.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 7268d19f0685..d22d4a985dec 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1 +1 @@ -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs (name); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index a2b665f73099..010ab6c5249e 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,7 +5866,7 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); -CREATE INDEX event_logs_name ON event_logs USING btree (name); +CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file +SELECT pg_catalog.setval('roles_id_seq', 3, true); From 91b9bea724d0f57a6c3ba7dcd3dc532ac436145b Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Mon, 7 Aug 2023 16:41:41 -0700 Subject: [PATCH 57/77] internal/event_logs: add GIN index Additional schema and migration infrastructure. --- internal/database/schema.json | 2 +- migrations/frontend/squashed.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index fbf06372c8af..1eada8719b31 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -10601,7 +10601,7 @@ "IsUnique": false, "IsExclusion": false, "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING btree (name)", + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", "ConstraintType": "", "ConstraintDefinition": "" }, diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 010ab6c5249e..df928458e8d0 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5866,7 +5866,7 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); -CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); @@ -6869,4 +6869,4 @@ SELECT pg_catalog.setval('lsif_configuration_policies_id_seq', 3, true); INSERT INTO roles VALUES (1, '2023-01-04 16:29:41.195966+00', true, 'USER'); INSERT INTO roles VALUES (2, '2023-01-04 16:29:41.195966+00', true, 'SITE_ADMINISTRATOR'); -SELECT pg_catalog.setval('roles_id_seq', 3, true); +SELECT pg_catalog.setval('roles_id_seq', 3, true); \ No newline at end of file From 2e903194113d50c684ca2e401fa10198b06f8ad7 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:57:09 -0700 Subject: [PATCH 58/77] internal/event_logs: query efficiency changes It's more efficient to perform case-insensitive comparisons using ilike instead of like. --- internal/database/event_logs.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index 25dceb6f9bed..e5a5102e2ae7 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1598,8 +1598,8 @@ sub_cohort AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND name not like '%%completion:started%%' - AND name not like '%%completion:suggested%%' + AND name not ilike '%%completion:started%%' + AND name not ilike '%%completion:suggested%%' ), events AS ( SELECT @@ -1613,9 +1613,8 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - (name like '%%cody%%' or name like '%%Cody%%') - AND name not like '%%CTA%%' - AND name not like '%%Cta%%' + name ilike '%%cody%%' + AND name not ilike '%%cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) ), code_generation_keys AS ( From e197a41566ca7c56fd9cb9462c6e423086b4e4c3 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:58:27 -0700 Subject: [PATCH 59/77] internal/event_logs: sql logic and lint fixes --- internal/database/event_logs.go | 16 ++++++---------- internal/database/schema.md | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index e5a5102e2ae7..e927b274f1d3 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1593,14 +1593,6 @@ var nonActiveCodyEvents = []string{ var aggregatedCodyUsageEventsQuery = ` WITH -sub_cohort AS ( - SELECT * - FROM event_logs - WHERE - timestamp >= %s::timestamp - '1 month'::interval - AND name not ilike '%%completion:started%%' - AND name not ilike '%%completion:suggested%%' -), events AS ( SELECT name AS key, @@ -1613,8 +1605,12 @@ events AS ( ` + makeDateTruncExpression("day", "%s::timestamp") + ` as current_day FROM event_logs WHERE - name ilike '%%cody%%' - AND name not ilike '%%cta%%' + timestamp >= %s::timestamp - '1 month'::interval + AND name not ilike '%%completion:started%%' + AND name not ilike '%%completion:suggested%%' + AND name ilike '%%cody%%' + AND name not like '%%cta%%' + AND name not like '%%Cta%%' AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) ), code_generation_keys AS ( diff --git a/internal/database/schema.md b/internal/database/schema.md index a885939ee210..559e47adbd3a 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,7 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) - "event_logs_name" btree (name) + "event_logs_name" gin (name gin_trgm_ops) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") From 9fb60bee157e3d9bebc49cfae6e8d285201351ac Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:38:38 -0700 Subject: [PATCH 60/77] internal/event_logs: add lookup fuctions Add lookup functions to categorize and filter event logs and aid performance. --- internal/database/schema.json | 42 ++++++++++++ internal/database/schema.md | 3 + .../1691043630_event_logs_indexing/down.sql | 12 ++++ .../1691043630_event_logs_indexing/up.sql | 63 ++++++++++++++++++ migrations/frontend/squashed.sql | 66 ++++++++++++++++++- 5 files changed, 185 insertions(+), 1 deletion(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index 1eada8719b31..3a5f5a5b4749 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -201,6 +201,18 @@ { "Name": "versions_insert_row_trigger", "Definition": "CREATE OR REPLACE FUNCTION public.versions_insert_row_trigger()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\nBEGIN\n NEW.first_version = NEW.version;\n RETURN NEW;\nEND $function$\n" + }, + { + "Name": "isCodyGenerationEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyGenerationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" + }, + { + "Name": "isCodyExplanationEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyExplanationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" + }, + { + "Name": "isCodyActiveEvent", + "Definition": "CREATE OR REPLACE FUNCTION public.isCodyActiveEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" } ], "Sequences": [ @@ -10605,6 +10617,36 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name_is_cody_explanation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_generation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_active_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 559e47adbd3a..34a739ed2053 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1380,6 +1380,9 @@ Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) "event_logs_name" gin (name gin_trgm_ops) + "event_logs_name_is_cody_explanation_event" isCodyExplanationEvent(name) + "event_logs_name_is_cody_generation_event" isCodyGenerationEvent(name) + "event_logs_name_is_cody_active_event" isCodyActiveEvent(name) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 52bc712a47ab..57ec8a573978 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1 +1,13 @@ +DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text) + +DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text) + +DROP FUNCTION IF EXISTS isCodyActiveEvent(name text) + DROP INDEX IF EXISTS event_logs_name; + +DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; + +DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event; + +DROP INDEX IF EXISTS event_logs_name_is_cody_active_event; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d22d4a985dec..db39785d2b62 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1 +1,64 @@ +CREATE FUNCTION isCodyGenerationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyExplanationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyActiveEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); + +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index df928458e8d0..85e972179c4a 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -865,7 +865,65 @@ CREATE FUNCTION versions_insert_row_trigger() RETURNS trigger BEGIN NEW.first_version = NEW.version; RETURN NEW; -END $$; +END +$$; + +CREATE FUNCTION isCodyGenerationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyExplanationEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE FUNCTION isCodyActiveEvent(name text) +RETURNS boolean AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$ LANGUAGE plpgsql IMMUTABLE; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, @@ -5868,6 +5926,12 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); + +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); + +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); + CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); CREATE INDEX event_logs_source ON event_logs USING btree (source); From e6990fb82cb62671725749b1b43e42f2ea493ad2 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:05:33 -0700 Subject: [PATCH 61/77] internal/event_logs: refactor cody usage Simplify the aggregatedCodyUsageEvents query to utilize added indexes to improve performance. --- internal/database/event_logs.go | 66 +++++---------------------------- internal/database/schema.json | 52 -------------------------- internal/database/schema.md | 5 --- 3 files changed, 10 insertions(+), 113 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index e927b274f1d3..fa5a0899931b 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1574,28 +1574,11 @@ func (l *eventLogStore) aggregatedSearchEvents(ctx context.Context, queryString return events, nil } -// List of events that don't meet the criteria of "active" usage of Cody. -var nonActiveCodyEvents = []string{ - "CodyVSCodeExtension:CodySavedLogin:executed", - "web:codyChat:tryOnPublicCode", - "web:codyEditorWidget:viewed", - "web:codyChat:pageViewed", - "CodyConfigurationPageViewed", - "ClickedOnTryCodySearchCTA", - "TryCodyWebOnboardingDisplayed", - "AboutGetCodyPopover", - "TryCodyWeb", - "CodySurveyToastViewed", - "SiteAdminCodyPageViewed", - "CodyUninstalled", - "SpeakToACodyEngineerCTA", -} - var aggregatedCodyUsageEventsQuery = ` WITH events AS ( SELECT - name AS key, + name, ` + aggregatedUserIDQueryFragment + ` AS user_id, ` + makeDateTruncExpression("month", "timestamp") + ` as month, ` + makeDateTruncExpression("week", "timestamp") + ` as week, @@ -1606,37 +1589,10 @@ events AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND name not ilike '%%completion:started%%' - AND name not ilike '%%completion:suggested%%' - AND name ilike '%%cody%%' - AND name not like '%%cta%%' - AND name not like '%%Cta%%' - AND (name NOT IN ('` + strings.Join(nonActiveCodyEvents, "','") + `')) -), -code_generation_keys AS ( - SELECT * FROM unnest(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]) AS key -), -explanation_keys AS ( - SELECT * FROM unnest(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]) AS key + AND isCodyActiveEvent(name) ) SELECT - key, + name, current_month, current_week, current_day, @@ -1646,25 +1602,23 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and key in - (SELECT * FROM code_generation_keys) + SUM(case when month = current_month and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and key in - (SELECT * FROM explanation_keys) + SUM(case when week = current_week and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and key in (SELECT * FROM code_generation_keys) + SUM(case when day = current_day and isCodeGenerationEvent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and key in (SELECT * FROM explanation_keys) + SUM(case when month = current_month and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and key in (SELECT * FROM explanation_keys) + SUM(case when week = current_week and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and key in (SELECT * FROM explanation_keys) + SUM(case when day = current_day and isCodeExplanationEvent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, 0 as invalid_day FROM events -GROUP BY key, current_month, current_week, current_day +GROUP BY name, current_month, current_week, current_day ` var searchLatencyEventNames = []string{ diff --git a/internal/database/schema.json b/internal/database/schema.json index 3a5f5a5b4749..6684115b5a16 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -201,18 +201,6 @@ { "Name": "versions_insert_row_trigger", "Definition": "CREATE OR REPLACE FUNCTION public.versions_insert_row_trigger()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\nBEGIN\n NEW.first_version = NEW.version;\n RETURN NEW;\nEND $function$\n" - }, - { - "Name": "isCodyGenerationEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyGenerationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" - }, - { - "Name": "isCodyExplanationEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyExplanationEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" - }, - { - "Name": "isCodyActiveEvent", - "Definition": "CREATE OR REPLACE FUNCTION public.isCodyActiveEvent(name text)\nRETURNS boolean AS $$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$$ LANGUAGE plpgsql IMMUTABLE;" } ], "Sequences": [ @@ -10607,46 +10595,6 @@ "ConstraintType": "", "ConstraintDefinition": "" }, - { - "Name": "event_logs_name", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_explanation_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_generation_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, - { - "Name": "event_logs_name_is_cody_active_event", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name))", - "ConstraintType": "", - "ConstraintDefinition": "" - }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 34a739ed2053..fa7792bfc8b0 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,11 +1379,6 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) - "event_logs_name" gin (name gin_trgm_ops) - "event_logs_name_is_cody_explanation_event" isCodyExplanationEvent(name) - "event_logs_name_is_cody_generation_event" isCodyGenerationEvent(name) - "event_logs_name_is_cody_active_event" isCodyActiveEvent(name) - "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") "event_logs_timestamp_at_utc" btree (date(timezone('UTC'::text, "timestamp"))) From 034f84463e14d47ac2800614a459f6fd9874181a Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:06:55 -0700 Subject: [PATCH 62/77] internal/event_logs: revert deletion --- internal/database/schema.md | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/database/schema.md b/internal/database/schema.md index fa7792bfc8b0..0d939903eded 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,7 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") "event_logs_timestamp_at_utc" btree (date(timezone('UTC'::text, "timestamp"))) From a70cab508c1de4dbe9a781a0d320a26827de3c8c Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:24:55 -0700 Subject: [PATCH 63/77] internal/event_logs: lint fixes --- .../1691043630_event_logs_indexing/up.sql | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index db39785d2b62..607aa2de166a 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,5 +1,6 @@ -CREATE FUNCTION isCodyGenerationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', @@ -13,10 +14,11 @@ BEGIN 'CodyVSCodeExtension:recipe:translate-to-language:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyExplanationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', @@ -28,8 +30,9 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; -CREATE FUNCTION isCodyActiveEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') @@ -57,8 +60,8 @@ $$ LANGUAGE plpgsql IMMUTABLE; CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); From 59d07886493ef03d844933d9c72c126708aefe87 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 01:41:12 -0700 Subject: [PATCH 64/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 85e972179c4a..fb30387acbaf 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5926,11 +5926,11 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs (isCodyExplanationEvent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs (isCodyGenerationEvent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs (isCodyActiveEvent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); From 370f2a8987ddad3940c0f7e80442f1d91b82026f Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 03:49:23 -0700 Subject: [PATCH 65/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index fb30387acbaf..6d7289ecb8ed 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -868,8 +868,9 @@ BEGIN END $$; -CREATE FUNCTION isCodyGenerationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', @@ -883,10 +884,11 @@ BEGIN 'CodyVSCodeExtension:recipe:translate-to-language:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyExplanationEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ BEGIN RETURN name = ANY(ARRAY[ 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', @@ -896,10 +898,11 @@ BEGIN 'CodyVSCodeExtension:recipe:rate-code:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; -CREATE FUNCTION isCodyActiveEvent(name text) -RETURNS boolean AS $$ +CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean +LANGUAGE plpgsql IMMUTABLE +AS $$ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') @@ -923,7 +926,7 @@ BEGIN 'CodyUninstalled', 'SpeakToACodyEngineerCTA'])); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, From da85d83208153e9df41d0fb951fbd1be7fd9bcf2 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:03:43 -0700 Subject: [PATCH 66/77] internal/event_logs: lint fixes --- migrations/frontend/1691043630_event_logs_indexing/up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 607aa2de166a..57e55d35f111 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -28,7 +28,7 @@ BEGIN 'CodyVSCodeExtension:recipe:rate-code:executed' ]); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE @@ -56,7 +56,7 @@ BEGIN 'CodyUninstalled', 'SpeakToACodyEngineerCTA'])); END; -$$ LANGUAGE plpgsql IMMUTABLE; +$$; CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); From 8a02a4bf6f166dba57ee2e46d19078f79e6e130b Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:32:36 -0700 Subject: [PATCH 67/77] internal/event_logs: lint fixes --- internal/database/event_logs.go | 14 ++-- internal/database/schema.json | 52 +++++++++++++ internal/database/schema.md | 4 + migrations/frontend/squashed.sql | 126 +++++++++++++++---------------- 4 files changed, 126 insertions(+), 70 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index fa5a0899931b..1e7aab2015c5 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1589,7 +1589,7 @@ events AS ( FROM event_logs WHERE timestamp >= %s::timestamp - '1 month'::interval - AND isCodyActiveEvent(name) + AND iscodyactiveevent(name) ) SELECT name, @@ -1602,17 +1602,17 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and isCodeGenerationEvent(name) + SUM(case when month = current_month and iscodegenerationevent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and isCodeGenerationEvent(name) + SUM(case when week = current_week and iscodegenerationevent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and isCodeGenerationEvent(name) + SUM(case when day = current_day and iscodegenerationevent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and isCodeExplanationEvent(name) + SUM(case when month = current_month and iscodeexplanationevent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and isCodeExplanationEvent(name) + SUM(case when week = current_week and iscodeexplanationevent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and isCodeExplanationEvent(name) + SUM(case when day = current_day and iscodeexplanationevent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, diff --git a/internal/database/schema.json b/internal/database/schema.json index 6684115b5a16..b59e884a0315 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -138,6 +138,18 @@ "Name": "invalidate_session_for_userid_on_password_change", "Definition": "CREATE OR REPLACE FUNCTION public.invalidate_session_for_userid_on_password_change()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\n BEGIN\n IF OLD.passwd != NEW.passwd THEN\n NEW.invalidated_sessions_at = now() + (1 * interval '1 second');\n RETURN NEW;\n END IF;\n RETURN NEW;\n END;\n$function$\n" }, + { + "Name": "iscodyactiveevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$function$\n" + }, + { + "Name": "iscodyexplanationevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyexplanationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:explain-code-high-level:executed',\n 'CodyVSCodeExtension:recipe:explain-code-detailed:executed',\n 'CodyVSCodeExtension:recipe:find-code-smells:executed',\n 'CodyVSCodeExtension:recipe:git-history:executed',\n 'CodyVSCodeExtension:recipe:rate-code:executed'\n ]);\nEND;\n$function$\n" + }, + { + "Name": "iscodygenerationevent", + "Definition": "CREATE OR REPLACE FUNCTION public.iscodygenerationevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN name = ANY(ARRAY[\n 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed',\n 'CodyVSCodeExtension:recipe:improve-variable-names:executed',\n 'CodyVSCodeExtension:recipe:replace:executed',\n 'CodyVSCodeExtension:recipe:generate-docstring:executed',\n 'CodyVSCodeExtension:recipe:generate-unit-test:executed',\n 'CodyVSCodeExtension:recipe:rewrite-functional:executed',\n 'CodyVSCodeExtension:recipe:code-refactor:executed',\n 'CodyVSCodeExtension:recipe:fixup:executed',\n\t'CodyVSCodeExtension:recipe:translate-to-language:executed'\n ]);\nEND;\n$function$\n" + }, { "Name": "merge_audit_log_transitions", "Definition": "CREATE OR REPLACE FUNCTION public.merge_audit_log_transitions(internal hstore, arrayhstore hstore[])\n RETURNS hstore\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\n DECLARE\n trans hstore;\n BEGIN\n FOREACH trans IN ARRAY arrayhstore\n LOOP\n internal := internal || hstore(trans-\u003e'column', trans-\u003e'new');\n END LOOP;\n\n RETURN internal;\n END;\n$function$\n" @@ -10595,6 +10607,46 @@ "ConstraintType": "", "ConstraintDefinition": "" }, + { + "Name": "event_logs_name", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_active_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_explanation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, + { + "Name": "event_logs_name_is_cody_generation_event", + "IsPrimaryKey": false, + "IsUnique": false, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name))", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "event_logs_name_timestamp", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index 0d939903eded..ab1d86852af8 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,6 +1379,10 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) + "event_logs_name" gin (name gin_trgm_ops) + "event_logs_name_is_cody_active_event" btree (iscodyactiveevent(name)) + "event_logs_name_is_cody_explanation_event" btree (iscodyexplanationevent(name)) + "event_logs_name_is_cody_generation_event" btree (iscodygenerationevent(name)) "event_logs_name_timestamp" btree (name, "timestamp" DESC) "event_logs_source" btree (source) "event_logs_timestamp" btree ("timestamp") diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 6d7289ecb8ed..450a3f91c3f6 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -451,6 +451,66 @@ CREATE FUNCTION invalidate_session_for_userid_on_password_change() RETURNS trigg END; $$; +CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN + (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') + AND NOT + ( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA'])); +END; +$$; + +CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$; + +CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$; + CREATE FUNCTION merge_audit_log_transitions(internal hstore, arrayhstore hstore[]) RETURNS hstore LANGUAGE plpgsql IMMUTABLE AS $$ @@ -868,66 +928,6 @@ BEGIN END $$; -CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean -LANGUAGE plpgsql IMMUTABLE -AS $$ -BEGIN - RETURN - (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); -END; -$$; - CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, STYPE = hstore, @@ -5929,11 +5929,11 @@ CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_exp CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); CREATE INDEX event_logs_name_timestamp ON event_logs USING btree (name, "timestamp" DESC); From c982b5c0c4394d57c88ffcfa6a03eaccfbb705b6 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:47:25 -0700 Subject: [PATCH 68/77] internal/event_logs: lint fixes --- migrations/frontend/squashed.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 450a3f91c3f6..0a652e92e0af 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -925,8 +925,7 @@ CREATE FUNCTION versions_insert_row_trigger() RETURNS trigger BEGIN NEW.first_version = NEW.version; RETURN NEW; -END -$$; +END $$; CREATE AGGREGATE snapshot_transition_columns(hstore[]) ( SFUNC = merge_audit_log_transitions, From 6dd80b279e26444d88e63492d1f97d4536516fd5 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 05:14:39 -0700 Subject: [PATCH 69/77] internal/event_logs: typo fixes --- internal/database/event_logs.go | 12 ++-- .../1691043630_event_logs_indexing/up.sql | 72 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index 1e7aab2015c5..42c4ac1bb40d 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -1602,17 +1602,17 @@ SELECT COUNT(DISTINCT user_id) FILTER (WHERE month = current_month) AS uniques_month, COUNT(DISTINCT user_id) FILTER (WHERE week = current_week) AS uniques_week, COUNT(DISTINCT user_id) FILTER (WHERE day = current_day) AS uniques_day, - SUM(case when month = current_month and iscodegenerationevent(name) + SUM(case when month = current_month and iscodygenerationevent(name) then 1 else 0 end) as code_generation_month, - SUM(case when week = current_week and iscodegenerationevent(name) + SUM(case when week = current_week and iscodygenerationevent(name) then 1 else 0 end) as code_generation_week, - SUM(case when day = current_day and iscodegenerationevent(name) + SUM(case when day = current_day and iscodygenerationevent(name) then 1 else 0 end) as code_generation_day, - SUM(case when month = current_month and iscodeexplanationevent(name) + SUM(case when month = current_month and iscodyexplanationevent(name) then 1 else 0 end) as explanation_month, - SUM(case when week = current_week and iscodeexplanationevent(name) + SUM(case when week = current_week and iscodyexplanationevent(name) then 1 else 0 end) as explanation_week, - SUM(case when day = current_day and iscodeexplanationevent(name) + SUM(case when day = current_day and iscodyexplanationevent(name) then 1 else 0 end) as explanation_day, 0 as invalid_month, 0 as invalid_week, diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 57e55d35f111..3a8855d4bf84 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,36 +1,4 @@ -CREATE FUNCTION isCodyGenerationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', - 'CodyVSCodeExtension:recipe:improve-variable-names:executed', - 'CodyVSCodeExtension:recipe:replace:executed', - 'CodyVSCodeExtension:recipe:generate-docstring:executed', - 'CodyVSCodeExtension:recipe:generate-unit-test:executed', - 'CodyVSCodeExtension:recipe:rewrite-functional:executed', - 'CodyVSCodeExtension:recipe:code-refactor:executed', - 'CodyVSCodeExtension:recipe:fixup:executed', - 'CodyVSCodeExtension:recipe:translate-to-language:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyExplanationEvent(name text) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ -BEGIN - RETURN name = ANY(ARRAY[ - 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', - 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', - 'CodyVSCodeExtension:recipe:find-code-smells:executed', - 'CodyVSCodeExtension:recipe:git-history:executed', - 'CodyVSCodeExtension:recipe:rate-code:executed' - ]); -END; -$$; - -CREATE FUNCTION isCodyActiveEvent(name text) RETURNS boolean +CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +26,42 @@ BEGIN END; $$; +CREATE FUNCTION IF NOT EXISTS iscodyexplanationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:explain-code-high-level:executed', + 'CodyVSCodeExtension:recipe:explain-code-detailed:executed', + 'CodyVSCodeExtension:recipe:find-code-smells:executed', + 'CodyVSCodeExtension:recipe:git-history:executed', + 'CodyVSCodeExtension:recipe:rate-code:executed' + ]); +END; +$$; + +CREATE FUNCTION IF NOT EXISTS iscodygenerationevent(name text) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + RETURN name = ANY(ARRAY[ + 'CodyVSCodeExtension:recipe:rewrite-to-functional:executed', + 'CodyVSCodeExtension:recipe:improve-variable-names:executed', + 'CodyVSCodeExtension:recipe:replace:executed', + 'CodyVSCodeExtension:recipe:generate-docstring:executed', + 'CodyVSCodeExtension:recipe:generate-unit-test:executed', + 'CodyVSCodeExtension:recipe:rewrite-functional:executed', + 'CodyVSCodeExtension:recipe:code-refactor:executed', + 'CodyVSCodeExtension:recipe:fixup:executed', + 'CodyVSCodeExtension:recipe:translate-to-language:executed' + ]); +END; +$$; + CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (isCodyExplanationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (isCodyGenerationEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (isCodyActiveEvent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 1ec016eb38281882512bdf04a59588fbf64ae4b9 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 10:32:17 -0700 Subject: [PATCH 70/77] internal/event_logs: bazel test fixes --- internal/database/schema.json | 2 +- .../1691043630_event_logs_indexing/up.sql | 37 ++++++++++--------- migrations/frontend/squashed.sql | 37 ++++++++++--------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index b59e884a0315..f66fa5c8992f 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -140,7 +140,7 @@ }, { "Name": "iscodyactiveevent", - "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n (\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA']));\nEND;\n$function$\n" + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n ((\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'])));\nEND;\n$function$\n" }, { "Name": "iscodyexplanationevent", diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 3a8855d4bf84..33692d6a2fb5 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -5,24 +5,25 @@ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); + (( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA']) + )); END; $$; diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 0a652e92e0af..1153abb91ce3 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -458,24 +458,25 @@ BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') AND NOT - ( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA'])); + (( + name LIKE '%%completion:started%%' OR + name LIKE '%%completion:suggested%%' OR + name LIKE '%%cta%%' OR + name LIKE '%%Cta%%' OR + name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA']) + )); END; $$; From ee134b0245c9e28b3a214d639786c74a9138ce34 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:15:09 -0700 Subject: [PATCH 71/77] internal/event_logs: bazel test fixes --- .../1691043630_event_logs_indexing/up.sql | 39 +++++++++---------- migrations/frontend/squashed.sql | 39 +++++++++---------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 33692d6a2fb5..d28a06b9f93c 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -4,26 +4,25 @@ CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - (( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA']) - )); + AND name NOT IN ( + '%completion:started%', + '%completion:suggested%', + '%cta%', + '%Cta%', + 'CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA' + ); END; $$; diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 1153abb91ce3..66bae0427356 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -457,26 +457,25 @@ CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean BEGIN RETURN (name LIKE '%%cody%%' OR name LIKE '%%Cody%%') - AND NOT - (( - name LIKE '%%completion:started%%' OR - name LIKE '%%completion:suggested%%' OR - name LIKE '%%cta%%' OR - name LIKE '%%Cta%%' OR - name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed', - 'web:codyChat:tryOnPublicCode', - 'web:codyEditorWidget:viewed', - 'web:codyChat:pageViewed', - 'CodyConfigurationPageViewed', - 'ClickedOnTryCodySearchCTA', - 'TryCodyWebOnboardingDisplayed', - 'AboutGetCodyPopover', - 'TryCodyWeb', - 'CodySurveyToastViewed', - 'SiteAdminCodyPageViewed', - 'CodyUninstalled', - 'SpeakToACodyEngineerCTA']) - )); + AND name NOT IN ( + '%completion:started%', + '%completion:suggested%', + '%cta%', + '%Cta%', + 'CodyVSCodeExtension:CodySavedLogin:executed', + 'web:codyChat:tryOnPublicCode', + 'web:codyEditorWidget:viewed', + 'web:codyChat:pageViewed', + 'CodyConfigurationPageViewed', + 'ClickedOnTryCodySearchCTA', + 'TryCodyWebOnboardingDisplayed', + 'AboutGetCodyPopover', + 'TryCodyWeb', + 'CodySurveyToastViewed', + 'SiteAdminCodyPageViewed', + 'CodyUninstalled', + 'SpeakToACodyEngineerCTA' + ); END; $$; From a2328b561c5b4a1970684fe455c9d7f8104e7641 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:57:43 -0700 Subject: [PATCH 72/77] internal/event_logs: backward compatibility fixes --- .../1691043630_event_logs_indexing/down.sql | 12 ++++++------ .../frontend/1691043630_event_logs_indexing/up.sql | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 57ec8a573978..66b5d2622430 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,9 +1,3 @@ -DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text) - -DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text) - -DROP FUNCTION IF EXISTS isCodyActiveEvent(name text) - DROP INDEX IF EXISTS event_logs_name; DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; @@ -11,3 +5,9 @@ DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event; DROP INDEX IF EXISTS event_logs_name_is_cody_active_event; + +DROP FUNCTION IF EXISTS isCodyGenerationEvent(name text); + +DROP FUNCTION IF EXISTS isCodyExplanationEvent(name text); + +DROP FUNCTION IF EXISTS isCodyActiveEvent(name text); diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index d28a06b9f93c..574c3f3980c7 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,4 +1,4 @@ -CREATE FUNCTION IF NOT EXISTS iscodyactiveevent(name text) RETURNS boolean +CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -26,7 +26,7 @@ BEGIN END; $$; -CREATE FUNCTION IF NOT EXISTS iscodyexplanationevent(name text) RETURNS boolean +CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -40,7 +40,7 @@ BEGIN END; $$; -CREATE FUNCTION IF NOT EXISTS iscodygenerationevent(name text) RETURNS boolean +CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +58,10 @@ BEGIN END; $$; -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); +CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); +CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); +CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 210c0a16d1522c6ffae44c5f2bd265df50e64ecb Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:06:40 -0700 Subject: [PATCH 73/77] internal/event_logs: lint format fix --- internal/database/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index f66fa5c8992f..c20cd2632228 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -140,7 +140,7 @@ }, { "Name": "iscodyactiveevent", - "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND NOT\n ((\n name LIKE '%%completion:started%%' OR\n name LIKE '%%completion:suggested%%' OR\n name LIKE '%%cta%%' OR\n name LIKE '%%Cta%%' OR\n name = ANY(ARRAY['CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'])));\nEND;\n$function$\n" + "Definition": "CREATE OR REPLACE FUNCTION public.iscodyactiveevent(name text)\n RETURNS boolean\n LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nBEGIN\n RETURN\n (name LIKE '%%cody%%' OR name LIKE '%%Cody%%')\n AND name NOT IN (\n '%completion:started%',\n '%completion:suggested%',\n '%cta%',\n '%Cta%',\n 'CodyVSCodeExtension:CodySavedLogin:executed',\n 'web:codyChat:tryOnPublicCode',\n 'web:codyEditorWidget:viewed',\n 'web:codyChat:pageViewed',\n 'CodyConfigurationPageViewed',\n 'ClickedOnTryCodySearchCTA',\n 'TryCodyWebOnboardingDisplayed',\n 'AboutGetCodyPopover',\n 'TryCodyWeb',\n 'CodySurveyToastViewed',\n 'SiteAdminCodyPageViewed',\n 'CodyUninstalled',\n 'SpeakToACodyEngineerCTA'\n );\nEND;\n$function$\n" }, { "Name": "iscodyexplanationevent", From f99d2111b9af0eb6c3dbaa6fa9a0f42694692eb6 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:47:28 -0700 Subject: [PATCH 74/77] internal/event_logs: make functions idempotent --- .../frontend/1691043630_event_logs_indexing/up.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 574c3f3980c7..16bd496e23ea 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -1,4 +1,4 @@ -CREATE FUNCTION iscodyactiveevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodyactiveevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -26,7 +26,7 @@ BEGIN END; $$; -CREATE FUNCTION iscodyexplanationevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodyexplanationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -40,7 +40,7 @@ BEGIN END; $$; -CREATE FUNCTION iscodygenerationevent(name text) RETURNS boolean +CREATE OR REPLACE FUNCTION iscodygenerationevent(name text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $$ BEGIN @@ -58,10 +58,10 @@ BEGIN END; $$; -CREATE INDEX event_logs_name ON event_logs USING GIN (name gin_trgm_ops); +CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); -CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); -CREATE INDEX event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); -CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); +CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); From 893afdee378a50c2bc5c92e62dfb56e195843bf3 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:22:13 -0700 Subject: [PATCH 75/77] internal/event_logs: remove gin index --- internal/database/schema.json | 10 ---------- internal/database/schema.md | 1 - .../frontend/1691043630_event_logs_indexing/down.sql | 2 -- .../frontend/1691043630_event_logs_indexing/up.sql | 2 -- 4 files changed, 15 deletions(-) diff --git a/internal/database/schema.json b/internal/database/schema.json index c20cd2632228..90fc2d576cc4 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -10607,16 +10607,6 @@ "ConstraintType": "", "ConstraintDefinition": "" }, - { - "Name": "event_logs_name", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops)", - "ConstraintType": "", - "ConstraintDefinition": "" - }, { "Name": "event_logs_name_is_cody_active_event", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index ab1d86852af8..e2179863e6e2 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1379,7 +1379,6 @@ Referenced by: Indexes: "event_logs_pkey" PRIMARY KEY, btree (id) "event_logs_anonymous_user_id" btree (anonymous_user_id) - "event_logs_name" gin (name gin_trgm_ops) "event_logs_name_is_cody_active_event" btree (iscodyactiveevent(name)) "event_logs_name_is_cody_explanation_event" btree (iscodyexplanationevent(name)) "event_logs_name_is_cody_generation_event" btree (iscodygenerationevent(name)) diff --git a/migrations/frontend/1691043630_event_logs_indexing/down.sql b/migrations/frontend/1691043630_event_logs_indexing/down.sql index 66b5d2622430..d8e182d39728 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/down.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/down.sql @@ -1,5 +1,3 @@ -DROP INDEX IF EXISTS event_logs_name; - DROP INDEX IF EXISTS event_logs_name_is_cody_explanation_event; DROP INDEX IF EXISTS event_logs_name_is_cody_generation_event; diff --git a/migrations/frontend/1691043630_event_logs_indexing/up.sql b/migrations/frontend/1691043630_event_logs_indexing/up.sql index 16bd496e23ea..40382eb49574 100644 --- a/migrations/frontend/1691043630_event_logs_indexing/up.sql +++ b/migrations/frontend/1691043630_event_logs_indexing/up.sql @@ -58,8 +58,6 @@ BEGIN END; $$; -CREATE INDEX IF NOT EXISTS event_logs_name ON event_logs USING GIN (name gin_trgm_ops); - CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name)); CREATE INDEX IF NOT EXISTS event_logs_name_is_cody_generation_event ON event_logs USING btree (iscodygenerationevent(name)); From dc940c33b68d756888c4351a8f433185216a2649 Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 18:07:15 -0700 Subject: [PATCH 76/77] internal/event_logs: update event_logs_test --- internal/database/event_logs_test.go | 61 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/internal/database/event_logs_test.go b/internal/database/event_logs_test.go index 7729e218bc86..e6db421b7fa6 100644 --- a/internal/database/event_logs_test.go +++ b/internal/database/event_logs_test.go @@ -1317,7 +1317,7 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) { codyEventNames := []string{"CodyVSCodeExtension:recipe:rewrite-to-functional:executed", "CodyVSCodeExtension:recipe:explain-code-high-level:executed"} - users := []uint32{1, 2} + users := []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20} days := []time.Time{ now, // Today @@ -1332,7 +1332,7 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) { for _, user := range users { for _, name := range codyEventNames { for _, day := range days { - for i := 0; i < 25; i++ { + for i := 0; i < 250; i++ { e := &Event{ UserID: user, Name: name, @@ -1361,35 +1361,46 @@ func TestEventLogs_AggregatedCodyEvents(t *testing.T) { expectedEvents := []types.CodyAggregatedEvent{ { - Name: "CodyVSCodeExtension:recipe:explain-code-high-level:executed", - Month: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC), - Week: now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5), - Day: now.Truncate(time.Hour * 24), - TotalMonth: 200, - TotalWeek: 150, - TotalDay: 50, - UniquesMonth: 2, - UniquesWeek: 2, - UniquesDay: 2, - CodeGenerationWeek: 150, - CodeGenerationDay: 0, - ExplanationMonth: 200, - ExplanationWeek: 150, - ExplanationDay: 50, + Name: "CodyVSCodeExtension:recipe:explain-code-high-level:executed", + Month: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC), + Week: now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5), + Day: now.Truncate(time.Hour * 24), + TotalMonth: 20000, + TotalWeek: 15000, + TotalDay: 5000, + UniquesMonth: 20, + UniquesWeek: 20, + UniquesDay: 20, + CodeGenerationMonth: 0, + CodeGenerationWeek: 0, + CodeGenerationDay: 0, + ExplanationMonth: 20000, + ExplanationWeek: 15000, + ExplanationDay: 5000, + InvalidMonth: 0, + InvalidWeek: 0, + InvalidDay: 0, }, { Name: "CodyVSCodeExtension:recipe:rewrite-to-functional:executed", Month: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC), Week: now.Truncate(time.Hour * 24).Add(-time.Hour * 24 * 5), Day: now.Truncate(time.Hour * 24), - TotalMonth: 200, - TotalWeek: 150, - TotalDay: 50, - UniquesMonth: 2, - UniquesWeek: 2, - UniquesDay: 2, - CodeGenerationMonth: 200, - CodeGenerationDay: 50, + TotalMonth: 20000, + TotalWeek: 15000, + TotalDay: 5000, + UniquesMonth: 20, + UniquesWeek: 20, + UniquesDay: 20, + CodeGenerationMonth: 20000, + CodeGenerationWeek: 15000, + CodeGenerationDay: 5000, + ExplanationMonth: 0, + ExplanationWeek: 0, + ExplanationDay: 0, + InvalidMonth: 0, + InvalidWeek: 0, + InvalidDay: 0, }, } From 50bba65fd4cf0e794871ee1203e8bc49d3a6650b Mon Sep 17 00:00:00 2001 From: Nathan Downs <85511556+nathan-downs@users.noreply.github.com> Date: Wed, 9 Aug 2023 18:15:35 -0700 Subject: [PATCH 77/77] internal/event_logs: remove gin index --- migrations/frontend/squashed.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 66bae0427356..ab508badf6a0 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5926,8 +5926,6 @@ CREATE INDEX event_logs_anonymous_user_id ON event_logs USING btree (anonymous_u CREATE UNIQUE INDEX event_logs_export_allowlist_event_name_idx ON event_logs_export_allowlist USING btree (event_name); -CREATE INDEX event_logs_name ON event_logs USING gin (name gin_trgm_ops); - CREATE INDEX event_logs_name_is_cody_active_event ON event_logs USING btree (iscodyactiveevent(name)); CREATE INDEX event_logs_name_is_cody_explanation_event ON event_logs USING btree (iscodyexplanationevent(name));