From c580b506f562c9294082d3162813aa0b3d736092 Mon Sep 17 00:00:00 2001 From: Kevin Ansfield Date: Mon, 25 Mar 2024 16:58:52 +0000 Subject: [PATCH] Fixed email_recipients indexes to match query usage closes https://linear.app/tryghost/issue/ENG-791/migration-to-fix-email-recipients-indexes Our indexes over single columns (`delivered_at`, `opened_at`, `failed_at`) were ineffective because the only time we query those is alongside `email_id` meaning we were frequently performing full table scans on very large tables during our email analytics jobs. - added migration to add new indexes covering `email_id` and the respective columns - added migration to drop the old indexes that weren't being used in any query plans --- ...-10-add-email-recipients-email-id-indexes.js | 17 +++++++++++++++++ ...rop-email-recipients-non-email-id-indexes.js | 17 +++++++++++++++++ ghost/core/core/server/data/schema/schema.js | 11 +++++++---- .../unit/server/data/schema/integrity.test.js | 2 +- 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js diff --git a/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js new file mode 100644 index 000000000000..5f97c8f71913 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js @@ -0,0 +1,17 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 + +const {createNonTransactionalMigration} = require('../../utils'); +const commands = require('../../../schema/commands'); + +module.exports = createNonTransactionalMigration( + async function up(knex) { + await commands.addIndex('email_recipients', ['email_id', 'delivered_at'], knex); + await commands.addIndex('email_recipients', ['email_id', 'opened_at'], knex); + await commands.addIndex('email_recipients', ['email_id', 'failed_at'], knex); + }, + async function down(knex) { + await commands.dropIndex('email_recipients', ['email_id', 'delivered_at'], knex); + await commands.dropIndex('email_recipients', ['email_id', 'opened_at'], knex); + await commands.dropIndex('email_recipients', ['email_id', 'failed_at'], knex); + } +); diff --git a/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js new file mode 100644 index 000000000000..7b6d5a071720 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js @@ -0,0 +1,17 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 + +const {createNonTransactionalMigration} = require('../../utils'); +const commands = require('../../../schema/commands'); + +module.exports = createNonTransactionalMigration( + async function up(knex) { + await commands.dropIndex('email_recipients', ['delivered_at'], knex); + await commands.dropIndex('email_recipients', ['opened_at'], knex); + await commands.dropIndex('email_recipients', ['failed_at'], knex); + }, + async function down(knex) { + await commands.addIndex('email_recipients', ['delivered_at'], knex); + await commands.addIndex('email_recipients', ['opened_at'], knex); + await commands.addIndex('email_recipients', ['failed_at'], knex); + } +); diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index 338318f0b9e6..21e0ed51f644 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -865,14 +865,17 @@ module.exports = { member_id: {type: 'string', maxlength: 24, nullable: false, index: true}, batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'}, processed_at: {type: 'dateTime', nullable: true}, - delivered_at: {type: 'dateTime', nullable: true, index: true}, - opened_at: {type: 'dateTime', nullable: true, index: true}, - failed_at: {type: 'dateTime', nullable: true, index: true}, + delivered_at: {type: 'dateTime', nullable: true}, + opened_at: {type: 'dateTime', nullable: true}, + failed_at: {type: 'dateTime', nullable: true}, member_uuid: {type: 'string', maxlength: 36, nullable: false}, member_email: {type: 'string', maxlength: 191, nullable: false}, member_name: {type: 'string', maxlength: 191, nullable: true}, '@@INDEXES@@': [ - ['email_id', 'member_email'] + ['email_id', 'member_email'], + ['email_id', 'delivered_at'], + ['email_id', 'opened_at'], + ['email_id', 'failed_at'] ] }, email_recipient_failures: { diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 6e5aa22ec06b..ccb1443be2e5 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '34a9fa4dc1223ef6c45f8ed991d25de5'; + const currentSchemaHash = 'ccf3893bc3f8930f0d1188e646abda6d'; const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd'; const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';