diff --git a/src/core/server/services/migrate/indexing.ts b/src/core/server/services/migrate/indexing.ts index bc7163743f..040ebf1306 100644 --- a/src/core/server/services/migrate/indexing.ts +++ b/src/core/server/services/migrate/indexing.ts @@ -1,10 +1,12 @@ import { merge } from "lodash"; -import { Collection, IndexOptions } from "mongodb"; +import { Collection, Db, IndexOptions } from "mongodb"; import now from "performance-now"; import { Writable } from "coral-common/types"; import logger from "coral-server/logger"; +import collections from "../mongodb/collections"; + type IndexType = 1 | -1 | "text"; export type IndexSpecification = { @@ -61,13 +63,9 @@ export function createIndexFactory( export function createConnectionOrderVariants( createIndexFn: IndexCreationFunction, - variants: Array>, - indexOptions: IndexOptions = { background: true } + variants: Array> ) { - return async ( - indexSpec: IndexSpecification, - variantIndexOptions: IndexOptions = {} - ) => { + return async (indexSpec: IndexSpecification) => { /** * createIndexVariant will create a variant on the specified `indexSpec` that * will include the new variation. @@ -75,10 +73,7 @@ export function createConnectionOrderVariants( * @param variantSpec the spec that makes this variant different */ const createIndexVariant = (variantSpec: IndexSpecification) => - createIndexFn( - merge({}, indexSpec, variantSpec), - merge({}, indexOptions, variantIndexOptions) - ); + createIndexFn(merge({}, indexSpec, variantSpec), { background: true }); // Create all the variants. for (const variant of variants) { @@ -86,3 +81,18 @@ export function createConnectionOrderVariants( } }; } + +export const createIndexesFactory = (mongo: Db) => ({ + users: createIndexFactory(collections.users(mongo)), + invites: createIndexFactory(collections.invites(mongo)), + tenants: createIndexFactory(collections.tenants(mongo)), + comments: createIndexFactory(collections.comments(mongo)), + stories: createIndexFactory(collections.stories(mongo)), + commentActions: createIndexFactory(collections.commentActions(mongo)), + commentModerationActions: createIndexFactory( + collections.commentModerationActions(mongo) + ), + queries: createIndexFactory(collections.queries(mongo)), + migrations: createIndexFactory(collections.migrations(mongo)), + sites: createIndexFactory(collections.sites(mongo)), +}); diff --git a/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts b/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts deleted file mode 100644 index d3a0adfe22..0000000000 --- a/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Db } from "mongodb"; - -import collections from "coral-server/services/mongodb/collections"; - -import { MigrationError } from "../error"; -import Migration from "../migration"; - -export default class extends Migration { - public async down(mongo: Db, tenantID: string) { - // Remove the premod user status from all users on this Tenant. - const result = await collections.users(mongo).updateMany( - { - tenantID, - "status.premod": { $ne: null }, - }, - { - $unset: { - "status.premod": "", - }, - } - ); - - this.log(tenantID).warn( - { matchedCount: result.matchedCount, modifiedCount: result.matchedCount }, - "cleared the premod status from users" - ); - } - - public async test(mongo: Db, tenantID: string) { - // Find all the users that still have premod status unset. - const cursor = collections - .users(mongo) - .find({ - "status.premod": null, - tenantID, - }) - .project({ id: 1 }); - - // Count them. - const users = await cursor.toArray(); - const count = users.length; - if (count === 0) { - this.log(tenantID).info("all users migrated successfully"); - return; - } - - throw new MigrationError( - tenantID, - "found users that were not updated", - "users", - users.map(({ id }) => id) - ); - } - - public async up(mongo: Db, tenantID: string) { - // Migrate users to include the premod status. - const result = await collections.users(mongo).updateMany( - { - "status.premod": null, - tenantID, - }, - { - $set: { - "status.premod": { - active: false, - history: [], - }, - }, - } - ); - - this.log(tenantID).info( - { matchedCount: result.matchedCount, modifiedCount: result.matchedCount }, - "updated user premod status" - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1569612830133_indexes.ts b/src/core/server/services/migrate/migrations/1569612830133_indexes.ts deleted file mode 100644 index e519452c7e..0000000000 --- a/src/core/server/services/migrate/migrations/1569612830133_indexes.ts +++ /dev/null @@ -1,372 +0,0 @@ -import { Db, MongoError } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createConnectionOrderVariants, createIndexFactory } from "../indexing"; - -async function createMigrationRecordIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.migrations(mongo)); - - // UNIQUE { id } - await createIndex({ id: 1 }, { unique: true }); -} - -async function createUserIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.users(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE - PARTIAL { email } - await createIndex( - { tenantID: 1, email: 1 }, - { unique: true, partialFilterExpression: { email: { $exists: true } } } - ); - - // UNIQUE { profiles.type, profiles.id } - await createIndex( - { tenantID: 1, "profiles.type": 1, "profiles.id": 1 }, - { - unique: true, - partialFilterExpression: { "profiles.id": { $exists: true } }, - } - ); - - // { profiles } - await createIndex( - { tenantID: 1, profiles: 1, email: 1 }, - { - partialFilterExpression: { profiles: { $exists: true } }, - background: true, - } - ); - - // TEXT { id, username, email, createdAt } - await createIndex( - { - tenantID: 1, - id: "text", - username: "text", - email: "text", - createdAt: -1, - }, - { background: true } - ); - - const variants = createConnectionOrderVariants( - createIndex, - [{ createdAt: -1 }], - { background: true } - ); - - // User Connection pagination. - // { ...connectionParams } - await variants({ - tenantID: 1, - }); - - // Role based User Connection pagination. - // { role, ...connectionParams } - await variants({ - tenantID: 1, - role: 1, - }); - - // Suspension based User Connection pagination. - await variants({ - tenantID: 1, - "status.suspension.history.from.start": 1, - "status.suspension.history.from.finish": 1, - }); - - // Ban based User Connection pagination. - await variants({ - tenantID: 1, - "status.ban.active": 1, - }); -} - -async function createInviteIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.invites(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE { email } - await createIndex({ tenantID: 1, email: 1 }, { unique: true }); -} - -async function createTenantIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.tenants(mongo)); - - // UNIQUE { id } - await createIndex({ id: 1 }, { unique: true }); - - // UNIQUE { domain } - await createIndex({ domain: 1 }, { unique: true }); -} - -async function createStoryIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.stories(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE { url } - await createIndex({ tenantID: 1, url: 1 }, { unique: true }); - - // TEXT { $**, createdAt } - await createIndex( - { tenantID: 1, "$**": "text", createdAt: -1 }, - { background: true } - ); - - const variants = createConnectionOrderVariants( - createIndex, - [{ createdAt: -1 }], - { background: true } - ); - - // Story Connection pagination. - // { ...connectionParams } - await variants({ - tenantID: 1, - }); - - // Closed At ordered Story Connection pagination. - // { closedAt, ...connectionParams } - await variants({ - tenantID: 1, - closedAt: 1, - }); -} - -async function createStoryCountIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.stories(mongo)); - - // { createdAt } - await createIndex({ tenantID: 1, createdAt: 1 }, { background: true }); -} - -async function createQueriesIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.queries(mongo)); - - // UNIQUE { id } - await createIndex({ id: 1 }, { unique: true }); -} - -async function createCommentActionIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.commentActions(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // { actionType, commentID, userID } - await createIndex( - { tenantID: 1, actionType: 1, commentID: 1, userID: 1 }, - { background: true } - ); - - const variants = createConnectionOrderVariants( - createIndex, - [{ createdAt: -1 }], - { background: true } - ); - - // Connection pagination. - // { ...connectionParams } - await variants({ - tenantID: 1, - actionType: 1, - commentID: 1, - }); -} - -async function createCommentModerationActionIndexes(mongo: Db) { - const createIndex = createIndexFactory( - collections.commentModerationActions(mongo) - ); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - const createVariants = createConnectionOrderVariants(createIndex, [ - { createdAt: -1 }, - ]); - - // { moderatorID, ...connectionParams } - await createVariants({ - moderatorID: 1, - }); -} - -async function createCommentIndexes(mongo: Db) { - const createIndex = createIndexFactory(collections.comments(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // Facility for counting the tags against a story. - await createIndex( - { - tenantID: 1, - storyID: 1, - "tags.type": 1, - status: 1, - }, - { - background: true, - partialFilterExpression: { - "tags.type": { $exists: true }, - }, - } - ); - - const streamVariants = createConnectionOrderVariants(createIndex, [ - { createdAt: -1 }, - { createdAt: 1 }, - { childCount: -1, createdAt: -1 }, - { "actionCounts.REACTION": -1, createdAt: -1 }, - ]); - - // Story based Comment Connection pagination. - // { storyID, ...connectionParams } - await streamVariants({ - tenantID: 1, - storyID: 1, - status: 1, - }); - - // Story + Reply based Comment Connection pagination. - // { storyID, ...connectionParams } - await streamVariants({ - tenantID: 1, - storyID: 1, - parentID: 1, - status: 1, - }); - - // Author based Comment Connection pagination. - // { authorID, ...connectionParams } - await streamVariants({ - tenantID: 1, - authorID: 1, - status: 1, - }); - - // Tag based Comment Connection pagination. - // { tags.type, ...connectionParams } - await streamVariants({ - tenantID: 1, - storyID: 1, - "tags.type": 1, - }); - - const adminVariants = createConnectionOrderVariants( - createIndex, - [{ createdAt: -1 }, { createdAt: 1 }], - { background: true } - ); - - // Moderation based Comment Connection pagination. - // { storyID, ...connectionParams } - await adminVariants({ - tenantID: 1, - status: 1, - }); - - // Story based Comment Connection pagination that are flagged. - // { storyID, ...connectionParams } - await adminVariants({ - tenantID: 1, - storyID: 1, - status: 1, - "actionCounts.FLAG": 1, - }); - - // Author based Comment Connection pagination. - // { authorID, ...connectionParams } - await adminVariants({ - tenantID: 1, - authorID: 1, - }); -} - -type IndexCreationFunction = (mongo: Db) => Promise; - -const indexes: Array<[string, IndexCreationFunction]> = [ - ["migrations", createMigrationRecordIndexes], - ["users", createUserIndexes], - ["invites", createInviteIndexes], - ["tenants", createTenantIndexes], - ["comments", createCommentIndexes], - ["stories", createStoryIndexes], - ["stories", createStoryCountIndexes], - ["commentActions", createCommentActionIndexes], - ["commentModerationActions", createCommentModerationActionIndexes], - ["queries", createQueriesIndexes], -]; - -export default class extends Migration { - /** - * ensureIndexes will ensure that all indexes have been created. - * - * @param mongo a MongoDB Database Connection - */ - private async ensureIndexes(mongo: Db) { - this.logger.info( - { indexGroupCount: indexes.length }, - "now ensuring indexes are created" - ); - - // For each of the index functions, call it. - for (const [indexGroup, indexFunction] of indexes) { - this.logger.info({ indexGroup }, "ensuring indexes are created"); - await indexFunction(mongo); - this.logger.info({ indexGroup }, "indexes have been created"); - } - - this.logger.info("all indexes have been created"); - } - - public async indexes(mongo: Db) { - // Find all the collections that exist already. - const results = await mongo - .listCollections({}, { nameOnly: true }) - .toArray(); - - const collectionNames = results - .filter(({ type }) => type === "collection") - .map(({ name }) => name); - - // Drop existing indexes on managed collections so we can re-create them. - for (const collectionName in collections) { - if (!collections.hasOwnProperty(collectionName)) { - continue; - } - - // Check to see if this collection exists. - if (!collectionNames.includes(collectionName)) { - continue; - } - - try { - await mongo.collection(collectionName).dropIndexes(); - } catch (err) { - if (err instanceof MongoError) { - // If we're dropping indexes on a collection that doesn't exist, then - // don't worry. - if (err.code === 26) { - continue; - } - } - - throw err; - } - } - - // Re-create the indexes for each collection now. - await this.ensureIndexes(mongo); - } -} diff --git a/src/core/server/services/migrate/migrations/1569947670260_add_moderator_notes_to_user.ts b/src/core/server/services/migrate/migrations/1569947670260_add_moderator_notes_to_user.ts deleted file mode 100644 index e3b849c993..0000000000 --- a/src/core/server/services/migrate/migrations/1569947670260_add_moderator_notes_to_user.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Db } from "mongodb"; - -import collections from "coral-server/services/mongodb/collections"; - -import Migration from "coral-server/services/migrate/migration"; - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - const result = await collections.users(mongo).updateMany( - { - tenantID, - moderatorNotes: null, - }, - { - $set: { - moderatorNotes: [], - }, - } - ); - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.modifiedCount, - }, - "added empty moderatorNotes array" - ); - } - - public async down(mongo: Db, tenantID: string) { - const result = await collections.users(mongo).updateMany( - { - tenantID, - moderatorNotes: { - $exists: true, - }, - }, - { - $unset: { - moderatorNotes: "", - }, - } - ); - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.matchedCount, - }, - "removed moderatorNotes" - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1569955183249_text_indexes.ts b/src/core/server/services/migrate/migrations/1569955183249_text_indexes.ts deleted file mode 100644 index 43918cdf53..0000000000 --- a/src/core/server/services/migrate/migrations/1569955183249_text_indexes.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createConnectionOrderVariants, createIndexFactory } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - const createIndex = createIndexFactory(collections.comments(mongo)); - - // Text searches. - await createIndex( - { - tenantID: 1, - "revisions.body": "text", - }, - { background: true } - ); - - const variants = createConnectionOrderVariants(createIndex, [ - { createdAt: -1 }, - { createdAt: 1 }, - { childCount: -1, createdAt: -1 }, - { "actionCounts.REACTION": -1, createdAt: -1 }, - ]); - - // Story based Comment Connection pagination. - // { storyID, ...connectionParams } - await variants({ - tenantID: 1, - storyID: 1, - }); - } -} diff --git a/src/core/server/services/migrate/migrations/1570118392071_staff.ts b/src/core/server/services/migrate/migrations/1570118392071_staff.ts deleted file mode 100644 index fd3b256643..0000000000 --- a/src/core/server/services/migrate/migrations/1570118392071_staff.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Db } from "mongodb"; - -import { getDefaultStaffConfiguration } from "coral-server/models/tenant"; -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { MigrationError } from "../error"; - -export default class extends Migration { - private async getTenant(mongo: Db, id: string) { - const tenant = await collections.tenants(mongo).findOne({ id }); - if (!tenant) { - throw new MigrationError(id, "tenant not found", "tenants", [id]); - } - - return tenant; - } - - public async test(mongo: Db, id: string) { - const tenant = await this.getTenant(mongo, id); - if (!tenant.staff) { - throw new MigrationError(id, "staff not set", "tenants", [id]); - } - } - - public async up(mongo: Db, id: string) { - const tenant = await this.getTenant(mongo, id); - const bundle = this.i18n.getBundle(tenant.locale); - await collections.tenants(mongo).updateOne( - { id, staff: null }, - { - $set: { - staff: getDefaultStaffConfiguration(bundle), - }, - } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1570712877087_add_newcommenters_settings_to_tenant.ts b/src/core/server/services/migrate/migrations/1570712877087_add_newcommenters_settings_to_tenant.ts deleted file mode 100644 index 1bea1db63b..0000000000 --- a/src/core/server/services/migrate/migrations/1570712877087_add_newcommenters_settings_to_tenant.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Db } from "mongodb"; - -// Use the following collections reference to interact with specific -// collections. -import collections from "coral-server/services/mongodb/collections"; - -import Migration from "coral-server/services/migrate/migration"; - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - const result = await collections.tenants(mongo).updateOne( - { - id: tenantID, - newCommenters: null, - }, - { - $set: { - newCommenters: { - premodEnabled: false, - approvedCommentsThreshold: 2, - }, - }, - } - ); - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.matchedCount, - }, - "added new commenters config" - ); - } - - public async down(mongo: Db, tenantID: string) { - const result = await collections.tenants(mongo).updateOne( - { - id: tenantID, - newCommenters: { - $exists: true, - }, - }, - { - $unset: { - newCommenters: "", - }, - } - ); - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.matchedCount, - }, - "removed new commenters config" - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1572920233903_comment_moderation_actions_indexes.ts b/src/core/server/services/migrate/migrations/1572920233903_comment_moderation_actions_indexes.ts deleted file mode 100644 index ed52e9d817..0000000000 --- a/src/core/server/services/migrate/migrations/1572920233903_comment_moderation_actions_indexes.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createIndexFactory } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - const createIndex = createIndexFactory( - collections.commentModerationActions(mongo) - ); - - await createIndex( - { tenantID: 1, commentID: 1, createdAt: -1 }, - { background: true } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1572991283040_user_profiles.ts b/src/core/server/services/migrate/migrations/1572991283040_user_profiles.ts deleted file mode 100644 index d9aeae4720..0000000000 --- a/src/core/server/services/migrate/migrations/1572991283040_user_profiles.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createIndex } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - // Drop the old index. - await collections - .users(mongo) - .dropIndex("tenantID_1_profiles.type_1_profiles.id_1"); - - // Clean up the old users that have deleted their accounts. - await collections - .users(mongo) - .updateMany({ profiles: [] }, { $unset: { profiles: "" } }); - - // Add the new index. - await createIndex( - collections.users(mongo), - { - tenantID: 1, - "profiles.id": 1, - "profiles.type": 1, - }, - { - unique: true, - partialFilterExpression: { profiles: { $exists: true } }, - } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1573073491825_sso_tokens.ts b/src/core/server/services/migrate/migrations/1573073491825_sso_tokens.ts deleted file mode 100644 index 2308912ced..0000000000 --- a/src/core/server/services/migrate/migrations/1573073491825_sso_tokens.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Db } from "mongodb"; - -import { Secret } from "coral-server/models/settings"; -import { generateSecret, Tenant } from "coral-server/models/tenant"; -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { GQLTime } from "coral-server/graph/schema/__generated__/types"; - -import { MigrationError } from "../error"; - -interface OldTenant { - auth: { - integrations: { - sso: { - key?: string; - keyGeneratedAt?: GQLTime; - }; - }; - }; -} - -function isOldTenant(tenant: Tenant | OldTenant): tenant is OldTenant { - if ((tenant as Tenant).auth.integrations.sso.keys) { - return false; - } - - return true; -} - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - // Get the Tenant so we can update it. We don't have to worry about two - // migration operations conflicting here because we will lock one instance - // to perform the operations. - const tenant = await collections - .tenants(mongo) - .findOne({ id: tenantID }); - if (!tenant) { - throw new MigrationError(tenantID, "could not find tenant", "tenants", [ - tenantID, - ]); - } - - if (!isOldTenant(tenant)) { - this.log(tenantID).info("tenant already has the new format for the keys"); - return; - } - - // Store the keys in an array. - const keys: Secret[] = []; - - // Check to see if a key is set. - const sso = tenant.auth.integrations.sso; - - if (sso.key && sso.keyGeneratedAt) { - // Create the new SSOKey based on this data. - const key = generateSecret("ssosec", sso.keyGeneratedAt); - - // Set the secret of the sso key to the secret of the current set key. - key.secret = sso.key; - - // Add this key to the set of keys. - keys.push(key); - } else { - throw new MigrationError( - tenantID, - "expected tenant to have at least one SSO key provided, found none", - "tenants", - [tenantID] - ); - } - - // Update the tenant with the new sso keys. - await collections.tenants(mongo).updateOne( - { id: tenantID }, - { - $set: { - "auth.integrations.sso.keys": keys, - }, - $unset: { - "auth.integrations.sso.key": "", - "auth.integrations.sso.keyGeneratedAt": "", - }, - } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1573858750460_sso_token_refactor.ts b/src/core/server/services/migrate/migrations/1573858750460_sso_token_refactor.ts deleted file mode 100644 index 344e368513..0000000000 --- a/src/core/server/services/migrate/migrations/1573858750460_sso_token_refactor.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { DateTime } from "luxon"; -import { Db } from "mongodb"; - -import { Secret } from "coral-server/models/settings"; -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { MigrationError } from "../error"; - -interface OldSSOKey { - kid: string; - secret?: string; - createdAt: Date; - deprecateAt?: Date; - deletedAt?: Date; -} - -function isOldSSOKey(key: Secret | OldSSOKey): key is OldSSOKey { - if (!key) { - return true; - } - - if ((key as Secret).inactiveAt) { - return false; - } - - if ((key as Secret).rotatedAt) { - return false; - } - - if (key.secret === "") { - return false; - } - - if ((key as OldSSOKey).deprecateAt) { - return true; - } - - if ((key as OldSSOKey).deletedAt) { - return true; - } - - return false; -} - -interface OldTenant { - auth: { - integrations: { - sso: { - keys: OldSSOKey[]; - }; - }; - }; -} - -export default class extends Migration { - public async down(mongo: Db, id: string) { - // Get the Tenant that stores the keys in the old format. - const tenant = await collections.tenants(mongo).findOne({ id }); - if (!tenant) { - throw new MigrationError(id, "tenant was not found", "tenants", [id]); - } - - // Transform the keys into the new format. - const keys: OldSSOKey[] = tenant.auth.integrations.sso.keys.map( - (key: OldSSOKey | Secret): OldSSOKey => - !isOldSSOKey(key) - ? { - kid: key.kid, - secret: key.secret === "" ? undefined : key.secret, - createdAt: key.createdAt, - deprecateAt: key.inactiveAt, - } - : key - ); - - // Update the key to the new format. - await collections - .tenants(mongo) - .updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } }); - } - - public async test(mongo: Db, id: string) { - // Get the Tenant that stores the keys in the old format. - const tenant = await collections.tenants(mongo).findOne({ id }); - if (!tenant) { - throw new MigrationError(id, "tenant was not found", "tenants", [id]); - } - - if (tenant.auth.integrations.sso.keys.some(key => isOldSSOKey(key))) { - throw new MigrationError(id, "old sso key was found", "tenants", [id]); - } - } - - public async up(mongo: Db, id: string) { - // Get the Tenant that stores the keys in the old format. - const tenant = await collections.tenants(mongo).findOne({ id }); - if (!tenant) { - throw new MigrationError(id, "tenant was not found", "tenants", [id]); - } - - // Transform the keys into the new format. - const keys: Secret[] = tenant.auth.integrations.sso.keys.map( - (key): Secret => ({ - kid: key.kid, - secret: key.secret || "", - createdAt: key.createdAt, - inactiveAt: key.deprecateAt, - rotatedAt: key.deprecateAt - ? DateTime.fromJSDate(key.deprecateAt) - .plus({ month: -1 }) - .toJSDate() - : undefined, - }) - ); - - // Update the key to the new format. - await collections - .tenants(mongo) - .updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } }); - } -} diff --git a/src/core/server/services/migrate/migrations/1574287034612_notification_digest_index.ts b/src/core/server/services/migrate/migrations/1574287034612_notification_digest_index.ts deleted file mode 100644 index be1b4ed2a9..0000000000 --- a/src/core/server/services/migrate/migrations/1574287034612_notification_digest_index.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createIndex } from "../indexing"; - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - const result = await collections.users(mongo).updateMany( - { - tenantID, - digests: { - $ne: [], - }, - }, - { - $set: { - hasDigests: true, - }, - } - ); - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.modifiedCount, - }, - "added hasDigests flag" - ); - } - - public async indexes(mongo: Db) { - await createIndex( - collections.users(mongo), - { - tenantID: 1, - "notifications.digestFrequency": 1, - hasDigests: 1, - }, - { - partialFilterExpression: { hasDigests: { $eq: true } }, - background: true, - } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1574289134415_scheduled_deletion_date_index.ts b/src/core/server/services/migrate/migrations/1574289134415_scheduled_deletion_date_index.ts deleted file mode 100644 index ad04f513aa..0000000000 --- a/src/core/server/services/migrate/migrations/1574289134415_scheduled_deletion_date_index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createIndex } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - await createIndex( - collections.users(mongo), - { - tenantID: 1, - scheduledDeletionDate: 1, - }, - { - partialFilterExpression: { scheduledDeletionDate: { $exists: true } }, - background: true, - } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1575649180000_user_comment_counts.ts b/src/core/server/services/migrate/migrations/1575649180000_user_comment_counts.ts deleted file mode 100644 index 98ea9b4a77..0000000000 --- a/src/core/server/services/migrate/migrations/1575649180000_user_comment_counts.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { Db } from "mongodb"; - -import { - CommentStatusCounts, - createEmptyCommentStatusCounts, -} from "coral-server/models/comment"; -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { GQLCOMMENT_STATUS } from "coral-server/graph/schema/__generated__/types"; - -const BATCH_SIZE = 500; - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - const cursor = collections - .comments<{ - _id: string; - statuses: { status: GQLCOMMENT_STATUS; sum: number }[]; - }>(mongo) - .aggregate([ - // Find all comments written by this Tenant. - { - $match: { tenantID }, - }, - // Group each comment into it's authorID and status. - { - $group: { - _id: { - authorID: "$authorID", - status: "$status", - }, - sum: { $sum: 1 }, - }, - }, - // Group the documents from the previous stage into a count of each - // author's status counts. - { - $group: { - _id: "$_id.authorID", - statuses: { - $push: { - status: "$_id.status", - sum: "$sum", - }, - }, - }, - }, - ]); - - let updates: { - updateOne: { - filter: { tenantID: string; id: string }; - update: { $set: { commentCounts: { status: CommentStatusCounts } } }; - }; - }[] = []; - - while (await cursor.hasNext()) { - const doc = await cursor.next(); - if (!doc) { - break; - } - - // Reconstruct the comment status counts. - const statuses = createEmptyCommentStatusCounts(); - for (const { status, sum } of doc.statuses) { - statuses[status] += sum; - } - - // Push the update. - updates.push({ - updateOne: { - filter: { - tenantID, - id: doc._id, - }, - update: { - $set: { - commentCounts: { - status: statuses, - }, - }, - }, - }, - }); - - // Process updates if we are at the batch size. - if (updates.length >= BATCH_SIZE) { - await collections.users(mongo).bulkWrite(updates); - updates = []; - } - } - - // Process any missed updates. - if (updates.length > 0) { - await collections.users(mongo).bulkWrite(updates); - updates = []; - } - } -} diff --git a/src/core/server/services/migrate/migrations/1578604997397_story_add_last_commented_at.ts b/src/core/server/services/migrate/migrations/1578604997397_story_add_last_commented_at.ts deleted file mode 100644 index fd55641bdb..0000000000 --- a/src/core/server/services/migrate/migrations/1578604997397_story_add_last_commented_at.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -import { createIndexFactory } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - const createIndex = createIndexFactory(collections.stories(mongo)); - - // Create a partial sparse index on lastCommentedAt: - // tenantID: 1 <- ASC tenantIDs - // lastCommentedAt: -1 <- DESC dates (more recent to latest) - // ^ explained here: https://docs.mongodb.com/manual/core/index-compound/#sort-order - await createIndex( - { tenantID: 1, lastCommentedAt: -1 }, - { partialFilterExpression: { lastCommentedAt: { $exists: true } } } - ); - } -} diff --git a/src/core/server/services/migrate/migrations/1579189174930_indexes.ts b/src/core/server/services/migrate/migrations/1579189174930_indexes.ts new file mode 100644 index 0000000000..85bf637772 --- /dev/null +++ b/src/core/server/services/migrate/migrations/1579189174930_indexes.ts @@ -0,0 +1,195 @@ +import { Db } from "mongodb"; + +import Migration from "coral-server/services/migrate/migration"; + +import { + createConnectionOrderVariants, + createIndexesFactory, +} from "../indexing"; + +function createIndexer(mongo: Db) { + // Get the indexing functions ready for each of the collections. + const index = createIndexesFactory(mongo); + + // Get the variants ready for pagination indexing. + const variants = { + comments: { + admin: createConnectionOrderVariants(index.comments, [ + { createdAt: -1 }, + { createdAt: 1 }, + ]), + stream: createConnectionOrderVariants(index.comments, [ + { createdAt: -1 }, + { createdAt: 1 }, + { childCount: -1, createdAt: -1 }, + { "actionCounts.REACTION": -1, createdAt: -1 }, + ]), + }, + users: createConnectionOrderVariants(index.users, [{ createdAt: -1 }]), + stories: createConnectionOrderVariants(index.stories, [{ createdAt: -1 }]), + commentActions: createConnectionOrderVariants(index.commentActions, [ + { createdAt: -1 }, + ]), + commentModerationActions: createConnectionOrderVariants( + index.commentModerationActions, + [{ createdAt: 1 }] + ), + }; + + return { index, variants }; +} + +export default class extends Migration { + public async indexes(mongo: Db) { + // Create the indexer functions. + const { index, variants } = createIndexer(mongo); + + // Comments + await index.comments({ tenantID: 1, id: 1 }, { unique: true }); + await index.comments( + { tenantID: 1, "revisions.body": "text" }, + { background: true } + ); + await index.comments( + { + tenantID: 1, + storyID: 1, + "tags.type": 1, + status: 1, + }, + { + partialFilterExpression: { + "tags.type": { $exists: true }, + }, + background: true, + } + ); + await variants.comments.stream({ tenantID: 1, storyID: 1 }); + await variants.comments.stream({ tenantID: 1, storyID: 1, status: 1 }); + await variants.comments.stream({ + tenantID: 1, + storyID: 1, + parentID: 1, + status: 1, + }); + await variants.comments.stream({ tenantID: 1, authorID: 1, status: 1 }); + await variants.comments.stream({ tenantID: 1, storyID: 1, "tags.type": 1 }); + await variants.comments.admin({ tenantID: 1, status: 1 }); + await variants.comments.admin({ tenantID: 1, authorID: 1 }); + await variants.comments.admin({ + tenantID: 1, + storyID: 1, + status: 1, + "actionCounts.FLAG": 1, + }); + + // Stories + await index.stories( + { tenantID: 1, lastCommentedAt: -1 }, + { + partialFilterExpression: { lastCommentedAt: { $exists: true } }, + background: true, + } + ); + await index.stories({ tenantID: 1, id: 1 }, { unique: true }); + await index.stories({ tenantID: 1, url: 1 }, { unique: true }); + await index.stories( + { tenantID: 1, "$**": "text", createdAt: -1 }, + { background: true } + ); + await index.stories({ tenantID: 1, createdAt: 1 }, { background: true }); + await variants.stories({ tenantID: 1 }); + await variants.stories({ tenantID: 1, closedAt: 1 }); + + // Comment Moderation Actions + await index.commentModerationActions( + { tenantID: 1, id: 1 }, + { unique: true } + ); + await index.commentModerationActions( + { tenantID: 1, commentID: 1, createdAt: -1 }, + { background: true } + ); + await variants.commentModerationActions({ tenantID: 1, moderatorID: 1 }); + + // Comment Actions + await index.commentActions({ tenantID: 1, id: 1 }, { unique: true }); + await index.commentActions( + { tenantID: 1, actionType: 1, commentID: 1, userID: 1 }, + { background: true } + ); + await variants.commentActions({ tenantID: 1, actionType: 1, commentID: 1 }); + + // Users + await index.users({ tenantID: 1, id: 1 }, { unique: true }); + await index.users( + { tenantID: 1, email: 1 }, + { partialFilterExpression: { email: { $exists: true } }, unique: true } + ); + await index.users( + { tenantID: 1, profiles: 1, email: 1 }, + { + partialFilterExpression: { profiles: { $exists: true } }, + background: true, + } + ); + await index.users( + { tenantID: 1, scheduledDeletionDate: 1 }, + { + partialFilterExpression: { scheduledDeletionDate: { $exists: true } }, + background: true, + } + ); + await index.users( + { tenantID: 1, "notifications.digestFrequency": 1, hasDigests: 1 }, + { + partialFilterExpression: { hasDigests: { $eq: true } }, + background: true, + } + ); + await index.users( + { tenantID: 1, "profiles.id": 1, "profiles.type": 1 }, + { + partialFilterExpression: { profiles: { $exists: true } }, + unique: true, + } + ); + await index.users( + { + tenantID: 1, + id: "text", + username: "text", + email: "text", + createdAt: -1, + }, + { background: true } + ); + await variants.users({ tenantID: 1 }); + await variants.users({ tenantID: 1, role: 1 }); + await variants.users({ + tenantID: 1, + "status.suspension.history.from.start": 1, + "status.suspension.history.from.finish": 1, + }); + await variants.users({ tenantID: 1, "status.ban.active": 1 }); + + // Invites + await index.invites({ tenantID: 1, id: 1 }, { unique: true }); + await index.invites({ tenantID: 1, email: 1 }, { unique: true }); + + // Migrations + await index.migrations({ id: 1 }, { unique: true }); + + // Tenants + await index.tenants({ id: 1 }, { unique: true }); + await index.tenants({ domain: 1 }, { unique: true }); + + // Sites + await index.sites({ tenantID: 1, id: 1 }, { background: true }); + await index.sites({ tenantID: 1, allowedOrigins: 1 }, { unique: true }); + await index.sites({ tenantID: 1, name: 1 }, { background: true }); + + // Queries + await index.queries({ id: 1 }, { unique: true }); + } +} diff --git a/src/core/server/services/migrate/migrations/1579878509162_add_indexes_to_sites.ts b/src/core/server/services/migrate/migrations/1579878509162_add_indexes_to_sites.ts deleted file mode 100644 index 63cfdbe00c..0000000000 --- a/src/core/server/services/migrate/migrations/1579878509162_add_indexes_to_sites.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Db } from "mongodb"; - -// Use the following collections reference to interact with specific -// collections. -import collections from "coral-server/services/mongodb/collections"; - -import Migration from "coral-server/services/migrate/migration"; -import { createIndexFactory } from "../indexing"; - -export default class extends Migration { - public async indexes(mongo: Db) { - const createIndex = createIndexFactory(collections.sites(mongo)); - await createIndex({ tenantID: 1, id: 1 }, { background: true }); - await createIndex({ tenantID: 1, allowedOrigins: 1 }, { unique: true }); - await createIndex({ tenantID: 1, name: 1 }, { background: true }); - } -} diff --git a/src/core/server/services/migrate/migrations/1580404849316_user_comment_counts.ts b/src/core/server/services/migrate/migrations/1580404849316_user_comment_counts.ts deleted file mode 100644 index 44a485b36a..0000000000 --- a/src/core/server/services/migrate/migrations/1580404849316_user_comment_counts.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Db } from "mongodb"; - -import Migration from "coral-server/services/migrate/migration"; -import collections from "coral-server/services/mongodb/collections"; - -export default class extends Migration { - public async up(mongo: Db, tenantID: string) { - const result = await collections.users(mongo).updateMany( - { tenantID, commentCounts: null }, - { - $set: { - commentCounts: { - status: { - APPROVED: 0, - NONE: 0, - PREMOD: 0, - REJECTED: 0, - SYSTEM_WITHHELD: 0, - }, - }, - }, - } - ); - - this.log(tenantID).warn( - { - matchedCount: result.matchedCount, - modifiedCount: result.modifiedCount, - }, - "applied fix to users" - ); - } -}