diff --git a/.mocharc.json b/.mocharc.json index 3817a0a2ffd..58809b203d7 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -1,6 +1,13 @@ { - "extension": ["js"], - "require": ["ts-node/register", "source-map-support/register"], + "$schema": "https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/mocharc.json", + "extension": [ + "js", + "ts" + ], + "require": [ + "ts-node/register", + "source-map-support/register" + ], "file": "test/tools/runner", "ui": "test/tools/runner/metadata_ui.js", "recursive": true, diff --git a/package-lock.json b/package-lock.json index 9d103f546f6..8147f2678ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -527,6 +527,21 @@ "@types/node": "*" } }, + "@types/chai": { + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.14.tgz", + "integrity": "sha512-G+ITQPXkwTrslfG5L/BksmbLUA0M1iybEsmCWPqzSxsRRhJZimBKJkoMi8fr/CPygPTj4zO5pJH7I2/cm9M7SQ==", + "dev": true + }, + "@types/chai-subset": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@types/chai-subset/-/chai-subset-1.3.3.tgz", + "integrity": "sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==", + "dev": true, + "requires": { + "@types/chai": "*" + } + }, "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", @@ -581,6 +596,12 @@ "integrity": "sha512-nh/fE/f/a8k6hF62GWNdw5kpR25SQyk1xvWLsJSoWLLvLFBRmT6rdCSUepuxuGGuwBDC+QcaIHqPoZ4ZRjxFBg==", "dev": true }, + "@types/semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==", + "dev": true + }, "@typescript-eslint/eslint-plugin": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-3.10.1.tgz", diff --git a/package.json b/package.json index 0d6dfb9d52a..feba51914d9 100644 --- a/package.json +++ b/package.json @@ -34,10 +34,13 @@ "@microsoft/tsdoc-config": "^0.13.9", "@types/aws4": "^1.5.1", "@types/bl": "^2.1.0", + "@types/chai": "^4.2.14", + "@types/chai-subset": "^1.3.3", "@types/kerberos": "^1.1.0", "@types/mocha": "^8.2.0", "@types/node": "^14.6.4", "@types/saslprep": "^1.0.0", + "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.10.0", "@typescript-eslint/parser": "^3.10.0", "chai": "^4.2.0", diff --git a/test/functional/unified-spec-runner/.eslintrc.json b/test/functional/unified-spec-runner/.eslintrc.json new file mode 100644 index 00000000000..cc92fa60ac0 --- /dev/null +++ b/test/functional/unified-spec-runner/.eslintrc.json @@ -0,0 +1,28 @@ +{ + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 2018 + }, + "plugins": [ + "@typescript-eslint", + "prettier", + "promise", + "eslint-plugin-tsdoc" + ], + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended", + "prettier/@typescript-eslint", + "plugin:prettier/recommended" + ], + "env": { + "node": true, + "mocha": true, + "es6": true + }, + "rules": { + "prettier/prettier": "error", + "tsdoc/syntax": "warn" + } +} diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts new file mode 100644 index 00000000000..9acf9064fca --- /dev/null +++ b/test/functional/unified-spec-runner/entities.ts @@ -0,0 +1,161 @@ +import { MongoClient, Db, Collection, GridFSBucket, Document } from '../../../src/index'; +import { ClientSession } from '../../../src/sessions'; +import { ChangeStream } from '../../../src/change_stream'; +import type { ClientEntity, EntityDescription } from './schema'; +import type { + CommandFailedEvent, + CommandStartedEvent, + CommandSucceededEvent +} from '../../../src/cmap/events'; +import { patchCollectionOptions, patchDbOptions } from './unified-utils'; +import { TestConfiguration } from './unified.test'; + +export type CommandEvent = CommandStartedEvent | CommandSucceededEvent | CommandFailedEvent; + +export class UnifiedMongoClient extends MongoClient { + events: CommandEvent[]; + observedEvents: ('commandStarted' | 'commandSucceeded' | 'commandFailed')[]; + + static EVENT_NAME_LOOKUP = { + commandStartedEvent: 'commandStarted', + commandSucceededEvent: 'commandSucceeded', + commandFailedEvent: 'commandFailed' + } as const; + + constructor(url: string, description: ClientEntity) { + super(url, { monitorCommands: true, ...description.uriOptions }); + this.events = []; + // apm + this.observedEvents = (description.observeEvents ?? []).map( + e => UnifiedMongoClient.EVENT_NAME_LOOKUP[e] + ); + for (const eventName of this.observedEvents) { + this.on(eventName, this.pushEvent); + } + } + + // NOTE: this must be an arrow function for `this` to work. + pushEvent: (e: CommandEvent) => void = e => { + this.events.push(e); + }; + + /** Disables command monitoring for the client and returns a list of the captured events. */ + stopCapturingEvents(): CommandEvent[] { + for (const eventName of this.observedEvents) { + this.off(eventName, this.pushEvent); + } + return this.events; + } +} + +export type Entity = + | UnifiedMongoClient + | Db + | Collection + | ClientSession + | ChangeStream + | GridFSBucket + | Document; // Results from operations + +export type EntityCtor = + | typeof UnifiedMongoClient + | typeof Db + | typeof Collection + | typeof ClientSession + | typeof ChangeStream + | typeof GridFSBucket; + +export type EntityTypeId = 'client' | 'db' | 'collection' | 'session' | 'bucket' | 'stream'; + +const ENTITY_CTORS = new Map(); +ENTITY_CTORS.set('client', UnifiedMongoClient); +ENTITY_CTORS.set('db', Db); +ENTITY_CTORS.set('collection', Collection); +ENTITY_CTORS.set('session', ClientSession); +ENTITY_CTORS.set('bucket', GridFSBucket); +ENTITY_CTORS.set('stream', ChangeStream); + +export class EntitiesMap extends Map { + mapOf(type: 'client'): EntitiesMap; + mapOf(type: 'db'): EntitiesMap; + mapOf(type: 'collection'): EntitiesMap; + mapOf(type: 'session'): EntitiesMap; + mapOf(type: 'bucket'): EntitiesMap; + mapOf(type: 'stream'): EntitiesMap; + mapOf(type: EntityTypeId): EntitiesMap { + const ctor = ENTITY_CTORS.get(type); + if (!ctor) { + throw new Error(`Unknown type ${type}`); + } + return new EntitiesMap(Array.from(this.entries()).filter(([, e]) => e instanceof ctor)); + } + + getEntity(type: 'client', key: string, assertExists?: boolean): UnifiedMongoClient; + getEntity(type: 'db', key: string, assertExists?: boolean): Db; + getEntity(type: 'collection', key: string, assertExists?: boolean): Collection; + getEntity(type: 'session', key: string, assertExists?: boolean): ClientSession; + getEntity(type: 'bucket', key: string, assertExists?: boolean): GridFSBucket; + getEntity(type: 'stream', key: string, assertExists?: boolean): ChangeStream; + getEntity(type: EntityTypeId, key: string, assertExists = true): Entity { + const entity = this.get(key); + if (!entity) { + if (assertExists) throw new Error(`Entity ${key} does not exist`); + return; + } + const ctor = ENTITY_CTORS.get(type); + if (!ctor) { + throw new Error(`Unknown type ${type}`); + } + if (!(entity instanceof ctor)) { + throw new Error(`${key} is not an instance of ${type}`); + } + return entity; + } + + async cleanup(): Promise { + for (const [, client] of this.mapOf('client')) { + await client.close(); + } + for (const [, session] of this.mapOf('session')) { + await session.endSession(); + } + this.clear(); + } + + static async createEntities( + config: TestConfiguration, + entities?: EntityDescription[] + ): Promise { + const map = new EntitiesMap(); + for (const entity of entities ?? []) { + if ('client' in entity) { + const client = new UnifiedMongoClient(config.url(), entity.client); + await client.connect(); + map.set(entity.client.id, client); + } else if ('database' in entity) { + const client = map.getEntity('client', entity.database.client); + const db = client.db( + entity.database.databaseName, + patchDbOptions(entity.database.databaseOptions) + ); + map.set(entity.database.id, db); + } else if ('collection' in entity) { + const db = map.getEntity('db', entity.collection.database); + const collection = db.collection( + entity.collection.collectionName, + patchCollectionOptions(entity.collection.collectionOptions) + ); + map.set(entity.collection.id, collection); + } else if ('session' in entity) { + map.set(entity.session.id, null); + } else if ('bucket' in entity) { + map.set(entity.bucket.id, null); + } else if ('stream' in entity) { + map.set(entity.stream.id, null); + } else { + throw new Error(`Unsupported Entity ${JSON.stringify(entity)}`); + } + } + return map; + } +} diff --git a/test/functional/unified-spec-runner/operations.ts b/test/functional/unified-spec-runner/operations.ts new file mode 100644 index 00000000000..6697d690e9f --- /dev/null +++ b/test/functional/unified-spec-runner/operations.ts @@ -0,0 +1,364 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { expect } from 'chai'; +import { Document, MongoError } from '../../../src'; +import type { EntitiesMap } from './entities'; +import type * as uni from './schema'; +import { + isExistsOperator, + isMatchesEntityOperator, + isMatchesHexBytesOperator, + isSessionLsidOperator, + isSpecialOperator, + isTypeOperator, + isUnsetOrMatchesOperator, + SpecialOperator +} from './unified-utils'; + +export class UnifiedOperation { + name: string; + constructor(op: uni.OperationDescription) { + this.name = op.name; + } +} + +async function abortTransactionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function aggregateOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertCollectionExistsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertCollectionNotExistsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertIndexExistsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertIndexNotExistsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertDifferentLsidOnLastTwoCommandsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSameLsidOnLastTwoCommandsOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSessionDirtyOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSessionNotDirtyOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSessionPinnedOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSessionUnpinnedOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function assertSessionTransactionStateOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function bulkWriteOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function commitTransactionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function createChangeStreamOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function createCollectionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function createIndexOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function deleteOneOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function dropCollectionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function endSessionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function findOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function findOneAndReplaceOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function findOneAndUpdateOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function failPointOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function insertOneOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + const collection = entities.getEntity('collection', op.object); + const session = entities.getEntity('session', op.arguments.session, false); + const result = await collection.insertOne(op.arguments.document); + return result; +} +async function insertManyOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + const collection = entities.getEntity('collection', op.object); + const session = entities.getEntity('session', op.arguments.session, false); + const options = { + ordered: op.arguments.ordered ?? true + }; + const result = await collection.insertMany(op.arguments.documents, options); + return result; +} +async function iterateUntilDocumentOrErrorOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function listDatabasesOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function replaceOneOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function startTransactionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function targetedFailPointOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function deleteOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function downloadOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function uploadOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} +async function withTransactionOperation( + entities: EntitiesMap, + op: uni.OperationDescription +): Promise { + throw new Error('not implemented.'); +} + +type RunOperationFn = (entities: EntitiesMap, op: uni.OperationDescription) => Promise; +export const operations = new Map(); + +operations.set('abortTransaction', abortTransactionOperation); +operations.set('aggregate', aggregateOperation); +operations.set('assertCollectionExists', assertCollectionExistsOperation); +operations.set('assertCollectionNotExists', assertCollectionNotExistsOperation); +operations.set('assertIndexExists', assertIndexExistsOperation); +operations.set('assertIndexNotExists', assertIndexNotExistsOperation); +operations.set( + 'assertDifferentLsidOnLastTwoCommands', + assertDifferentLsidOnLastTwoCommandsOperation +); +operations.set('assertSameLsidOnLastTwoCommands', assertSameLsidOnLastTwoCommandsOperation); +operations.set('assertSessionDirty', assertSessionDirtyOperation); +operations.set('assertSessionNotDirty', assertSessionNotDirtyOperation); +operations.set('assertSessionPinned', assertSessionPinnedOperation); +operations.set('assertSessionUnpinned', assertSessionUnpinnedOperation); +operations.set('assertSessionTransactionState', assertSessionTransactionStateOperation); +operations.set('bulkWrite', bulkWriteOperation); +operations.set('commitTransaction', commitTransactionOperation); +operations.set('createChangeStream', createChangeStreamOperation); +operations.set('createCollection', createCollectionOperation); +operations.set('createIndex', createIndexOperation); +operations.set('deleteOne', deleteOneOperation); +operations.set('dropCollection', dropCollectionOperation); +operations.set('endSession', endSessionOperation); +operations.set('find', findOperation); +operations.set('findOneAndReplace', findOneAndReplaceOperation); +operations.set('findOneAndUpdate', findOneAndUpdateOperation); +operations.set('failPoint', failPointOperation); +operations.set('insertOne', insertOneOperation); +operations.set('insertMany', insertManyOperation); +operations.set('iterateUntilDocumentOrError', iterateUntilDocumentOrErrorOperation); +operations.set('listDatabases', listDatabasesOperation); +operations.set('replaceOne', replaceOneOperation); +operations.set('startTransaction', startTransactionOperation); +operations.set('targetedFailPoint', targetedFailPointOperation); +operations.set('delete', deleteOperation); +operations.set('download', downloadOperation); +operations.set('upload', uploadOperation); +operations.set('withTransaction', withTransactionOperation); + +export async function executeOperationAndCheck( + operation: uni.OperationDescription, + entities: EntitiesMap +): Promise { + const operationName = operation.name; + const opFunc = operations.get(operationName); + expect(opFunc, `Unknown operation: ${operationName}`).to.exist; + try { + const result = await opFunc(entities, operation); + + if (operation.expectError) { + expect.fail(`Operation ${operationName} succeeded but was not supposed to`); + } + + if (operation.expectResult) { + if (isSpecialOperator(operation.expectResult)) { + specialCheck(result, operation.expectResult); + } else { + for (const [resultKey, resultValue] of Object.entries(operation.expectResult)) { + // each key/value expectation can be a special op + if (isSpecialOperator(resultValue)) { + specialCheck(result, resultValue); + } else { + expect(result[resultKey]).to.deep.equal(resultValue); + } + } + } + } + + if (operation.saveResultAsEntity) { + entities.set(operation.saveResultAsEntity, result); + } + } catch (error) { + if (operation.expectError) { + expect(error).to.be.instanceof(MongoError); + // TODO more checking of the error + } else { + expect.fail(`Operation ${operationName} failed with ${error.message}`); + } + } +} + +export function specialCheck(result: Document, check: SpecialOperator): void { + if (isUnsetOrMatchesOperator(check)) { + if (result == null) return; // acceptable unset + if (typeof check.$$unsetOrMatches === 'object') { + // We need to a "deep equals" check but the props can also point to special checks + for (const [k, v] of Object.entries(check.$$unsetOrMatches)) { + expect(result).to.have.property(k); + if (isSpecialOperator(v)) { + specialCheck(result[k], v); + } else { + expect(v).to.equal(check.$$unsetOrMatches); + } + } + } else { + expect(result).to.equal(check.$$unsetOrMatches); + } + } else if (isExistsOperator(check)) { + throw new Error('not implemented.'); + } else if (isMatchesEntityOperator(check)) { + throw new Error('not implemented.'); + } else if (isMatchesHexBytesOperator(check)) { + throw new Error('not implemented.'); + } else if (isSessionLsidOperator(check)) { + throw new Error('not implemented.'); + } else if (isTypeOperator(check)) { + throw new Error('not implemented.'); + } else { + throw new Error('not implemented.'); + } +} diff --git a/test/functional/unified-spec-runner/schema.ts b/test/functional/unified-spec-runner/schema.ts new file mode 100644 index 00000000000..55526faf025 --- /dev/null +++ b/test/functional/unified-spec-runner/schema.ts @@ -0,0 +1,146 @@ +import type { Document } from '../../../src/bson'; +import type { ReadConcernLevelId } from '../../../src/read_concern'; +import type { ReadPreferenceModeId } from '../../../src/read_preference'; +import type { TagSet } from '../../../src/sdam/server_description'; +import type { W } from '../../../src/write_concern'; + +export const SupportedVersion = '^1.0'; + +export interface OperationDescription { + name: string; + object: string; + arguments: Document; + expectError?: ExpectedError; + expectResult?: unknown; + saveResultAsEntity?: string; +} +export interface UnifiedSuite { + description: string; + schemaVersion: string; + runOnRequirements?: [RunOnRequirement, ...RunOnRequirement[]]; + createEntities?: [EntityDescription, ...EntityDescription[]]; + initialData?: [CollectionData, ...CollectionData[]]; + tests: [Test, ...Test[]]; + _yamlAnchors?: Document; +} +export const TopologyType = { + single: 'single', + replicaset: 'replicaset', + sharded: 'sharded', + shardedReplicaset: 'sharded-replicaset' +} as const; +export type TopologyId = typeof TopologyType[keyof typeof TopologyType]; +export interface RunOnRequirement { + maxServerVersion?: string; + minServerVersion?: string; + topologies?: TopologyId[]; + serverParameters?: Document; +} +export type ObservableEventId = + | 'commandStartedEvent' + | 'commandSucceededEvent' + | 'commandFailedEvent'; + +export interface ClientEntity { + id: string; + uriOptions?: Document; + useMultipleMongoses?: boolean; + observeEvents?: ObservableEventId[]; + ignoreCommandMonitoringEvents?: [string, ...string[]]; + serverApi?: ServerApi; +} +export interface DatabaseEntity { + id: string; + client: string; + databaseName: string; + databaseOptions?: CollectionOrDatabaseOptions; +} +export interface CollectionEntity { + id: string; + database: string; + collectionName: string; + collectionOptions?: CollectionOrDatabaseOptions; +} +export interface SessionEntity { + id: string; + client: string; + sessionOptions?: Document; +} +export interface BucketEntity { + id: string; + database: string; + bucketOptions?: Document; +} +export interface StreamEntity { + id: string; + hexBytes: string; +} +export type EntityDescription = + | { client: ClientEntity } + | { database: DatabaseEntity } + | { collection: CollectionEntity } + | { bucket: BucketEntity } + | { stream: StreamEntity } + | { session: SessionEntity }; +export interface ServerApi { + version: string; + strict?: boolean; + deprecationErrors?: boolean; +} +export interface CollectionOrDatabaseOptions { + readConcern?: { + level: ReadConcernLevelId; + }; + readPreference?: { + mode: ReadPreferenceModeId; + maxStalenessSeconds: number; + tags: TagSet[]; + hedge: { enabled: boolean }; + }; + writeConcern?: { + w: W; + wtimeoutMS: number; + journal: boolean; + }; +} +export interface CollectionData { + collectionName: string; + databaseName: string; + documents: Document[]; +} +export interface Test { + description: string; + runOnRequirements?: [RunOnRequirement, ...RunOnRequirement[]]; + skipReason?: string; + operations: OperationDescription[]; + expectEvents?: ExpectedEventsForClient[]; + outcome?: [CollectionData, ...CollectionData[]]; +} +export interface ExpectedEventsForClient { + client: string; + events: ExpectedEvent[]; +} +export interface ExpectedEvent { + commandStartedEvent?: { + command?: Document; + commandName?: string; + databaseName?: string; + }; + commandSucceededEvent?: { + reply?: Document; + commandName?: string; + }; + commandFailedEvent?: { + commandName?: string; + }; +} +export interface ExpectedError { + isError?: true; + isClientError?: boolean; + errorContains?: string; + errorCode?: number; + errorCodeName?: string; + errorLabelsContain?: [string, ...string[]]; + errorLabelsOmit?: [string, ...string[]]; + expectResult?: unknown; +} diff --git a/test/functional/unified-spec-runner/unified-utils.ts b/test/functional/unified-spec-runner/unified-utils.ts new file mode 100644 index 00000000000..572792f82c4 --- /dev/null +++ b/test/functional/unified-spec-runner/unified-utils.ts @@ -0,0 +1,168 @@ +import { expect } from 'chai'; +import { + CommandFailedEvent, + CommandStartedEvent, + CommandSucceededEvent +} from '../../../src/cmap/events'; +import type { CommandEvent } from './entities'; +import type { CollectionOrDatabaseOptions, ExpectedEvent, RunOnRequirement } from './schema'; +import type { TestConfiguration } from './unified.test'; +import { gte as semverGte, lte as semverLte } from 'semver'; +import { CollectionOptions, DbOptions } from '../../../src'; + +const ENABLE_UNIFIED_TEST_LOGGING = false; +export function log(message: unknown, ...optionalParameters: unknown[]): void { + if (ENABLE_UNIFIED_TEST_LOGGING) console.warn(message, ...optionalParameters); +} + +export function getUnmetRequirements(config: TestConfiguration, r: RunOnRequirement): boolean { + let ok = true; + if (r.minServerVersion) { + const minVersion = patchVersion(r.minServerVersion); + ok &&= semverGte(config.version, minVersion); + } + if (r.maxServerVersion) { + const maxVersion = patchVersion(r.maxServerVersion); + ok &&= semverLte(config.version, maxVersion); + } + + if (r.topologies) { + const topologyType = { + Single: 'single', + ReplicaSetNoPrimary: 'replicaset', + ReplicaSetWithPrimary: 'replicaset', + Sharded: 'sharded' + }[config.topologyType]; + if (!topologyType) throw new Error(`Topology undiscovered: ${config.topologyType}`); + ok &&= r.topologies.includes(topologyType); + } + + if (r.serverParameters) { + // for (const [name, value] of Object.entries(r.serverParameters)) { + // // TODO + // } + } + + return ok; +} + +/** Turns two lists into a joined list of tuples. Uses longer array length */ +export function* zip( + iter1: T[], + iter2: U[] +): Generator<[T | undefined, U | undefined], void> { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const longerArrayLength = Math.max(iter1.length, iter2.length); + for (let index = 0; index < longerArrayLength; index++) { + yield [iter1[index], iter2[index]]; + } +} + +export function matchesEvents(expected: ExpectedEvent[], actual: CommandEvent[]): void { + expect(expected).to.have.lengthOf(actual.length); + + for (const [index, actualEvent] of actual.entries()) { + const expectedEvent = expected[index]; + + if (expectedEvent.commandStartedEvent && actualEvent instanceof CommandStartedEvent) { + expect(actualEvent.commandName).to.equal(expectedEvent.commandStartedEvent.commandName); + expect(actualEvent.command).to.containSubset(expectedEvent.commandStartedEvent.command); + expect(actualEvent.databaseName).to.equal(expectedEvent.commandStartedEvent.databaseName); + } else if ( + expectedEvent.commandSucceededEvent && + actualEvent instanceof CommandSucceededEvent + ) { + expect(actualEvent.commandName).to.equal(expectedEvent.commandSucceededEvent.commandName); + expect(actualEvent.reply).to.containSubset(expectedEvent.commandSucceededEvent.reply); + } else if (expectedEvent.commandFailedEvent && actualEvent instanceof CommandFailedEvent) { + expect(actualEvent.commandName).to.equal(expectedEvent.commandFailedEvent.commandName); + } else { + expect.fail(`Events must be one of the known types, got ${actualEvent}`); + } + } +} + +/** Correct schema version to be semver compliant */ +export function patchVersion(version: string): string { + expect(version).to.be.a('string'); + const [major, minor, patch] = version.split('.'); + return `${major}.${minor ?? 0}.${patch ?? 0}`; +} + +export function patchDbOptions(options: CollectionOrDatabaseOptions): DbOptions { + // TODO + return { ...options } as DbOptions; +} + +export function patchCollectionOptions(options: CollectionOrDatabaseOptions): CollectionOptions { + // TODO + return { ...options } as CollectionOptions; +} + +export interface ExistsOperator { + $$exists: boolean; +} +export function isExistsOperator(value: unknown): value is ExistsOperator { + return typeof value === 'object' && value != null && '$$exists' in value; +} +export interface TypeOperator { + $$type: boolean; +} +export function isTypeOperator(value: unknown): value is TypeOperator { + return typeof value === 'object' && value != null && '$$type' in value; +} +export interface MatchesEntityOperator { + $$matchesEntity: string; +} +export function isMatchesEntityOperator(value: unknown): value is MatchesEntityOperator { + return typeof value === 'object' && value != null && '$$matchesEntity' in value; +} +export interface MatchesHexBytesOperator { + $$matchesHexBytes: string; +} +export function isMatchesHexBytesOperator(value: unknown): value is MatchesHexBytesOperator { + return typeof value === 'object' && value != null && '$$matchesHexBytes' in value; +} +export interface UnsetOrMatchesOperator { + $$unsetOrMatches: unknown; +} +export function isUnsetOrMatchesOperator(value: unknown): value is UnsetOrMatchesOperator { + return typeof value === 'object' && value != null && '$$unsetOrMatches' in value; +} +export interface SessionLsidOperator { + $$sessionLsid: unknown; +} +export function isSessionLsidOperator(value: unknown): value is SessionLsidOperator { + return typeof value === 'object' && value != null && '$$sessionLsid' in value; +} + +export const SpecialOperatorKeys = [ + '$$exists', + '$$type', + '$$matchesEntity', + '$$matchesHexBytes', + '$$unsetOrMatches', + '$$sessionLsid' +]; + +export type SpecialOperator = + | ExistsOperator + | TypeOperator + | MatchesEntityOperator + | MatchesHexBytesOperator + | UnsetOrMatchesOperator + | SessionLsidOperator; + +// eslint-disable-next-line @typescript-eslint/ban-types +type KeysOfUnion = T extends object ? keyof T : never; +export type SpecialOperatorKey = KeysOfUnion; +export function isSpecialOperator(value: unknown): value is SpecialOperator { + return ( + isExistsOperator(value) || + isTypeOperator(value) || + isMatchesEntityOperator(value) || + isMatchesHexBytesOperator(value) || + isUnsetOrMatchesOperator(value) || + isSessionLsidOperator(value) + ); +} diff --git a/test/functional/unified-spec-runner/unified.test.ts b/test/functional/unified-spec-runner/unified.test.ts new file mode 100644 index 00000000000..82e8d3cfe39 --- /dev/null +++ b/test/functional/unified-spec-runner/unified.test.ts @@ -0,0 +1,166 @@ +import { expect } from 'chai'; +import { ReadPreference } from '../../../src/read_preference'; +import { loadSpecTests } from '../../spec/index'; +import * as uni from './schema'; +import { getUnmetRequirements, matchesEvents, patchVersion, zip, log } from './unified-utils'; +import { EntitiesMap } from './entities'; +import { ns } from '../../../src/utils'; +import { executeOperationAndCheck } from './operations'; +import { satisfies as semverSatisfies } from 'semver'; + +export type TestConfiguration = InstanceType< + typeof import('../../tools/runner/config')['TestConfiguration'] +>; +interface MongoDBMochaTestContext extends Mocha.Context { + configuration: TestConfiguration; +} + +async function runOne( + ctx: MongoDBMochaTestContext, + unifiedSuite: uni.UnifiedSuite, + test: uni.Test +) { + // Some basic expectations we can catch early + expect(test).to.exist; + expect(unifiedSuite).to.exist; + expect(ctx).to.exist; + expect(ctx.configuration).to.exist; + + // If test.skipReason is specified, the test runner MUST skip this + // test and MAY use the string value to log a message. + if (test.skipReason) { + console.warn(`Skipping test ${test.description}: ${test.skipReason}.`); + ctx.skip(); + } + + const UTIL_CLIENT = ctx.configuration.newClient(); + await UTIL_CLIENT.connect(); + ctx.defer(async () => await UTIL_CLIENT.close()); + + // If test.runOnRequirements is specified, the test runner MUST skip the test unless one or more + // runOnRequirement objects are satisfied. + if (test.runOnRequirements) { + if (!test.runOnRequirements.some(r => getUnmetRequirements(ctx.configuration, r))) { + ctx.skip(); + } + } + + // If initialData is specified, for each collectionData therein the test runner MUST drop the + // collection and insert the specified documents (if any) using a "majority" write concern. If no + // documents are specified, the test runner MUST create the collection with a "majority" write concern. + // The test runner MUST use the internal MongoClient for these operations. + if (unifiedSuite.initialData) { + for (const collData of unifiedSuite.initialData) { + const db = UTIL_CLIENT.db(collData.databaseName); + const collection = db.collection(collData.collectionName, { + writeConcern: { w: 'majority' } + }); + const collectionList = await db.listCollections({ name: collData.collectionName }).toArray(); + if (collectionList.length !== 0) { + expect(await collection.drop()).to.be.true; + } + + if (collData.documents.length === 0) { + await db.createCollection(collData.collectionName, { + writeConcern: { w: 'majority' } + }); + continue; + } + + await collection.insertMany(collData.documents); + } + } + + const entities = await EntitiesMap.createEntities(ctx.configuration, unifiedSuite.createEntities); + ctx.defer(async () => await entities.cleanup()); + + // Workaround for SERVER-39704: + // test runners MUST execute a non-transactional distinct command on + // each mongos server before running any test that might execute distinct within a transaction. + // To ease the implementation, test runners MAY execute distinct before every test. + if ( + ctx.topologyType === uni.TopologyType.sharded || + ctx.topologyType === uni.TopologyType.shardedReplicaset + ) { + for (const [, collection] of entities.mapOf('collection')) { + await UTIL_CLIENT.db(ns(collection.namespace).db).command({ + distinct: collection.collectionName, + key: '_id' + }); + } + } + + for (const operation of test.operations) { + await executeOperationAndCheck(operation, entities); + } + + const clientEvents = new Map(); + // If any event listeners were enabled on any client entities, + // the test runner MUST now disable those event listeners. + for (const [id, client] of entities.mapOf('client')) { + clientEvents.set(id, client.stopCapturingEvents()); + } + + if (test.expectEvents) { + for (const expectedEventList of test.expectEvents) { + const clientId = expectedEventList.client; + const actualEvents = clientEvents.get(clientId); + + expect(actualEvents, `No client entity found with id ${clientId}`).to.exist; + matchesEvents(expectedEventList.events, actualEvents); + } + } + + if (test.outcome) { + for (const collectionData of test.outcome) { + const collection = UTIL_CLIENT.db(collectionData.databaseName).collection( + collectionData.collectionName + ); + const findOpts = { + readConcern: 'local' as const, + readPreference: ReadPreference.primary, + sort: { _id: 'asc' as const } + }; + const documents = await collection.find({}, findOpts).toArray(); + + expect(documents).to.have.lengthOf(collectionData.documents.length); + for (const [expected, actual] of zip(collectionData.documents, documents)) { + expect(actual).to.include(expected, 'Test outcome did not match expected'); + } + } + } +} + +describe('Unified test format', function unifiedTestRunner() { + // Valid tests that should pass + for (const unifiedSuite of loadSpecTests('unified-test-format/valid-pass')) { + const schemaVersion = patchVersion(unifiedSuite.schemaVersion); + expect(semverSatisfies(schemaVersion, uni.SupportedVersion)).to.be.true; + context(String(unifiedSuite.description), function runUnifiedTest() { + for (const test of unifiedSuite.tests) { + it(String(test.description), async function runOneUnifiedTest() { + try { + await runOne(this as MongoDBMochaTestContext, unifiedSuite, test); + } catch (error) { + if (error.message.includes('not implemented.')) { + log(`${test.description}: was skipped due to missing functionality`); + this.skip(); + } else { + throw error; + } + } + }); + } + }); + } + + // Valid tests that should fail + // for (const unifiedSuite of loadSpecTests('unified-test-format/valid-fail')) { + // // TODO + // } + + // Tests that are invalid, would be good to gracefully fail on + // for (const unifiedSuite of loadSpecTests('unified-test-format/invalid')) { + // // TODO + // } +}); diff --git a/test/spec/index.js b/test/spec/index.js index c857af84ab3..d5a8c028b8e 100644 --- a/test/spec/index.js +++ b/test/spec/index.js @@ -3,17 +3,22 @@ const path = require('path'); const fs = require('fs'); const { EJSON } = require('bson'); -function loadSpecTests() { - const specPath = path.resolve.apply(null, [__dirname].concat(Array.from(arguments))); +/** + * Given spec test folder names, loads the corresponding JSON + * + * @param {...string} args - the spec test name to load + * @returns {any[]} + */ +function loadSpecTests(...args) { + const specPath = path.resolve(...[__dirname].concat(args)); return fs .readdirSync(specPath) - .filter(x => x.indexOf('.json') !== -1) - .map(x => - Object.assign(EJSON.parse(fs.readFileSync(path.join(specPath, x)), { relaxed: true }), { - name: path.basename(x, '.json') - }) - ); + .filter(x => x.includes('.json')) + .map(x => ({ + ...EJSON.parse(fs.readFileSync(path.join(specPath, x)), { relaxed: true }), + name: path.basename(x, '.json') + })); } module.exports = { diff --git a/test/tools/runner/index.js b/test/tools/runner/index.js index d1d13b32655..a1c2929ff12 100644 --- a/test/tools/runner/index.js +++ b/test/tools/runner/index.js @@ -95,6 +95,7 @@ require('./plugins/client_leak_checker'); require('mocha-sinon'); const chai = require('chai'); chai.use(require('sinon-chai')); +chai.use(require('chai-subset')); chai.use(require('../../functional/spec-runner/matcher').default); chai.config.includeStack = true; chai.config.showDiff = true;