From 896b9fcdc398ea81b2bca7a067b8713eb3f777da Mon Sep 17 00:00:00 2001 From: emadum Date: Mon, 12 Jul 2021 11:16:45 -0400 Subject: [PATCH 01/13] chore(NODE-3072): serverless testing --- .evergreen/config.yml | 82 ++++++++++++++++--- .evergreen/config.yml.in | 56 +++++++++++++ .evergreen/generate_evergreen_tasks.js | 11 +++ .evergreen/run-serverless-tests.sh | 13 +++ test/functional/spec-runner/index.js | 7 +- test/tools/runner/config.js | 4 + .../tools/runner/filters/serverless_filter.js | 51 ++++++++++++ test/tools/runner/index.js | 7 ++ 8 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 .evergreen/run-serverless-tests.sh create mode 100755 test/tools/runner/filters/serverless_filter.js diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 255ca39bf62..2465cad1a4b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -134,6 +134,20 @@ functions: MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + run serverless tests: + - command: shell.exec + type: test + params: + working_dir: src + script: | + ${PREPARE_SHELL} + set +o xtrace + MONGODB_URI="${MONGODB_URI}" \ + MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ + AUTH="auth" SSL="ssl" SERVERLESS=1 \ + SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \ + SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-serverless-tests.sh run checks: - command: shell.exec type: test @@ -568,18 +582,12 @@ functions: - command: attach.xunit_results params: file: src/xunit.xml -pre: - - func: fetch source - - func: prepare resources - - func: windows fix - - func: fix absolute paths - - func: make files executable -post: - - func: upload test results - - func: cleanup -ignore: - - '*.md' tasks: + - name: test-serverless + tags: + - serverless + commands: + - func: run serverless tests - name: test-latest-server tags: - latest @@ -1638,6 +1646,51 @@ tasks: - func: run bson-ext test vars: NODE_LTS_NAME: erbium +task_groups: + - name: serverless_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + setup_group: + - func: fetch source + - func: prepare resources + - command: shell.exec + params: + shell: bash + script: | + ${PREPARE_SHELL} + set +o xtrace + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_group: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + set +o xtrace + #SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + #SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + #SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + #SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ + # bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + echo "SKIPPING DELETE TO EASE TESTING" + tasks: + - .serverless +pre: + - func: fetch source + - func: prepare resources + - func: windows fix + - func: fix absolute paths + - func: make files executable +post: + - func: upload test results + - func: cleanup +ignore: + - '*.md' buildvariants: - name: macos-1014-erbium display_name: macOS 10.14 Node Erbium @@ -1923,3 +1976,10 @@ buildvariants: - aws-4.4-auth-test-run-aws-auth-test-with-aws-credentials-as-environment-variables - aws-4.4-auth-test-run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables - aws-4.4-auth-test-run-aws-ECS-auth-test + - name: ubuntu1804-test-serverless + display_name: Serverless Test + run_on: ubuntu1804-test + expansions: + NODE_LTS_NAME: erbium + tasks: + - serverless_task_group diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index 54a5d0cee14..00920fbb7cc 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -155,6 +155,21 @@ functions: NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + "run serverless tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + set +o xtrace + MONGODB_URI="${MONGODB_URI}" \ + MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ + AUTH="auth" SSL="ssl" SERVERLESS=1 \ + SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \ + SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-serverless-tests.sh + "run checks": - command: shell.exec type: test @@ -608,6 +623,47 @@ functions: params: file: "src/xunit.xml" +tasks: + - name: "test-serverless" + tags: ["serverless"] + commands: + - func: "run serverless tests" + +task_groups: + - name: serverless_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: "fetch source" + - func: "prepare resources" + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + set +o xtrace + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_group: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + set +o xtrace + #SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + #SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + #SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + #SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ + # bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + echo "SKIPPING DELETE TO EASE TESTING" + tasks: + - ".serverless" + pre: - func: "fetch source" - func: "prepare resources" diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 4a48f671615..5583b23553f 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -607,6 +607,17 @@ SINGLETON_TASKS.push({ ] }); +// special case for serverless testing +BUILD_VARIANTS.push({ + name: 'ubuntu1804-test-serverless', + display_name: 'Serverless Test', + run_on: 'ubuntu1804-test', + expansions: { + NODE_LTS_NAME: LOWEST_LTS + }, + tasks: ['serverless_task_group'] +}); + const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8')); fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS); fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS); diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh new file mode 100644 index 00000000000..589a595f68e --- /dev/null +++ b/.evergreen/run-serverless-tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source "${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" + +SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \ +SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" \ +SERVERLESS=1 AUTH=auth SSL=ssl \ +MONGODB_URI=${MONGODB_URI} npx mocha \ + test/functional/crud_spec.test.js \ + test/functional/retryable_reads.test.js \ + test/functional/retryable_writes.test.js \ + test/functional/versioned-api.test.js \ + test/functional/sessions.test.js \ + test/functional/transactions.test.js \ No newline at end of file diff --git a/test/functional/spec-runner/index.js b/test/functional/spec-runner/index.js index 7a30de025d8..25fdd7c9ede 100644 --- a/test/functional/spec-runner/index.js +++ b/test/functional/spec-runner/index.js @@ -110,7 +110,12 @@ function parseRunOn(runOn) { } const mongodb = version.join(' '); - return { topology, mongodb, authEnabled: !!config.authEnabled }; + return { + topology, + mongodb, + authEnabled: !!config.authEnabled, + serverless: config.serverless + }; }); } diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 4943a2da1b9..50ca34740d6 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -58,6 +58,10 @@ class TestConfiguration { password: url.password }; } + if (context.serverlessCredentials) { + const { username, password } = context.serverlessCredentials; + this.options.auth = { username, password }; + } } writeConcern() { diff --git a/test/tools/runner/filters/serverless_filter.js b/test/tools/runner/filters/serverless_filter.js new file mode 100755 index 00000000000..b08edbada8c --- /dev/null +++ b/test/tools/runner/filters/serverless_filter.js @@ -0,0 +1,51 @@ +'use strict'; + +/** + * Filter to allow to tests to run on serverless + * + * example: + * metadata: { + * requires: { + * serverless: 'forbid' + * } + * } + */ +class ServerlessFilter { + constructor() { + // Get environmental variables that are known + this.serverless = !!process.env.SERVERLESS; + } + + initializeFilter(client, context, callback) { + if (this.serverless) { + console.log('saving serverless credentials in test filter'); + context.serverlessCredentials = { + username: process.env.SERVERLESS_ATLAS_USER, + password: process.env.SERVERLESS_ATLAS_PASSWORD + }; + } + callback(); + } + + filter(test) { + if (!test.metadata) return true; + if (!test.metadata.requires) return true; + const serverless = test.metadata.requires.serverless; + if (!serverless) return true; + switch (serverless) { + case 'forbid': + // return true if the configuration is NOT serverless + return !this.serverless; + case 'allow': + // always return true + return true; + case 'require': + // only return true if the configuration is serverless + return this.serverless; + default: + throw new Error(`Invalid serverless filter: ${serverless}`); + } + } +} + +module.exports = ServerlessFilter; diff --git a/test/tools/runner/index.js b/test/tools/runner/index.js index 313f72688e1..6f1872a2b0b 100644 --- a/test/tools/runner/index.js +++ b/test/tools/runner/index.js @@ -62,6 +62,13 @@ before(function (_done) { // ); const options = MONGODB_API_VERSION ? { serverApi: MONGODB_API_VERSION } : {}; + if (process.env.SERVERLESS) { + options.auth = { + username: process.env.SERVERLESS_ATLAS_USER, + password: process.env.SERVERLESS_ATLAS_PASSWORD + }; + console.log('set serverless auth in before hook', { auth: options.auth }); + } const client = new MongoClient(MONGODB_URI, options); const done = err => client.close(err2 => _done(err || err2)); From e518d7c904ce6f37709f863c53ee245188516857 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 14 Jul 2021 13:29:08 -0400 Subject: [PATCH 02/13] refactor: extract getEnvironmentalOptions utility --- .../unified-spec-runner/entities.ts | 13 ++++------- test/tools/runner/config.js | 12 +++++----- test/tools/runner/index.js | 17 ++------------ test/tools/utils.js | 23 ++++++++++++++++++- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 1fcdcd50273..995a1e62ae4 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -20,6 +20,8 @@ import type { import { patchCollectionOptions, patchDbOptions } from './unified-utils'; import { expect } from 'chai'; import { TestConfiguration } from './runner'; +import { getEnvironmentalOptions } from '../../tools/utils'; +import { MongoClientOptions } from '../../../src/mongo_client'; interface UnifiedChangeStream extends ChangeStream { eventCollector: InstanceType; @@ -27,15 +29,8 @@ interface UnifiedChangeStream extends ChangeStream { export type CommandEvent = CommandStartedEvent | CommandSucceededEvent | CommandFailedEvent; -function serverApiConfig() { - if (process.env.MONGODB_API_VERSION) { - return { version: process.env.MONGODB_API_VERSION }; - } -} - function getClient(address) { - const serverApi = serverApiConfig(); - return new MongoClient(`mongodb://${address}`, serverApi ? { serverApi } : {}); + return new MongoClient(`mongodb://${address}`, getEnvironmentalOptions() as MongoClientOptions); } export class UnifiedMongoClient extends MongoClient { @@ -54,7 +49,7 @@ export class UnifiedMongoClient extends MongoClient { super(url, { monitorCommands: true, ...description.uriOptions, - serverApi: description.serverApi ? description.serverApi : serverApiConfig() + ...getEnvironmentalOptions() }); this.events = []; this.failPoints = []; diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 50ca34740d6..0f66df13094 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -8,6 +8,7 @@ const { MongoClient } = require('../../../src/mongo_client'); const { Topology } = require('../../../src/sdam/topology'); const { TopologyType } = require('../../../src/sdam/common'); const { HostAddress } = require('../../../src/utils'); +const { getEnvironmentalOptions } = require('../utils'); /** * @typedef {Object} UrlOptions @@ -41,7 +42,6 @@ class TestConfiguration { this.topologyType = context.topologyType; this.version = context.version; this.clientSideEncryption = context.clientSideEncryption; - this.serverApi = context.serverApi; this.parameters = undefined; this.options = { hosts, @@ -102,13 +102,13 @@ class TestConfiguration { } newClient(dbOptions, serverOptions) { - const defaultOptions = { minHeartbeatFrequencyMS: 100 }; - if (this.serverApi) { - Object.assign(defaultOptions, { serverApi: this.serverApi }); - } + const defaultOptions = Object.assign( + { minHeartbeatFrequencyMS: 100 }, + getEnvironmentalOptions() + ); // support MongoClient constructor form (url, options) for `newClient` if (typeof dbOptions === 'string') { - return new MongoClient(dbOptions, Object.assign(defaultOptions, serverOptions)); + return new MongoClient(dbOptions, Object.assign({}, defaultOptions, serverOptions)); } dbOptions = dbOptions || {}; diff --git a/test/tools/runner/index.js b/test/tools/runner/index.js index 6f1872a2b0b..98f54b17124 100644 --- a/test/tools/runner/index.js +++ b/test/tools/runner/index.js @@ -4,12 +4,12 @@ const path = require('path'); const fs = require('fs'); const { MongoClient } = require('../../../src'); const { TestConfiguration } = require('./config'); +const { getEnvironmentalOptions } = require('../utils'); const { eachAsync } = require('../../../src/utils'); const mock = require('../mock'); const wtfnode = require('wtfnode'); const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017'; -const MONGODB_API_VERSION = process.env.MONGODB_API_VERSION; const filters = []; function initializeFilters(client, callback) { @@ -61,15 +61,7 @@ before(function (_done) { // )} topology` // ); - const options = MONGODB_API_VERSION ? { serverApi: MONGODB_API_VERSION } : {}; - if (process.env.SERVERLESS) { - options.auth = { - username: process.env.SERVERLESS_ATLAS_USER, - password: process.env.SERVERLESS_ATLAS_PASSWORD - }; - console.log('set serverless auth in before hook', { auth: options.auth }); - } - const client = new MongoClient(MONGODB_URI, options); + const client = new MongoClient(MONGODB_URI, getEnvironmentalOptions()); const done = err => client.close(err2 => _done(err || err2)); client.connect(err => { @@ -84,11 +76,6 @@ before(function (_done) { return; } - // Ensure test MongoClients set a serverApi parameter when required - if (MONGODB_API_VERSION) { - context.serverApi = MONGODB_API_VERSION; - } - // replace this when mocha supports dynamic skipping with `afterEach` filterOutTests(this._runnable.parent); this.configuration = new TestConfiguration(MONGODB_URI, context); diff --git a/test/tools/utils.js b/test/tools/utils.js index 88930825be4..9017087b02e 100644 --- a/test/tools/utils.js +++ b/test/tools/utils.js @@ -258,6 +258,26 @@ function getSymbolFrom(target, symbolName, assertExists = true) { return symbol; } +function getEnvironmentalOptions() { + const options = {}; + if (process.env.MONGODB_API_VERSION) { + Object.assign(options, { + serverApi: { version: process.env.MONGODB_API_VERSION } + }); + } + if (process.env.SERVERLESS) { + Object.assign(options, { + auth: { + username: process.env.SERVERLESS_ATLAS_USER, + password: process.env.SERVERLESS_ATLAS_PASSWORD + }, + tls: true, + compressors: 'snappy,zlib' + }); + } + return options; +} + module.exports = { EventCollector, makeTestFunction, @@ -266,5 +286,6 @@ module.exports = { ClassWithoutLogger, ClassWithUndefinedLogger, visualizeMonitoringEvents, - getSymbolFrom + getSymbolFrom, + getEnvironmentalOptions }; From 83ed969c34063d226b376d4465dbef32ca1c3b63 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 14 Jul 2021 17:43:53 -0400 Subject: [PATCH 03/13] get test runners working with serverless --- test/functional/crud_spec.test.js | 15 --------------- test/functional/spec-runner/context.js | 5 +++++ test/functional/spec-runner/index.js | 3 +++ test/functional/spec-runner/utils.js | 2 +- test/functional/unified-spec-runner/entities.ts | 7 +++++-- test/tools/runner/config.js | 2 +- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/test/functional/crud_spec.test.js b/test/functional/crud_spec.test.js index 14c4da4244a..2cda5ac4b22 100644 --- a/test/functional/crud_spec.test.js +++ b/test/functional/crud_spec.test.js @@ -6,10 +6,6 @@ const chai = require('chai'); const expect = chai.expect; chai.use(require('chai-subset')); -const TestRunnerContext = require('./spec-runner').TestRunnerContext; -const gatherTestSuites = require('./spec-runner').gatherTestSuites; -const generateTopologyTests = require('./spec-runner').generateTopologyTests; - const { loadSpecTests } = require('../spec/index'); const { runUnifiedTest } = require('./unified-spec-runner/runner'); @@ -425,17 +421,6 @@ describe('CRUD spec', function () { } }); -describe('CRUD v2', function () { - const testContext = new TestRunnerContext(); - const testSuites = gatherTestSuites(path.resolve(__dirname, '../spec/crud/v2')); - after(() => testContext.teardown()); - before(function () { - return testContext.setup(this.configuration); - }); - - generateTopologyTests(testSuites, testContext); -}); - describe('CRUD unified', function () { for (const crudSpecTest of loadSpecTests('crud/unified')) { expect(crudSpecTest).to.exist; diff --git a/test/functional/spec-runner/context.js b/test/functional/spec-runner/context.js index a696d5f4aa6..f4f0de980f9 100644 --- a/test/functional/spec-runner/context.js +++ b/test/functional/spec-runner/context.js @@ -44,6 +44,11 @@ class TestRunnerContext { this.user = opts.user; this.password = opts.password; this.authSource = opts.authSource; + if (process.env.SERVERLESS) { + this.user = process.env.SERVERLESS_ATLAS_USER; + this.password = process.env.SERVERLESS_ATLAS_PASSWORD; + this.authSource = 'admin'; + } this.sharedClient = null; this.failPointClients = []; this.appliedFailPoints = []; diff --git a/test/functional/spec-runner/index.js b/test/functional/spec-runner/index.js index 25fdd7c9ede..f1c53cfb0e8 100644 --- a/test/functional/spec-runner/index.js +++ b/test/functional/spec-runner/index.js @@ -201,6 +201,9 @@ function prepareDatabaseForSuite(suite, context) { if (err.message.match(/no such (cmd|command)/) || err.code === 11601) { return; } + if (process.env.SERVERLESS) { + return; + } throw err; }); diff --git a/test/functional/spec-runner/utils.js b/test/functional/spec-runner/utils.js index a4112f2cc08..c4664ac1564 100644 --- a/test/functional/spec-runner/utils.js +++ b/test/functional/spec-runner/utils.js @@ -10,7 +10,7 @@ function resolveConnectionString(configuration, spec, context) { isShardedEnvironment && !useMultipleMongoses ? `mongodb://${configuration.host}:${configuration.port}/${ configuration.db - }?directConnection=false${authSource ? '&authSource=${authSource}' : ''}` + }?directConnection=false${authSource ? `&authSource=${authSource}` : ''}` : configuration.url({ username, password, authSource }); return connectionString; } diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 77cd51bad54..f7f0cb86854 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -108,7 +108,8 @@ export class UnifiedMongoClient extends MongoClient { this.failPoints = []; this.ignoredEvents = [ ...(description.ignoreCommandMonitoringEvents ?? []), - 'configureFailPoint' + 'configureFailPoint', + 'ping' ]; this.observedCommandEvents = (description.observeEvents ?? []) .map(e => UnifiedMongoClient.COMMAND_EVENT_NAME_LOOKUP[e]) @@ -314,7 +315,9 @@ export class EntitiesMap extends Map { const useMultipleMongoses = (config.topologyType === 'LoadBalanced' || config.topologyType === 'Sharded') && entity.client.useMultipleMongoses; - const uri = config.url({ useMultipleMongoses }); + const uri = process.env.SERVERLESS + ? process.env.MONGODB_URI + : config.url({ useMultipleMongoses }); const client = new UnifiedMongoClient(uri, entity.client); await client.connect(); map.set(entity.client.id, client); diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 8b9f27b2434..6bf84467c26 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -62,7 +62,7 @@ class TestConfiguration { } if (context.serverlessCredentials) { const { username, password } = context.serverlessCredentials; - this.options.auth = { username, password }; + this.options.auth = { username, password, authSource: 'admin' }; } } From 7509733e04b61dcc72763817ded07207cf2af797 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 14 Jul 2021 18:57:05 -0400 Subject: [PATCH 04/13] fix versioned api tests --- test/functional/unified-spec-runner/entities.ts | 3 ++- .../unified-spec-runner/unified-utils.ts | 16 ++++++++++++++++ test/tools/runner/config.js | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index f7f0cb86854..337e226157e 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -101,7 +101,8 @@ export class UnifiedMongoClient extends MongoClient { super(url, { monitorCommands: true, ...description.uriOptions, - ...getEnvironmentalOptions() + ...getEnvironmentalOptions(), + ...(description.serverApi ? { serverApi: description.serverApi } : {}) }); this.commandEvents = []; this.cmapEvents = []; diff --git a/test/functional/unified-spec-runner/unified-utils.ts b/test/functional/unified-spec-runner/unified-utils.ts index 9f9ad171db3..5c3ba4b9c27 100644 --- a/test/functional/unified-spec-runner/unified-utils.ts +++ b/test/functional/unified-spec-runner/unified-utils.ts @@ -63,6 +63,22 @@ export async function topologySatisfies( !!utilClient.options.authMechanism; } + if (r.serverless) { + switch (r.serverless) { + case 'forbid': + // return true if the configuration is NOT serverless + return !config.isServerless; + case 'allow': + // always return true + return true; + case 'require': + // only return true if the configuration is serverless + return config.isServerless; + default: + throw new Error(`Invalid serverless filter: ${r.serverless}`); + } + } + return ok; } diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 6bf84467c26..4ba6ca41158 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -45,6 +45,7 @@ class TestConfiguration { this.parameters = undefined; this.singleMongosLoadBalancerUri = context.singleMongosLoadBalancerUri; this.multiMongosLoadBalancerUri = context.multiMongosLoadBalancerUri; + this.isServerless = !!process.env.SERVERLESS; this.options = { hosts, hostAddresses, From 5265a9001fb1930195697afbb63319325ed386a8 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 14 Jul 2021 21:10:26 -0400 Subject: [PATCH 05/13] fix retryable writes spec tests --- .evergreen/config.yml | 11 +++++------ .evergreen/config.yml.in | 11 +++++------ test/functional/retryable_writes.test.js | 2 +- test/functional/unified-spec-runner/entities.ts | 3 +-- test/tools/runner/filters/serverless_filter.js | 1 - 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index bd633ac8469..d27b16fdb69 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1672,12 +1672,11 @@ task_groups: script: | ${PREPARE_SHELL} set +o xtrace - #SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ - #SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ - #SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ - #SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ - # bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - echo "SKIPPING DELETE TO EASE TESTING" + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh tasks: - .serverless pre: diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index cc2124bb32e..c238160ba5b 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -655,12 +655,11 @@ task_groups: script: | ${PREPARE_SHELL} set +o xtrace - #SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ - #SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ - #SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ - #SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ - # bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - echo "SKIPPING DELETE TO EASE TESTING" + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh tasks: - ".serverless" diff --git a/test/functional/retryable_writes.test.js b/test/functional/retryable_writes.test.js index 1c79a06fcd6..00d47261302 100644 --- a/test/functional/retryable_writes.test.js +++ b/test/functional/retryable_writes.test.js @@ -46,7 +46,7 @@ describe('Retryable Writes', function () { }); function executeScenarioSetup(scenario, test, config, ctx) { - const url = config.url(); + const url = process.env.SERVERLESS ? process.env.MONGODB_URI : config.url(); const options = Object.assign({}, test.clientOptions, { heartbeatFrequencyMS: 100, monitorCommands: true, diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 337e226157e..b846490ab9d 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -109,8 +109,7 @@ export class UnifiedMongoClient extends MongoClient { this.failPoints = []; this.ignoredEvents = [ ...(description.ignoreCommandMonitoringEvents ?? []), - 'configureFailPoint', - 'ping' + 'configureFailPoint' ]; this.observedCommandEvents = (description.observeEvents ?? []) .map(e => UnifiedMongoClient.COMMAND_EVENT_NAME_LOOKUP[e]) diff --git a/test/tools/runner/filters/serverless_filter.js b/test/tools/runner/filters/serverless_filter.js index b08edbada8c..215eeb96795 100755 --- a/test/tools/runner/filters/serverless_filter.js +++ b/test/tools/runner/filters/serverless_filter.js @@ -18,7 +18,6 @@ class ServerlessFilter { initializeFilter(client, context, callback) { if (this.serverless) { - console.log('saving serverless credentials in test filter'); context.serverlessCredentials = { username: process.env.SERVERLESS_ATLAS_USER, password: process.env.SERVERLESS_ATLAS_PASSWORD From 592905f5f604753fc384e223460c000066b64f48 Mon Sep 17 00:00:00 2001 From: emadum Date: Thu, 15 Jul 2021 13:33:28 -0400 Subject: [PATCH 06/13] report xunit to evg, add fixmes --- .evergreen/config.yml | 5 +++++ .evergreen/config.yml.in | 5 +++++ .evergreen/run-serverless-tests.sh | 4 ++-- test/functional/crud_spec.test.js | 5 ++++- test/functional/retryable_writes.test.js | 2 +- test/functional/sessions.test.js | 7 ++++++- test/functional/spec-runner/index.js | 19 +++++++++++++++++++ .../unified-spec-runner/entities.ts | 4 ++++ test/tools/runner/config.js | 2 ++ 9 files changed, 48 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d27b16fdb69..7c2e18d0e24 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -135,10 +135,14 @@ functions: NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh run serverless tests: + - command: timeout.update + params: + exec_timeout_secs: 1800 - command: shell.exec type: test params: working_dir: src + continue_on_err: true script: | ${PREPARE_SHELL} set +o xtrace @@ -588,6 +592,7 @@ tasks: - serverless commands: - func: run serverless tests + - func: upload test results - name: test-latest-server tags: - latest diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index c238160ba5b..2fbf877fc81 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -156,10 +156,14 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh "run serverless tests": + - command: timeout.update + params: + exec_timeout_secs: 1800 - command: shell.exec type: test params: working_dir: "src" + continue_on_err: true script: | ${PREPARE_SHELL} set +o xtrace @@ -628,6 +632,7 @@ tasks: tags: ["serverless"] commands: - func: "run serverless tests" + - func: "upload test results" task_groups: - name: serverless_task_group diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh index 589a595f68e..a19c0f15a19 100644 --- a/.evergreen/run-serverless-tests.sh +++ b/.evergreen/run-serverless-tests.sh @@ -8,6 +8,6 @@ MONGODB_URI=${MONGODB_URI} npx mocha \ test/functional/crud_spec.test.js \ test/functional/retryable_reads.test.js \ test/functional/retryable_writes.test.js \ - test/functional/versioned-api.test.js \ test/functional/sessions.test.js \ - test/functional/transactions.test.js \ No newline at end of file + test/functional/transactions.test.js \ + test/functional/versioned-api.test.js diff --git a/test/functional/crud_spec.test.js b/test/functional/crud_spec.test.js index 2cda5ac4b22..00d0ab10a89 100644 --- a/test/functional/crud_spec.test.js +++ b/test/functional/crud_spec.test.js @@ -20,6 +20,9 @@ function enforceServerVersionLimits(requires, scenario) { if (versionLimits.length) { requires.mongodb = versionLimits.join(' '); } + if (scenario.serverless) { + requires.serverless = scenario.serverless; + } } function findScenarios() { @@ -35,7 +38,7 @@ const readScenarios = findScenarios('v1', 'read'); const writeScenarios = findScenarios('v1', 'write'); const testContext = {}; -describe('CRUD spec', function () { +describe('CRUD spec v1', function () { beforeEach(function () { const configuration = this.configuration; const client = configuration.newClient(); diff --git a/test/functional/retryable_writes.test.js b/test/functional/retryable_writes.test.js index 00d47261302..1c79a06fcd6 100644 --- a/test/functional/retryable_writes.test.js +++ b/test/functional/retryable_writes.test.js @@ -46,7 +46,7 @@ describe('Retryable Writes', function () { }); function executeScenarioSetup(scenario, test, config, ctx) { - const url = process.env.SERVERLESS ? process.env.MONGODB_URI : config.url(); + const url = config.url(); const options = Object.assign({}, test.clientOptions, { heartbeatFrequencyMS: 100, monitorCommands: true, diff --git a/test/functional/sessions.test.js b/test/functional/sessions.test.js index f0e0fb5cb75..154e90700a3 100644 --- a/test/functional/sessions.test.js +++ b/test/functional/sessions.test.js @@ -64,7 +64,12 @@ describe('Sessions - functional', function () { }); describe('withSession', { - metadata: { requires: { mongodb: '>=3.6.0' } }, + metadata: { + requires: { + mongodb: '>=3.6.0', + serverless: 'forbid' // UnhandledPromiseRejectionWarning: MongoServerError: CMD_NOT_ALLOWED: dropAllUsersFromDatabase + } + }, test: function () { beforeEach(function () { return test.setup(this.configuration); diff --git a/test/functional/spec-runner/index.js b/test/functional/spec-runner/index.js index f1c53cfb0e8..aed5f734186 100644 --- a/test/functional/spec-runner/index.js +++ b/test/functional/spec-runner/index.js @@ -147,6 +147,25 @@ function generateTopologyTests(testSuites, testContext, filter) { this.skip(); } + if (requires.serverless) { + console.log('applying serverless requirement in spec runner'); + const isServerless = !!process.env.SERVERLESS; + switch (requires.serverless) { + case 'forbid': + // return true if the configuration is NOT serverless + if (isServerless) this.skip(); + break; + case 'allow': + // always run + break; + case 'require': + if (!isServerless) this.skip(); + break; + default: + throw new Error(`Invalid serverless filter: ${requires.serverless}`); + } + } + if ( spec.operations.some( op => op.name === 'waitForEvent' && op.arguments.event === 'PoolReadyEvent' diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index b846490ab9d..758681e3afa 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -111,6 +111,10 @@ export class UnifiedMongoClient extends MongoClient { ...(description.ignoreCommandMonitoringEvents ?? []), 'configureFailPoint' ]; + // FIXME: hack to get tests passing, extra unexpected events otherwise + if (process.env.SERVERLESS) { + this.ignoredEvents.push('ping'); + } this.observedCommandEvents = (description.observeEvents ?? []) .map(e => UnifiedMongoClient.COMMAND_EVENT_NAME_LOOKUP[e]) .filter(e => !!e); diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 4ba6ca41158..1aea6a68b10 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -202,6 +202,8 @@ class TestConfiguration { * @param {UrlOptions} [options] - overrides and settings for URI generation */ url(options) { + // FIXME: hack to get tests passing, auth fails without this + if (this.isServerless) return process.env.MONGODB_URI; options = { db: this.options.db, replicaSet: this.options.replicaSet, ...options }; const FILLER_HOST = 'fillerHost'; From 5db4f1ca32047f808929bdb9d71867f244c2ab48 Mon Sep 17 00:00:00 2001 From: emadum Date: Mon, 19 Jul 2021 13:11:11 -0400 Subject: [PATCH 07/13] refactor: extract shouldRunServerlessTest --- .evergreen/config.yml | 4 +-- .evergreen/config.yml.in | 4 +-- .evergreen/run-serverless-tests.sh | 4 +-- test/functional/spec-runner/index.js | 33 +++++++------------ .../unified-spec-runner/entities.ts | 4 +-- .../unified-spec-runner/unified-utils.ts | 16 ++------- test/tools/runner/config.js | 1 + .../tools/runner/filters/serverless_filter.js | 17 ++-------- test/tools/utils.js | 20 ++++++++++- 9 files changed, 43 insertions(+), 60 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 7c2e18d0e24..a466eeb6a50 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -142,10 +142,10 @@ functions: type: test params: working_dir: src - continue_on_err: true script: | ${PREPARE_SHELL} - set +o xtrace + # Disable xtrace (just in case it was accidentally set). + set +x MONGODB_URI="${MONGODB_URI}" \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ AUTH="auth" SSL="ssl" SERVERLESS=1 \ diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index 2fbf877fc81..f537b5b10b0 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -163,10 +163,10 @@ functions: type: test params: working_dir: "src" - continue_on_err: true script: | ${PREPARE_SHELL} - set +o xtrace + # Disable xtrace (just in case it was accidentally set). + set +x MONGODB_URI="${MONGODB_URI}" \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ AUTH="auth" SSL="ssl" SERVERLESS=1 \ diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh index a19c0f15a19..706e31c6a96 100644 --- a/.evergreen/run-serverless-tests.sh +++ b/.evergreen/run-serverless-tests.sh @@ -7,7 +7,7 @@ SERVERLESS=1 AUTH=auth SSL=ssl \ MONGODB_URI=${MONGODB_URI} npx mocha \ test/functional/crud_spec.test.js \ test/functional/retryable_reads.test.js \ - test/functional/retryable_writes.test.js \ +# test/functional/retryable_writes.test.js \ test/functional/sessions.test.js \ - test/functional/transactions.test.js \ +# test/functional/transactions.test.js \ test/functional/versioned-api.test.js diff --git a/test/functional/spec-runner/index.js b/test/functional/spec-runner/index.js index aed5f734186..808abf70ed6 100644 --- a/test/functional/spec-runner/index.js +++ b/test/functional/spec-runner/index.js @@ -7,6 +7,7 @@ const { EJSON } = require('bson'); const { isRecord } = require('../../../src/utils'); const TestRunnerContext = require('./context').TestRunnerContext; const resolveConnectionString = require('./utils').resolveConnectionString; +const { shouldRunServerlessTest } = require('../../tools/utils'); // Promise.try alternative https://stackoverflow.com/questions/60624081/promise-try-without-bluebird/60624164?noredirect=1#comment107255389_60624164 function promiseTry(callback) { @@ -147,23 +148,12 @@ function generateTopologyTests(testSuites, testContext, filter) { this.skip(); } - if (requires.serverless) { - console.log('applying serverless requirement in spec runner'); - const isServerless = !!process.env.SERVERLESS; - switch (requires.serverless) { - case 'forbid': - // return true if the configuration is NOT serverless - if (isServerless) this.skip(); - break; - case 'allow': - // always run - break; - case 'require': - if (!isServerless) this.skip(); - break; - default: - throw new Error(`Invalid serverless filter: ${requires.serverless}`); - } + if ( + requires.serverless && + !shouldRunServerlessTest(requires.serverless, !!process.env.SERVERLESS) + ) { + console.log('skipping serverless requirement in spec runner'); + return this.skip(); } if ( @@ -217,10 +207,11 @@ function prepareDatabaseForSuite(suite, context) { .admin() .command({ killAllSessions: [] }) .catch(err => { - if (err.message.match(/no such (cmd|command)/) || err.code === 11601) { - return; - } - if (process.env.SERVERLESS) { + if ( + err.message.match(/no such (cmd|command)/) || + err.code === 11601 || + process.env.SERVERLESS // killAllSessions is not supported on serverless + ) { return; } diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 758681e3afa..0625fbdff3c 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -319,9 +319,7 @@ export class EntitiesMap extends Map { const useMultipleMongoses = (config.topologyType === 'LoadBalanced' || config.topologyType === 'Sharded') && entity.client.useMultipleMongoses; - const uri = process.env.SERVERLESS - ? process.env.MONGODB_URI - : config.url({ useMultipleMongoses }); + const uri = config.url({ useMultipleMongoses }); const client = new UnifiedMongoClient(uri, entity.client); await client.connect(); map.set(entity.client.id, client); diff --git a/test/functional/unified-spec-runner/unified-utils.ts b/test/functional/unified-spec-runner/unified-utils.ts index 5c3ba4b9c27..8049072d159 100644 --- a/test/functional/unified-spec-runner/unified-utils.ts +++ b/test/functional/unified-spec-runner/unified-utils.ts @@ -5,7 +5,7 @@ import { gte as semverGte, lte as semverLte } from 'semver'; import { CollectionOptions, DbOptions, MongoClient } from '../../../src'; import { isDeepStrictEqual } from 'util'; import { TestConfiguration } from './runner'; - +import { shouldRunServerlessTest } from '../../tools/utils'; const ENABLE_UNIFIED_TEST_LOGGING = false; export function log(message: unknown, ...optionalParameters: unknown[]): void { if (ENABLE_UNIFIED_TEST_LOGGING) console.warn(message, ...optionalParameters); @@ -64,19 +64,7 @@ export async function topologySatisfies( } if (r.serverless) { - switch (r.serverless) { - case 'forbid': - // return true if the configuration is NOT serverless - return !config.isServerless; - case 'allow': - // always return true - return true; - case 'require': - // only return true if the configuration is serverless - return config.isServerless; - default: - throw new Error(`Invalid serverless filter: ${r.serverless}`); - } + ok &&= shouldRunServerlessTest(r.serverless, config.isServerless); } return ok; diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 1aea6a68b10..9f0f9776c8e 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -204,6 +204,7 @@ class TestConfiguration { url(options) { // FIXME: hack to get tests passing, auth fails without this if (this.isServerless) return process.env.MONGODB_URI; + options = { db: this.options.db, replicaSet: this.options.replicaSet, ...options }; const FILLER_HOST = 'fillerHost'; diff --git a/test/tools/runner/filters/serverless_filter.js b/test/tools/runner/filters/serverless_filter.js index 215eeb96795..72288c8f576 100755 --- a/test/tools/runner/filters/serverless_filter.js +++ b/test/tools/runner/filters/serverless_filter.js @@ -1,4 +1,5 @@ 'use strict'; +const { shouldRunServerlessTest } = require('../../utils'); /** * Filter to allow to tests to run on serverless @@ -29,21 +30,7 @@ class ServerlessFilter { filter(test) { if (!test.metadata) return true; if (!test.metadata.requires) return true; - const serverless = test.metadata.requires.serverless; - if (!serverless) return true; - switch (serverless) { - case 'forbid': - // return true if the configuration is NOT serverless - return !this.serverless; - case 'allow': - // always return true - return true; - case 'require': - // only return true if the configuration is serverless - return this.serverless; - default: - throw new Error(`Invalid serverless filter: ${serverless}`); - } + return shouldRunServerlessTest(test.metadata.requires.serverless, this.serverless); } } diff --git a/test/tools/utils.js b/test/tools/utils.js index 9017087b02e..e03fee62f3d 100644 --- a/test/tools/utils.js +++ b/test/tools/utils.js @@ -278,6 +278,23 @@ function getEnvironmentalOptions() { return options; } +function shouldRunServerlessTest(testRequirement, isServerless) { + if (!testRequirement) return true; + switch (testRequirement) { + case 'forbid': + // return true if the configuration is NOT serverless + return !isServerless; + case 'allow': + // always return true + return true; + case 'require': + // only return true if the configuration is serverless + return isServerless; + default: + throw new Error(`Invalid serverless filter: ${testRequirement}`); + } +} + module.exports = { EventCollector, makeTestFunction, @@ -287,5 +304,6 @@ module.exports = { ClassWithUndefinedLogger, visualizeMonitoringEvents, getSymbolFrom, - getEnvironmentalOptions + getEnvironmentalOptions, + shouldRunServerlessTest }; From b9891064112ce8e3730e336123799c4a626b6559 Mon Sep 17 00:00:00 2001 From: emadum Date: Tue, 20 Jul 2021 20:08:57 -0400 Subject: [PATCH 08/13] enable skipped specs --- .evergreen/config.yml | 2 +- .evergreen/config.yml.in | 2 +- .evergreen/run-serverless-tests.sh | 4 +- test/functional/spec-runner/context.js | 1 + test/functional/transactions.test.js | 79 +++++++++++++++----------- test/tools/runner/config.js | 8 ++- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a466eeb6a50..dc2be534c60 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -592,7 +592,6 @@ tasks: - serverless commands: - func: run serverless tests - - func: upload test results - name: test-latest-server tags: - latest @@ -1672,6 +1671,7 @@ task_groups: params: file: serverless-expansion.yml teardown_group: + - func: upload test results - command: shell.exec params: script: | diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index f537b5b10b0..bdb7373f58e 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -632,7 +632,6 @@ tasks: tags: ["serverless"] commands: - func: "run serverless tests" - - func: "upload test results" task_groups: - name: serverless_task_group @@ -655,6 +654,7 @@ task_groups: params: file: serverless-expansion.yml teardown_group: + - func: "upload test results" - command: shell.exec params: script: | diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh index 706e31c6a96..a19c0f15a19 100644 --- a/.evergreen/run-serverless-tests.sh +++ b/.evergreen/run-serverless-tests.sh @@ -7,7 +7,7 @@ SERVERLESS=1 AUTH=auth SSL=ssl \ MONGODB_URI=${MONGODB_URI} npx mocha \ test/functional/crud_spec.test.js \ test/functional/retryable_reads.test.js \ -# test/functional/retryable_writes.test.js \ + test/functional/retryable_writes.test.js \ test/functional/sessions.test.js \ -# test/functional/transactions.test.js \ + test/functional/transactions.test.js \ test/functional/versioned-api.test.js diff --git a/test/functional/spec-runner/context.js b/test/functional/spec-runner/context.js index f4f0de980f9..5cff9ef809f 100644 --- a/test/functional/spec-runner/context.js +++ b/test/functional/spec-runner/context.js @@ -48,6 +48,7 @@ class TestRunnerContext { this.user = process.env.SERVERLESS_ATLAS_USER; this.password = process.env.SERVERLESS_ATLAS_PASSWORD; this.authSource = 'admin'; + this.serverless = true; } this.sharedClient = null; this.failPointClients = []; diff --git a/test/functional/transactions.test.js b/test/functional/transactions.test.js index 170047258fe..115835c5c65 100644 --- a/test/functional/transactions.test.js +++ b/test/functional/transactions.test.js @@ -97,16 +97,49 @@ describe('Transactions Spec Unified Tests', function () { } }); -describe('Transactions', function () { +const SKIP_TESTS = [ + // commitTransaction retry seems to be swallowed by mongos in these three cases + 'commitTransaction retry succeeds on new mongos', + 'commitTransaction retry fails on new mongos', + 'unpin after transient error within a transaction and commit', + // FIXME(NODE-3074): unskip count tests when spec tests have been updated + 'count', + // This test needs there to be multiple mongoses + // 'increment txnNumber', + // Skipping this until SPEC-1320 is resolved + // 'remain pinned after non-transient error on commit', + + // Will be implemented as part of NODE-2034 + 'Client side error in command starting transaction', + 'Client side error when transaction is in progress', + + // Will be implemented as part of NODE-2538 + 'abortTransaction only retries once with RetryableWriteError from server', + 'abortTransaction does not retry without RetryableWriteError label', + 'commitTransaction does not retry error without RetryableWriteError label', + 'commitTransaction retries once with RetryableWriteError from server' +]; + +describe('Transactions Spec Legacy Tests', function () { const testContext = new TransactionsRunnerContext(); - - [ - { name: 'spec tests', specPath: path.join('transactions', 'legacy') }, - { + const suitesToRun = [{ name: 'spec tests', specPath: path.join('transactions', 'legacy') }]; + // Note: convenient-api tests are skipped for serverless + if (!process.env.SERVERLESS) { + suitesToRun.push({ name: 'withTransaction spec tests', specPath: path.join('transactions', 'convenient-api') - } - ].forEach(suiteSpec => { + }); + } else { + SKIP_TESTS.push( + 'abortTransaction only performs a single retry', + 'abortTransaction does not retry after Interrupted', + 'abortTransaction does not retry after WriteConcernError Interrupted', + 'commitTransaction does not retry error without RetryableWriteError label', + 'commitTransaction is not retried after UnsatisfiableWriteConcern error', + 'commitTransaction fails after Interrupted' + ); + } + suitesToRun.forEach(suiteSpec => { describe(suiteSpec.name, function () { const testSuites = loadSpecTests(suiteSpec.specPath); after(() => testContext.teardown()); @@ -115,29 +148,6 @@ describe('Transactions', function () { }); function testFilter(spec) { - const SKIP_TESTS = [ - // commitTransaction retry seems to be swallowed by mongos in these three cases - 'commitTransaction retry succeeds on new mongos', - 'commitTransaction retry fails on new mongos', - 'unpin after transient error within a transaction and commit', - // FIXME(NODE-3074): unskip count tests when spec tests have been updated - 'count', - // This test needs there to be multiple mongoses - // 'increment txnNumber', - // Skipping this until SPEC-1320 is resolved - // 'remain pinned after non-transient error on commit', - - // Will be implemented as part of NODE-2034 - 'Client side error in command starting transaction', - 'Client side error when transaction is in progress', - - // Will be implemented as part of NODE-2538 - 'abortTransaction only retries once with RetryableWriteError from server', - 'abortTransaction does not retry without RetryableWriteError label', - 'commitTransaction does not retry error without RetryableWriteError label', - 'commitTransaction retries once with RetryableWriteError from server' - ]; - return SKIP_TESTS.indexOf(spec.description) === -1; } @@ -158,7 +168,9 @@ describe('Transactions', function () { }); it('should provide a useful error if a Promise is not returned', { - metadata: { requires: { topology: ['replicaset', 'sharded'], mongodb: '>=4.1.5' } }, + metadata: { + requires: { topology: ['replicaset', 'sharded'], mongodb: '>=4.1.5', serverless: 'forbid' } + }, test: function (done) { function fnThatDoesntReturnPromise() { return false; @@ -173,7 +185,10 @@ describe('Transactions', function () { }); it('should return readable error if promise rejected with no reason', { - metadata: { requires: { topology: ['replicaset', 'sharded'], mongodb: '>=4.0.2' } }, + metadata: { + requires: { topology: ['replicaset', 'sharded'], mongodb: '>=4.0.2' }, + serverless: 'forbid' + }, test: function (done) { function fnThatReturnsBadPromise() { return Promise.reject(); diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 9f0f9776c8e..289b59f7c0c 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -202,9 +202,6 @@ class TestConfiguration { * @param {UrlOptions} [options] - overrides and settings for URI generation */ url(options) { - // FIXME: hack to get tests passing, auth fails without this - if (this.isServerless) return process.env.MONGODB_URI; - options = { db: this.options.db, replicaSet: this.options.replicaSet, ...options }; const FILLER_HOST = 'fillerHost'; @@ -249,6 +246,11 @@ class TestConfiguration { actualHostsString = this.options.hostAddresses[0].toString(); } + if (this.isServerless) { + url.searchParams.append('ssl', true); + url.searchParams.append('authSource', 'admin'); + } + const connectionString = url.toString().replace(FILLER_HOST, actualHostsString); return connectionString; From 8b83c4bb72c3bc63ad450fc88eb1a1b28ad05434 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 21 Jul 2021 15:48:58 -0400 Subject: [PATCH 09/13] better skipping in legacy test runner --- test/functional/spec-runner/index.js | 90 ++++++++++++++-------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/test/functional/spec-runner/index.js b/test/functional/spec-runner/index.js index 808abf70ed6..d0085652997 100644 --- a/test/functional/spec-runner/index.js +++ b/test/functional/spec-runner/index.js @@ -141,37 +141,8 @@ function generateTopologyTests(testSuites, testContext, filter) { beforeEach(() => prepareDatabaseForSuite(testSuite, testContext)); afterEach(() => testContext.cleanupAfterSuite()); testSuite.tests.forEach(spec => { - it(spec.description, function () { - if (requires.authEnabled && process.env.AUTH !== 'auth') { - // TODO: We do not have a way to determine if auth is enabled in our mocha metadata - // We need to do a admin.command({getCmdLineOpts: 1}) if it errors (code=13) auth is on - this.skip(); - } - - if ( - requires.serverless && - !shouldRunServerlessTest(requires.serverless, !!process.env.SERVERLESS) - ) { - console.log('skipping serverless requirement in spec runner'); - return this.skip(); - } - - if ( - spec.operations.some( - op => op.name === 'waitForEvent' && op.arguments.event === 'PoolReadyEvent' - ) - ) { - // TODO(NODE-2994): Connection storms work will add new events to connection pool - this.skip(); - } - - if ( - spec.skipReason || - (filter && typeof filter === 'function' && !filter(spec, this.configuration)) - ) { - return this.skip(); - } - + const maybeIt = shouldRunSpecTest.call(this, requires, spec, filter) ? it : it.skip; + maybeIt(spec.description, function () { let testPromise = Promise.resolve(); if (spec.failPoint) { testPromise = testPromise.then(() => testContext.enableFailPoint(spec.failPoint)); @@ -194,6 +165,38 @@ function generateTopologyTests(testSuites, testContext, filter) { }); } +function shouldRunSpecTest(requires, spec, filter) { + if (requires.authEnabled && process.env.AUTH !== 'auth') { + // TODO: We do not have a way to determine if auth is enabled in our mocha metadata + // We need to do a admin.command({getCmdLineOpts: 1}) if it errors (code=13) auth is on + return false; + } + + if ( + requires.serverless && + !shouldRunServerlessTest(requires.serverless, !!process.env.SERVERLESS) + ) { + return false; + } + + if ( + spec.operations.some( + op => op.name === 'waitForEvent' && op.arguments.event === 'PoolReadyEvent' + ) + ) { + // TODO(NODE-2994): Connection storms work will add new events to connection pool + return false; + } + + if ( + spec.skipReason || + (filter && typeof filter === 'function' && !filter(spec, this.configuration)) + ) { + return false; + } + return true; +} + // Test runner helpers function prepareDatabaseForSuite(suite, context) { context.dbName = suite.database_name || 'spec_db'; @@ -203,20 +206,19 @@ function prepareDatabaseForSuite(suite, context) { if (context.skipPrepareDatabase) return Promise.resolve(); - const setupPromise = db - .admin() - .command({ killAllSessions: [] }) - .catch(err => { - if ( - err.message.match(/no such (cmd|command)/) || - err.code === 11601 || - process.env.SERVERLESS // killAllSessions is not supported on serverless - ) { - return; - } + // Note: killAllSession is not supported on serverless, see CLOUDP-84298 + const setupPromise = context.serverless + ? Promise.resolve() + : db + .admin() + .command({ killAllSessions: [] }) + .catch(err => { + if (err.message.match(/no such (cmd|command)/) || err.code === 11601) { + return; + } - throw err; - }); + throw err; + }); if (context.collectionName == null || context.dbName === 'admin') { return setupPromise; From 69309498bebf223744959c92287f73bf366aabda Mon Sep 17 00:00:00 2001 From: emadum Date: Thu, 5 Aug 2021 11:09:39 -0400 Subject: [PATCH 10/13] fix unified test runner for serverless --- test/functional/unified-spec-runner/runner.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/functional/unified-spec-runner/runner.ts b/test/functional/unified-spec-runner/runner.ts index 525e8209a1e..e1a2e73e63d 100644 --- a/test/functional/unified-spec-runner/runner.ts +++ b/test/functional/unified-spec-runner/runner.ts @@ -23,6 +23,10 @@ export function trace(message: string): void { } async function terminateOpenTransactions(client: MongoClient) { + // Note: killAllSession is not supported on serverless, see CLOUDP-84298 + if (process.env.SERVERLESS) { + return; + } // TODO(NODE-3491): on sharded clusters this has to be run on each mongos try { await client.db().admin().command({ killAllSessions: [] }); From 966c7f7b3de33ae0bfd7f2226f780c3b5cde12ef Mon Sep 17 00:00:00 2001 From: emadum Date: Thu, 12 Aug 2021 08:14:34 -0400 Subject: [PATCH 11/13] remove unnecessary type assertion --- test/functional/unified-spec-runner/entities.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 58e1072766d..8f7f12c97b8 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -33,7 +33,6 @@ import type { import { makeConnectionString, patchCollectionOptions, patchDbOptions } from './unified-utils'; import { expect } from 'chai'; import { getEnvironmentalOptions } from '../../tools/utils'; -import { MongoClientOptions } from '../../../src/mongo_client'; import { TestConfiguration, trace } from './runner'; interface UnifiedChangeStream extends ChangeStream { @@ -54,7 +53,7 @@ export type CmapEvent = | ConnectionPoolClearedEvent; function getClient(address) { - return new MongoClient(`mongodb://${address}`, getEnvironmentalOptions() as MongoClientOptions); + return new MongoClient(`mongodb://${address}`, getEnvironmentalOptions()); } type PushFunction = (e: CommandEvent | CmapEvent) => void; From 2d16d4cd28588a986c172a2728fa5f78371d53a5 Mon Sep 17 00:00:00 2001 From: emadum Date: Mon, 16 Aug 2021 12:03:42 -0400 Subject: [PATCH 12/13] review feedback --- test/functional/transactions.test.js | 1 + test/functional/unified-spec-runner/entities.ts | 2 +- test/tools/runner/config.js | 10 +++++----- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/functional/transactions.test.js b/test/functional/transactions.test.js index 7fd79d712e6..61b29e9b9e0 100644 --- a/test/functional/transactions.test.js +++ b/test/functional/transactions.test.js @@ -130,6 +130,7 @@ describe('Transactions Spec Legacy Tests', function () { specPath: path.join('transactions', 'convenient-api') }); } else { + // FIXME(NODE-3550): these tests should pass on serverless but currently fail SKIP_TESTS.push( 'abortTransaction only performs a single retry', 'abortTransaction does not retry after Interrupted', diff --git a/test/functional/unified-spec-runner/entities.ts b/test/functional/unified-spec-runner/entities.ts index 8f7f12c97b8..bf79ccf3947 100644 --- a/test/functional/unified-spec-runner/entities.ts +++ b/test/functional/unified-spec-runner/entities.ts @@ -111,7 +111,7 @@ export class UnifiedMongoClient extends MongoClient { ...(description.ignoreCommandMonitoringEvents ?? []), 'configureFailPoint' ]; - // FIXME: hack to get tests passing, extra unexpected events otherwise + // FIXME(NODE-3549): hack to get tests passing, extra unexpected events otherwise if (process.env.SERVERLESS) { this.ignoredEvents.push('ping'); } diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 5c21f46b64b..775830ea317 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -109,18 +109,18 @@ class TestConfiguration { } newClient(dbOptions, serverOptions) { - const defaultOptions = Object.assign( + serverOptions = Object.assign( { minHeartbeatFrequencyMS: 100 }, - getEnvironmentalOptions() + getEnvironmentalOptions(), + serverOptions ); + // support MongoClient constructor form (url, options) for `newClient` if (typeof dbOptions === 'string') { - return new MongoClient(dbOptions, Object.assign({}, defaultOptions, serverOptions)); + return new MongoClient(dbOptions, serverOptions); } dbOptions = dbOptions || {}; - serverOptions = Object.assign({}, defaultOptions, serverOptions); - // Fall back let dbHost = (serverOptions && serverOptions.host) || this.options.host; const dbPort = (serverOptions && serverOptions.port) || this.options.port; From 79db1efde32b64959f74225c29f1ea9f70faf7d6 Mon Sep 17 00:00:00 2001 From: emadum Date: Wed, 18 Aug 2021 16:23:21 -0400 Subject: [PATCH 13/13] more review feedback --- test/functional/sessions.test.js | 3 +-- test/tools/runner/config.js | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/test/functional/sessions.test.js b/test/functional/sessions.test.js index f4643631684..d5a60d3159d 100644 --- a/test/functional/sessions.test.js +++ b/test/functional/sessions.test.js @@ -68,8 +68,7 @@ describe('Sessions', function () { describe('withSession', { metadata: { requires: { - mongodb: '>=3.6.0', - serverless: 'forbid' // UnhandledPromiseRejectionWarning: MongoServerError: CMD_NOT_ALLOWED: dropAllUsersFromDatabase + mongodb: '>=3.6.0' } }, test: function () { diff --git a/test/tools/runner/config.js b/test/tools/runner/config.js index 775830ea317..b01ce842167 100644 --- a/test/tools/runner/config.js +++ b/test/tools/runner/config.js @@ -236,6 +236,9 @@ class TestConfiguration { if (options.authSource) { url.searchParams.append('authSource', options.authSource); } + } else if (this.isServerless) { + url.searchParams.append('ssl', true); + url.searchParams.append('authSource', 'admin'); } let actualHostsString; @@ -256,11 +259,6 @@ class TestConfiguration { } } - if (this.isServerless) { - url.searchParams.append('ssl', true); - url.searchParams.append('authSource', 'admin'); - } - const connectionString = url.toString().replace(FILLER_HOST, actualHostsString); return connectionString;