From bec94daa7bd602a4752ca570db03b13e9c02d037 Mon Sep 17 00:00:00 2001 From: Lydia Date: Wed, 22 Jul 2020 13:12:15 +0100 Subject: [PATCH] fix(deps): Updates Mongo to 4.2 (LLC-234) (#1541) BREAKING CHANGE: Removes maxScan parameter from API BREAKING CHANGE: Requires Mongo 4.2 --- .circleci/config.yml | 2 +- .env.example | 4 ---- api/src/controllers/PersonaController.js | 5 ++--- api/src/controllers/PersonaIdentifierController.js | 5 ++--- api/src/controllers/StatementController.js | 5 ----- api/src/utils/exports.js | 1 - .../v2-migrations/20171122100800_common_indexes.js | 1 - lib/helpers/convert$personaIdent.js | 5 ++--- lib/models/plugins/addCRUDFunctions.js | 14 +++++--------- lib/models/statement.js | 10 ++-------- lib/services/statements/aggregate.js | 3 +-- lib/services/statements/countStatements.js | 3 +-- 12 files changed, 16 insertions(+), 42 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0790bc8d09..4c5e661a72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: build: docker: - image: circleci/node:10 - - image: mongo:3.4-jessie + - image: mongo:4.2-bionic - image: redis:4-alpine environment: TERM: xterm diff --git a/.env.example b/.env.example index 8332937dd0..5d84f40aa2 100644 --- a/.env.example +++ b/.env.example @@ -87,10 +87,6 @@ REDIS_PREFIX=LEARNINGLOCKER # https://docs.mongodb.com/manual/reference/operator/meta/maxTimeMS/ #MAX_TIME_MS= -# Maximum number of documents an aggregation can scan -# https://docs.mongodb.com/manual/reference/operator/meta/maxScan/ -#MAX_SCAN= - # Turn off fuzzy scoring on persona matching # Set to true to make persona workers much faster at scale (default is true) #DISABLE_PERSONA_SCORING=true diff --git a/api/src/controllers/PersonaController.js b/api/src/controllers/PersonaController.js index 3ddadcdce3..3a8868eb21 100644 --- a/api/src/controllers/PersonaController.js +++ b/api/src/controllers/PersonaController.js @@ -4,7 +4,7 @@ import getJSONFromQuery from 'api/utils/getJSONFromQuery'; import getFromQuery from 'api/utils/getFromQuery'; import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo'; import getScopeFilter from 'lib/services/auth/filters/getScopeFilter'; -import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions'; +import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions'; import parseQuery from 'lib/helpers/parseQuery'; import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants'; import { entityResponse, entitiesResponse } from 'api/controllers/utils/entitiesResponse'; @@ -47,8 +47,7 @@ const personaConnection = catchErrors(async (req, res) => { filter, project, hint, - maxTimeMS: MAX_TIME_MS, - maxScan: MAX_SCAN + maxTimeMS: MAX_TIME_MS }; const result = await personaService.getPersonasConnection(params); diff --git a/api/src/controllers/PersonaIdentifierController.js b/api/src/controllers/PersonaIdentifierController.js index 52bb4ac1ac..3bcb34ba95 100644 --- a/api/src/controllers/PersonaIdentifierController.js +++ b/api/src/controllers/PersonaIdentifierController.js @@ -9,7 +9,7 @@ import getAuthFromRequest from 'lib/helpers/getAuthFromRequest'; import getScopeFilter from 'lib/services/auth/filters/getScopeFilter'; import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants'; import Locked from '@learninglocker/persona-service/dist/errors/Locked'; -import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions'; +import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions'; import parseQuery from 'lib/helpers/parseQuery'; import asignIdentifierToStatements from 'lib/services/persona/asignIdentifierToStatements'; import identifierHasStatements from 'lib/services/persona/identifierHasStatements'; @@ -67,8 +67,7 @@ const personaIdentifierConnection = catchErrors(async (req, res) => { filter: filterNoUndefined, project, hint, - maxTimeMS: MAX_TIME_MS, - maxScan: MAX_SCAN + maxTimeMS: MAX_TIME_MS }); return res.status(200).send(result); diff --git a/api/src/controllers/StatementController.js b/api/src/controllers/StatementController.js index 3fd17ea728..94e49fcb1c 100644 --- a/api/src/controllers/StatementController.js +++ b/api/src/controllers/StatementController.js @@ -6,7 +6,6 @@ import catchErrors from 'api/controllers/utils/catchErrors'; import defaultTo from 'lodash/defaultTo'; const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0); -const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0); const aggregate = (req) => { const authInfo = req.user.authInfo || {}; @@ -14,7 +13,6 @@ const aggregate = (req) => { const skip = Number(req.query.skip) || 0; const cache = (!!req.query.cache && req.query.cache !== 'false') || false; const maxTimeMS = Number(req.query.maxTimeMS) || MAX_TIME_MS; - const maxScan = Number(req.query.maxScan) || MAX_SCAN; const pipeline = JSON.parse(req.query.pipeline); const sampleSize = Number(req.query.sampleSize) || undefined; const out = statementsService.aggregate({ @@ -23,7 +21,6 @@ const aggregate = (req) => { skip, cache, maxTimeMS, - maxScan, pipeline, sampleSize }); @@ -87,7 +84,6 @@ const countStatements = catchErrors(async (req, res) => { getJSONFromQuery(req, 'query', {}) ); const maxTimeMS = getFromQuery(req, 'maxTimeMS', MAX_TIME_MS, Number); - const maxScan = getFromQuery(req, 'maxScan', MAX_SCAN, Number); const hint = getJSONFromQuery(req, 'hint', null); const authInfo = getAuthFromRequest(req); @@ -95,7 +91,6 @@ const countStatements = catchErrors(async (req, res) => { authInfo, filter, maxTimeMS, - maxScan, hint }); diff --git a/api/src/utils/exports.js b/api/src/utils/exports.js index 97c2eda3e6..d00a450d25 100644 --- a/api/src/utils/exports.js +++ b/api/src/utils/exports.js @@ -49,7 +49,6 @@ export const exportCSV = ({ authInfo, pipelines }) => new Promise((resolve, reje limit: 500000, getStream: true, maxTimeMS: 0, - maxScan: 0, }, next), (err, streams) => { if (err) reject(err); diff --git a/cli/src/commands/v2-migrations/20171122100800_common_indexes.js b/cli/src/commands/v2-migrations/20171122100800_common_indexes.js index 498d72e7c6..f0ce759985 100644 --- a/cli/src/commands/v2-migrations/20171122100800_common_indexes.js +++ b/cli/src/commands/v2-migrations/20171122100800_common_indexes.js @@ -32,7 +32,6 @@ const createStatementIndexes = (connection) => { createIndexWithVoid('timestamp__id', { timestamp: -1, _id: -1 }); createIndexWithVoid('stored__id', { stored: -1, _id: -1 }); createIndex(stmts, 'statementId_lrs_id', { 'statement.id': 1, lrs_id: 1 }); - createIndexWithOrg('timestamp__id', { timestamp: -1, _id: 1 }); createIndexWithOrg('stored__id', { stored: -1, _id: 1 }); createIndexWithOrg('objId', objectIdIndex); createIndexWithOrg('verbId_objId', { ...verbIdIndex, ...objectIdIndex }); diff --git a/lib/helpers/convert$personaIdent.js b/lib/helpers/convert$personaIdent.js index aa49bde08d..aa1b4957a9 100644 --- a/lib/helpers/convert$personaIdent.js +++ b/lib/helpers/convert$personaIdent.js @@ -1,7 +1,7 @@ import _, { map, without } from 'lodash'; import getService from 'lib/connections/personaService'; import logger from 'lib/logger'; -import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions'; +import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions'; import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo'; /** @@ -30,8 +30,7 @@ const convertOpValue = async (identifier, { filter, organisation, limit: Number.MAX_SAFE_INTEGER, - maxTimeMS: MAX_TIME_MS, - maxScan: MAX_SCAN + maxTimeMS: MAX_TIME_MS }); attributes = map(result.edges, ({ node }) => node); } catch (err) { diff --git a/lib/models/plugins/addCRUDFunctions.js b/lib/models/plugins/addCRUDFunctions.js index 75502c5575..26598e2232 100644 --- a/lib/models/plugins/addCRUDFunctions.js +++ b/lib/models/plugins/addCRUDFunctions.js @@ -13,7 +13,6 @@ const FORWARDS = 'forwards'; export const MAX_TIME_MS = process.env.MAX_TIME_MS ? Number(process.env.MAX_TIME_MS) : 0; -export const MAX_SCAN = process.env.MAX_SCAN ? Number(process.env.MAX_SCAN) : 0; const sortDirectionToOperator = (direction, paginationDirection) => { if (paginationDirection === FORWARDS) { @@ -71,9 +70,9 @@ const modelToCursor = (model, sort) => { return cursor; }; -const getLrsStatementCount = ({ orgFilter, maxTimeMS, maxScan, hint }) => { +const getLrsStatementCount = ({ orgFilter, maxTimeMS, hint }) => { const LRS = getDBConnection().model('Lrs'); - const query = LRS.collection.find(orgFilter, {}, { maxTimeMS, maxScan }); + const query = LRS.collection.find(orgFilter, {}, { maxTimeMS }); if (hint) query.hint(hint); return query .project({ statementCount: 1 }) @@ -149,8 +148,7 @@ async function getConnection({ authInfo, hint, project, - maxTimeMS = MAX_TIME_MS, - maxScan = MAX_SCAN + maxTimeMS = MAX_TIME_MS }) { const modelName = this.modelName; const actionName = 'view'; @@ -162,7 +160,7 @@ async function getConnection({ actionName, }); - const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan }); + const query = this.find(parsedScopeFilter, {}, { maxTimeMS }); // apply sort query.sort(sort); @@ -265,7 +263,6 @@ async function getCount({ filter, authInfo, maxTimeMS = MAX_TIME_MS, - maxScan = MAX_SCAN, hint }) { const parsedScopeFilter = await getParsedScopedFilter({ @@ -283,11 +280,10 @@ async function getCount({ return getLrsStatementCount({ orgFilter: parsedScopeFilter, maxTimeMS, - maxScan, hint }); } - const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan }); + const query = this.find(parsedScopeFilter, {}, { maxTimeMS }); if (hint) query.hint(hint); return query.select({ _id: 0, organisation: 1 }).countDocuments(); } diff --git a/lib/models/statement.js b/lib/models/statement.js index 511c6543d4..1583a4de9e 100644 --- a/lib/models/statement.js +++ b/lib/models/statement.js @@ -33,7 +33,6 @@ const ASYNC_AGGREGATION_CACHE_SECONDS = defaultTo(Number(process.env.ASYNC_AGGRE const ASYNC_AGGREGATION_TIMEOUT_MS = defaultTo(Number(process.env.ASYNC_AGGREGATION_TIMEOUT_MS), 0); const ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS = defaultTo(Number(process.env.ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS), 60); // 1 minute const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0); -const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0); let Statement; const schema = new mongoose.Schema({ @@ -107,11 +106,10 @@ schema.post('remove', async (statement, next) => { * limit * batchSize * maxTimeMS - * maxScan * } * @return {Promise} */ -const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxScan }) => { +const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS }) => { let query = Statement .aggregate(pipeline) .read('secondaryPreferred') @@ -122,7 +120,6 @@ const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxSca query.options = {}; } query.options.maxTimeMS = maxTimeMS; - query.options.maxScan = maxScan; return Promise.resolve(query .cursor({ batchSize }) @@ -187,7 +184,6 @@ schema.statics.aggregateByAuth = function aggregateByAuth( batchSize = 100, getStream = false, maxTimeMS = MAX_TIME_MS, - maxScan = MAX_SCAN, sampleSize = undefined }, cb = () => { } @@ -220,13 +216,12 @@ schema.statics.aggregateByAuth = function aggregateByAuth( limit, batchSize, maxTimeMS, - maxScan, }).then(stream => cb(null, stream)); } if (cache === false) { return streamAggregation({ - pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS, maxScan + pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS }).then(stream => streamToStringResult(stream) ).then((result) => { @@ -322,7 +317,6 @@ export const runAggregationAsync = async ( limit, batchSize: 100, maxTimeMS: ASYNC_AGGREGATION_TIMEOUT_MS, - maxScan: MAX_SCAN, }).then(streamToStringResult); const startedAt = await redisClient.get(`${prefix}-STARTED-AT`); diff --git a/lib/services/statements/aggregate.js b/lib/services/statements/aggregate.js index 2560e1188c..221a6527c8 100644 --- a/lib/services/statements/aggregate.js +++ b/lib/services/statements/aggregate.js @@ -62,13 +62,12 @@ export default async ({ skip, cache, maxTimeMS, - maxScan, pipeline, sampleSize }) => { const pipelineWithDashboard = await dashboardPipeline({ authInfo, pipeline }); - const options = { skip, limit, cache, batchSize: 100, maxTimeMS, maxScan, sampleSize }; + const options = { skip, limit, cache, batchSize: 100, maxTimeMS, sampleSize }; const organisationId = await checkDashboardAuth(authInfo, pipelineWithDashboard); const authInfoWithOrgId = setOrgIdOnAuthInfo(authInfo, organisationId); diff --git a/lib/services/statements/countStatements.js b/lib/services/statements/countStatements.js index 759a4d05b9..b3db521253 100644 --- a/lib/services/statements/countStatements.js +++ b/lib/services/statements/countStatements.js @@ -1,13 +1,12 @@ import Statement from 'lib/models/statement'; import parseQuery from 'lib/helpers/parseQuery'; -export default async ({ filter, authInfo, maxTimeMS, maxScan, hint }) => { +export default async ({ filter, authInfo, maxTimeMS, hint }) => { const parsedFilter = await parseQuery(filter, { authInfo }); return Statement.getCount({ filter: parsedFilter, authInfo, maxTimeMS, - maxScan, hint }); };