refactor: Upgrade ajv to v8 along with related packages

.github/workflows/integrate.yml

@@ -9,6 +9,7 @@ on:
 env:
   SLS_IGNORE_WARNING: '*'
   FORCE_COLOR: 1
+  SLS_SCHEMA_CACHE_BASE_DIR: '/home/runner'
 
 jobs:
   linuxNode16:

.github/workflows/validate.yml

@@ -9,6 +9,7 @@ on:
 env:
   SLS_IGNORE_WARNING: '*'
   FORCE_COLOR: 1
+  SLS_SCHEMA_CACHE_BASE_DIR: '/home/runner'
 
 jobs:
   linuxNode16:

lib/Serverless.js

@@ -432,7 +432,7 @@ class Serverless {
       this.service.setFunctionNames(this.processedInput.options);
 
       // If in context of service, validate the service configuration
-      if (this.serviceDir) this.service.validate();
+      if (this.serviceDir) await this.service.validate();
     }
 
     this.serviceOutputs = new Map();

lib/classes/ConfigSchemaHandler/index.js

@@ -1,11 +1,11 @@
 'use strict';
 
-const Ajv = require('ajv');
 const _ = require('lodash');
 const ensurePlainObject = require('type/plain-object/ensure');
 const schema = require('../../configSchema');
 const ServerlessError = require('../../serverless-error');
 const normalizeAjvErrors = require('./normalizeAjvErrors');
+const resolveAjvValidate = require('./resolveAjvValidate');
 const { legacy, log, style } = require('@serverless/utils/log');
 
 const FUNCTION_NAME_PATTERN = '^[a-zA-Z0-9-_]+$';
@@ -75,7 +75,7 @@ class ConfigSchemaHandler {
     return configurationValidationResults.get(ensurePlainObject(configuration));
   }
 
-  validateConfig(userConfig) {
+  async validateConfig(userConfig) {
     if (!this.schema.properties.provider.properties.name) {
       configurationValidationResults.set(this.serverless.configurationInput, false);
       if (this.serverless.service.configValidationMode !== 'off') {
@@ -119,11 +119,9 @@ class ConfigSchemaHandler {
       this.relaxProviderSchema();
     }
 
-    const ajv = new Ajv({ allErrors: true, coerceTypes: 'array', verbose: true });
-    require('ajv-keywords')(ajv, 'regexp');
     // Workaround https://github.com/ajv-validator/ajv/issues/1255
     normalizeSchemaObject(this.schema, this.schema);
-    const validate = ajv.compile(this.schema);
+    const validate = await resolveAjvValidate(this.schema);
 
     const denormalizeOptions = normalizeUserConfig(userConfig);
     validate(userConfig);

lib/classes/ConfigSchemaHandler/normalizeAjvErrors.js

@@ -1,24 +1,22 @@
 'use strict';
 
 const _ = require('lodash');
-const resolveDataPathSize = require('./resolveDataPathSize');
 
-const isEventTypeDataPath = RegExp.prototype.test.bind(/^\.functions\[[^\]]+\]\.events\[\d+\]$/);
+const isEventTypeInstancePath = RegExp.prototype.test.bind(/^\/functions\/[^/]+\/events\/\d+$/);
 const oneOfPathPattern = /\/(?:anyOf|oneOf)(?:\/\d+\/|$)/;
 const isAnyOfPathTypePostfix = RegExp.prototype.test.bind(/^\/\d+\/type$/);
-const dataPathPropertyBracketsPattern = /\['([a-zA-Z_0-9]+)'\]/g;
 const oneOfKeywords = new Set(['anyOf', 'oneOf']);
 
 const filterIrreleventEventConfigurationErrors = (resultErrorsSet) => {
   // 1. Resolve all errors at event type configuration level
-  const eventTypeErrors = Array.from(resultErrorsSet).filter(({ dataPath }) =>
-    isEventTypeDataPath(dataPath)
+  const eventTypeErrors = Array.from(resultErrorsSet).filter(({ instancePath }) =>
+    isEventTypeInstancePath(instancePath)
   );
   // 2. Group event type configuration errors by event instance
-  const eventTypeErrorsByEvent = _.groupBy(eventTypeErrors, ({ dataPath }) => dataPath);
+  const eventTypeErrorsByEvent = _.groupBy(eventTypeErrors, ({ instancePath }) => instancePath);
 
   // 3. Process each error group individually
-  for (const [dataPath, eventEventTypeErrors] of Object.entries(eventTypeErrorsByEvent)) {
+  for (const [instancePath, eventEventTypeErrors] of Object.entries(eventTypeErrorsByEvent)) {
     // 3.1 Resolve error that signals that no event schema was matched
     const noMatchingEventError = eventEventTypeErrors.find(({ keyword }) =>
       oneOfKeywords.has(keyword)
@@ -43,7 +41,8 @@ const filterIrreleventEventConfigurationErrors = (resultErrorsSet) => {
       // 3.4 If there are no event type configuration errors for intended event type
       if (
         !Array.from(resultErrorsSet).some(
-          (error) => error.dataPath.startsWith(dataPath) && error.dataPath !== dataPath
+          (error) =>
+            error.instancePath.startsWith(instancePath) && error.instancePath !== instancePath
         )
       ) {
         // 3.4.1 If there are no event configuration errors, it means it's not supported event type:
@@ -69,8 +68,11 @@ const filterIrreleventEventConfigurationErrors = (resultErrorsSet) => {
       //       - All event configuration errors
       const meaningfulSchemaPath = eventConfiguredEventTypeErrors[0];
       for (const error of resultErrorsSet) {
-        if (!error.dataPath.startsWith(dataPath)) continue;
-        if (error.dataPath === dataPath && error.schemaPath.startsWith(meaningfulSchemaPath)) {
+        if (!error.instancePath.startsWith(instancePath)) continue;
+        if (
+          error.instancePath === instancePath &&
+          error.schemaPath.startsWith(meaningfulSchemaPath)
+        ) {
           continue;
         }
         resultErrorsSet.delete(error);
@@ -98,82 +100,88 @@ const filterIrrelevantAnyOfErrors = (resultErrorsSet) => {
   for (const [oneOfPath, oneOfPathErrors] of Object.entries(oneOfErrorsByPath)) {
     // 2.1. If just one error, set was already filtered by event configuration errors filter
     if (oneOfPathErrors.length === 1) continue;
-    // 2.2. Group by dataPath
-    oneOfPathErrors.sort(({ dataPath: dataPathA }, { dataPath: dataPathB }) =>
-      dataPathA.localeCompare(dataPathB)
+    // 2.2. Group by instancePath
+    oneOfPathErrors.sort(({ instancePath: instancePathA }, { instancePath: instancePathB }) =>
+      instancePathA.localeCompare(instancePathB)
     );
-    let currentDataPath = oneOfPathErrors[0].dataPath;
+    let currentInstancePath = oneOfPathErrors[0].instancePath;
     Object.values(
-      _.groupBy(oneOfPathErrors, ({ dataPath }) => {
+      _.groupBy(oneOfPathErrors, ({ instancePath }) => {
         if (
-          dataPath !== currentDataPath &&
-          !dataPath.startsWith(`${currentDataPath}.`) &&
-          !dataPath.startsWith(`${currentDataPath}[`)
+          instancePath !== currentInstancePath &&
+          !instancePath.startsWith(`${currentInstancePath}/`)
         ) {
-          currentDataPath = dataPath;
+          currentInstancePath = instancePath;
         }
-        return currentDataPath;
+        return currentInstancePath;
       })
-    ).forEach((dataPathOneOfPathErrors) => {
+    ).forEach((instancePathOneOfPathErrors) => {
       // 2.2.1.If just one error, set was already filtered by event configuration errors filter
-      if (dataPathOneOfPathErrors.length === 1) return;
+      if (instancePathOneOfPathErrors.length === 1) return;
       // 2.2.2 Group by anyOf variant
       const groupFromIndex = oneOfPath.length + 1;
-      const dataPathOneOfPathErrorsByVariant = _.groupBy(
-        dataPathOneOfPathErrors,
+      const instancePathOneOfPathErrorsByVariant = _.groupBy(
+        instancePathOneOfPathErrors,
         ({ schemaPath }) => {
           if (groupFromIndex > schemaPath.length) return 'root';
           return schemaPath.slice(groupFromIndex, schemaPath.indexOf('/', groupFromIndex));
         }
       );
 
       // 2.2.3 If no root error, set was already filtered by event configuration errors filter
-      if (!dataPathOneOfPathErrorsByVariant.root) return;
-      const noMatchingVariantError = dataPathOneOfPathErrorsByVariant.root[0];
-      delete dataPathOneOfPathErrorsByVariant.root;
-      let dataPathOneOfPathVariants = Object.values(dataPathOneOfPathErrorsByVariant);
+      if (!instancePathOneOfPathErrorsByVariant.root) return;
+      const noMatchingVariantError = instancePathOneOfPathErrorsByVariant.root[0];
+      delete instancePathOneOfPathErrorsByVariant.root;
+      let instancePathOneOfPathVariants = Object.values(instancePathOneOfPathErrorsByVariant);
       // 2.2.4 If no variants, set was already filtered by event configuration errors filter
-      if (!dataPathOneOfPathVariants.length) return;
+      if (!instancePathOneOfPathVariants.length) return;
 
-      if (dataPathOneOfPathVariants.length > 1) {
+      if (instancePathOneOfPathVariants.length > 1) {
         // 2.2.5 If errors reported for more than one variant
         // 2.2.5.1 Filter variants where value type was not met
-        dataPathOneOfPathVariants = dataPathOneOfPathVariants.filter(
-          (dataPathOneOfPathVariantErrors) => {
-            if (dataPathOneOfPathVariantErrors.length !== 1) return true;
-            if (dataPathOneOfPathVariantErrors[0].keyword !== 'type') return true;
+        instancePathOneOfPathVariants = instancePathOneOfPathVariants.filter(
+          (instancePathOneOfPathVariantErrors) => {
+            if (instancePathOneOfPathVariantErrors.length !== 1) return true;
+            if (instancePathOneOfPathVariantErrors[0].keyword !== 'type') return true;
             if (
               !isAnyOfPathTypePostfix(
-                dataPathOneOfPathVariantErrors[0].schemaPath.slice(oneOfPath.length)
+                instancePathOneOfPathVariantErrors[0].schemaPath.slice(oneOfPath.length)
               )
             ) {
               return true;
             }
-            for (const dataPathOneOfPathVariantError of dataPathOneOfPathVariantErrors) {
-              resultErrorsSet.delete(dataPathOneOfPathVariantError);
+            for (const instancePathOneOfPathVariantError of instancePathOneOfPathVariantErrors) {
+              resultErrorsSet.delete(instancePathOneOfPathVariantError);
             }
             return false;
           }
         );
-        if (dataPathOneOfPathVariants.length > 1) {
+        if (instancePathOneOfPathVariants.length > 1) {
           // 2.2.5.2 Leave out variants where errors address deepest data paths
-          let deepestDataPathSize = 0;
-          for (const dataPathOneOfPathVariantErrors of dataPathOneOfPathVariants) {
-            dataPathOneOfPathVariantErrors.deepestDataPathSize = Math.max(
-              ...dataPathOneOfPathVariantErrors.map(({ dataPath }) => resolveDataPathSize(dataPath))
+          let deepestInstancePathSize = 0;
+          for (const instancePathOneOfPathVariantErrors of instancePathOneOfPathVariants) {
+            instancePathOneOfPathVariantErrors.deepestInstancePathSize = Math.max(
+              ...instancePathOneOfPathVariantErrors.map(
+                ({ instancePath }) => (instancePath.match(/\//g) || []).length
+              )
             );
-            if (dataPathOneOfPathVariantErrors.deepestDataPathSize > deepestDataPathSize) {
-              deepestDataPathSize = dataPathOneOfPathVariantErrors.deepestDataPathSize;
+            if (
+              instancePathOneOfPathVariantErrors.deepestInstancePathSize > deepestInstancePathSize
+            ) {
+              deepestInstancePathSize = instancePathOneOfPathVariantErrors.deepestInstancePathSize;
             }
           }
 
-          dataPathOneOfPathVariants = dataPathOneOfPathVariants.filter(
-            (dataPathAnyOfPathVariantErrors) => {
-              if (dataPathAnyOfPathVariantErrors.deepestDataPathSize === deepestDataPathSize) {
+          instancePathOneOfPathVariants = instancePathOneOfPathVariants.filter(
+            (instancePathAnyOfPathVariantErrors) => {
+              if (
+                instancePathAnyOfPathVariantErrors.deepestInstancePathSize ===
+                deepestInstancePathSize
+              ) {
                 return true;
               }
-              for (const dataPathOneOfPathVariantError of dataPathAnyOfPathVariantErrors) {
-                resultErrorsSet.delete(dataPathOneOfPathVariantError);
+              for (const instancePathOneOfPathVariantError of instancePathAnyOfPathVariantErrors) {
+                resultErrorsSet.delete(instancePathOneOfPathVariantError);
               }
               return false;
             }
@@ -182,38 +190,38 @@ const filterIrrelevantAnyOfErrors = (resultErrorsSet) => {
       }
 
       // 2.2.6 If all variants were filtered, expose just "no matching variant" error
-      if (!dataPathOneOfPathVariants.length) return;
+      if (!instancePathOneOfPathVariants.length) return;
       // 2.2.7 If just one variant left, expose only errors for that variant
-      if (dataPathOneOfPathVariants.length === 1) {
+      if (instancePathOneOfPathVariants.length === 1) {
         resultErrorsSet.delete(noMatchingVariantError);
         return;
       }
       // 2.2.8 If more than one variant left, expose just "no matching variant" error
-      for (const dataPathOneOfPathVariantErrors of dataPathOneOfPathVariants) {
+      for (const instancePathOneOfPathVariantErrors of instancePathOneOfPathVariants) {
         const types = new Set();
-        for (const dataPathOneOfPathVariantError of dataPathOneOfPathVariantErrors) {
-          const parentSchema = dataPathOneOfPathVariantError.parentSchema;
+        for (const instancePathOneOfPathVariantError of instancePathOneOfPathVariantErrors) {
+          const parentSchema = instancePathOneOfPathVariantError.parentSchema;
           types.add(parentSchema.const ? 'const' : parentSchema.type);
-          resultErrorsSet.delete(dataPathOneOfPathVariantError);
+          resultErrorsSet.delete(instancePathOneOfPathVariantError);
         }
         if (types.size === 1) noMatchingVariantError.commonType = types.values().next().value;
       }
     });
   }
 };
 
-const normalizeDataPath = (dataPath) => {
-  if (!dataPath) return 'root';
+const normalizeInstancePath = (instancePath) => {
+  if (!instancePath) return 'root';
 
-  // This regex helps replace functions['someFunc'].foo with functions.someFunc.foo
-  return `'${dataPath.slice(1).replace(dataPathPropertyBracketsPattern, '.$1')}'`;
+  // This code removes leading / and replaces / with . in error path indication
+  return `'${instancePath.slice(1).replace(/\//g, '.')}'`;
 };
 
 const improveMessages = (resultErrorsSet) => {
   for (const error of resultErrorsSet) {
     switch (error.keyword) {
       case 'additionalProperties':
-        if (error.dataPath === '.functions') {
+        if (error.instancePath === '/functions') {
           error.message = `name '${error.params.additionalProperty}' must be alphanumeric`;
         } else {
           error.message = `unrecognized property '${error.params.additionalProperty}'`;
@@ -223,7 +231,7 @@ const improveMessages = (resultErrorsSet) => {
         error.message = `value '${error.data}' does not satisfy pattern ${error.schema}`;
         break;
       case 'anyOf':
-        if (isEventTypeDataPath(error.dataPath)) {
+        if (isEventTypeInstancePath(error.instancePath)) {
           error.message = 'unsupported function event';
           break;
         }
@@ -243,7 +251,7 @@ const improveMessages = (resultErrorsSet) => {
         break;
       default:
     }
-    error.message = `at ${normalizeDataPath(error.dataPath)}: ${error.message}`;
+    error.message = `at ${normalizeInstancePath(error.instancePath)}: ${error.message}`;
   }
 };
 

lib/classes/ConfigSchemaHandler/resolveAjvValidate.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const Ajv = require('ajv').default;
+const objectHash = require('object-hash');
+const path = require('path');
+const os = require('os');
+const standaloneCode = require('ajv/dist/standalone').default;
+const fs = require('fs');
+const requireFromString = require('require-from-string');
+const deepSortObjectByKey = require('../../utils/deepSortObjectByKey');
+const ensureExists = require('../../utils/ensureExists');
+
+const getCacheDir = () => {
+  return path.resolve(
+    process.env.SLS_SCHEMA_CACHE_BASE_DIR || os.homedir(),
+    `.serverless/artifacts/ajv-validate-${require('ajv/package').version}`
+  );
+};
+
+// Validators are cached by schema hash for the purpose
+// of speeding up tests and reducing their memory footprint.
+// If that solution proves to not be enough, we can improve it
+// with `uni-global` package.
+const cachedValidatorsBySchemaHash = {};
+
+const getValidate = async (schema) => {
+  const schemaHash = objectHash(deepSortObjectByKey(schema));
+  if (cachedValidatorsBySchemaHash[schemaHash]) {
+    return cachedValidatorsBySchemaHash[schemaHash];
+  }
+  const filename = `${schemaHash}.js`;
+  const cachePath = path.resolve(getCacheDir(), filename);
+
+  const generate = async () => {
+    const ajv = new Ajv({
+      allErrors: true,
+      coerceTypes: 'array',
+      verbose: true,
+      strict: true,
+      strictRequired: false,
+      code: { source: true },
+    });
+    require('ajv-formats').default(ajv);
+    // Ensure AJV related packages work well when there are mutliple AJV installations around
+    // See: https://github.com/ajv-validator/ajv/issues/1390#issuecomment-763138202
+    ajv.opts.code.formats = Ajv._`require("ajv-formats/dist/formats").fullFormats`;
+    require('ajv-keywords')(ajv, 'regexp');
+    const validate = ajv.compile(schema);
+    const moduleCode = standaloneCode(ajv, validate);
+    await fs.promises.writeFile(cachePath, moduleCode);
+  };
+  await ensureExists(cachePath, generate);
+  const loadedModuleCode = await fs.promises.readFile(cachePath, 'utf-8');
+  const validator = requireFromString(
+    loadedModuleCode,
+    path.resolve(__dirname, `[generated-ajv-validate]${filename}`)
+  );
+  cachedValidatorsBySchemaHash[schemaHash] = validator;
+  return validator;
+};
+
+module.exports = getValidate;