feat(Plugins): Support variables in configuration extensions (#11558)

docs/guides/plugins/custom-configuration.md

@@ -1,8 +1,8 @@
 <!--
 title: Serverless Framework - Plugins - Extending the configuration
-menuText: Extending the configuration
+menuText: Extending the configuration schema
 menuOrder: 5
-description: How to extend the serverless.yml syntax with custom configuration via a plugin
+description: How to extend the serverless.yml schema with custom configuration via a plugin
 layout: Doc
 -->
 
@@ -12,7 +12,7 @@ layout: Doc
 
 <!-- DOCS-SITE-LINK:END -->
 
-# Extending the configuration
+# Extending the configuration schema
 
 Plugin can extend the `serverless.yml` syntax with custom configuration:
 

docs/guides/plugins/extending-configuration.md

@@ -0,0 +1,66 @@
+<!--
+title: Serverless Framework - Plugins - Extending and overriding the configuration
+menuText: Extending and overriding configuration
+menuOrder: 6
+description: How to extend and override configuration via a plugin
+layout: Doc
+-->
+
+<!-- DOCS-SITE-LINK:START automatically generated  -->
+
+### [Read this on the main serverless docs site](https://www.serverless.com/framework/docs/guides/plugins/extending-configuration)
+
+<!-- DOCS-SITE-LINK:END -->
+
+# Extending and overriding configuration
+
+Plugins can extend and override the internal configuration.
+
+To do so, plugins may use the `serverless.extendConfiguration(...)` method.
+This is only allowed at pre-init stage of serverless.
+The method also takes care of resolving all variables in the given value. But it **does not validate you input** nor the target. Improper usage can cause serverless to fail.
+
+The `serverless.extendConfiguration(configurationPathKeys, value)` method takes two arguments.
+
+| Argument                | Type                      | Description                                                        |
+| ----------------------- | ------------------------- | ------------------------------------------------------------------ |
+| `configurationPathKeys` | string[]                  | Path of the configuration property to set; must not be empty       |
+| `value`                 | string \| object \| array | New value of the configuration property in `configurationPathKeys` |
+
+If configuration in `configurationPathKeys` **does exist** the value will be overwritten.  
+If configuration in `configurationPathKeys` **does not exist** the whole path will be created.
+
+You can use it in plugin constructor, or if for some reason configuration extension is resolved asynchronously you may resort to `asyncInit()` method
+
+```js
+class MyPlugin {
+  constructor(serverless) {
+    this.serverless = serverless;
+
+    const value = {
+      myKey: 'myValue',
+    };
+    this.serverless.extendConfiguration(['custom', 'myPlugin'], value);
+  }
+}
+
+module.exports = MyPlugin;
+```
+
+If your plugin needs merging you need to take care of it yourself.
+
+```js
+class MyPlugin {
+  constructor(serverless) {
+    this.serverless = serverless;
+
+    const currentConfig = this.serverless.configurationInput.custom.myPlugin;
+    const value = Object.assign(currentConfig, {
+      myKey: 'myValue',
+    });
+    this.serverless.extendConfiguration(['custom', 'myPlugin'], value);
+  }
+}
+
+module.exports = MyPlugin;
+```

docs/menu.json

@@ -50,7 +50,8 @@
       "CLI Output": "guides/plugins/cli-output",
       "Custom Commands": "guides/plugins/custom-commands",
       "Custom Variables": "guides/plugins/custom-variables",
-      "Extending the Configuration": "guides/plugins/custom-configuration"
+      "Extending the Configuration schema": "guides/plugins/custom-configuration",
+      "Extending and overriding configuration": "guides/plugins/extending-configuration"
     }
   },
   "Examples and Tutorials": "examples-and-tutorials",

lib/serverless.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const os = require('os');
+const _ = require('lodash');
 const ensureString = require('type/string/ensure');
 const ensureValue = require('type/value/ensure');
 const ensureArray = require('type/array/ensure');
@@ -23,6 +24,7 @@ const eventuallyUpdate = require('./utils/eventually-update');
 const commmandsSchema = require('./cli/commands-schema');
 const resolveCliInput = require('./cli/resolve-input');
 const isDashboardEnabled = require('./configuration/is-dashboard-enabled');
+const parseEntries = require('./configuration/variables/resolve-meta').parseEntries;
 
 // Old local fallback is triggered in older versions by Serverless constructor directly
 const isStackFromOldLocalFallback = RegExp.prototype.test.bind(
@@ -99,6 +101,11 @@ class Serverless {
     // Old variables resolver is dropped, yet some plugins access service properties through
     // `variables` class. Below patch ensures those plugins won't get broken
     this.variables = { service: this.service };
+
+    // `config.variablesMeta` will not be provided if the initial resolution of variables failed.
+    // We're ensuring it locally not to disrupt configuration extensions as eventually done by
+    // the plugins (which are still loaded in spite of the error, if e.g. help output was requested)
+    this.variablesMeta = config.variablesMeta || new Map([]);
     this.pluginManager = new PluginManager(this);
     this.configSchemaHandler = new ConfigSchemaHandler(this);
 
@@ -116,6 +123,7 @@ class Serverless {
     this.serverlessDirPath = path.join(os.homedir(), '.serverless');
     this.isStandaloneExecutable = isStandaloneExecutable;
     this.triggeredDeprecations = logDeprecation.triggeredDeprecations;
+    this.isConfigurationExtendable = true;
 
     // TODO: Remove once "@serverless/dashboard-plugin" is integrated into this repository
     this._commandsSchema = commmandsSchema;
@@ -136,6 +144,7 @@ class Serverless {
     await this.service.load(this.processedInput.options);
     // load all plugins
     await this.pluginManager.loadAllPlugins(this.service.plugins);
+    this.isConfigurationExtendable = false;
     // give the CLI the plugins and commands so that it can print out
     // information such as options when the user enters --help
     this.cli.setLoadedPlugins(this.pluginManager.getPlugins());
@@ -215,6 +224,41 @@ class Serverless {
   logDeprecation(code, message) {
     return this._logDeprecation(`EXT_${ensureString(code)}`, ensureString(message));
   }
+
+  extendConfiguration(configurationPathKeys, value) {
+    configurationPathKeys = ensureArray(configurationPathKeys, {
+      ensureItem: ensureString,
+    });
+    if (configurationPathKeys.length < 1) {
+      throw new Error(
+        'Cannot extend configuration: ConfigurationPathKeys needs to contain at least one element.'
+      );
+    }
+
+    if (!this.isConfigurationExtendable) {
+      throw new Error(
+        'Cannot extend configuration: It can only be extended during initialization phase.'
+      );
+    }
+    try {
+      value = JSON.parse(JSON.stringify(value));
+    } catch (error) {
+      throw new Error(`Cannot extend configuration: Received non JSON value: ${value}`);
+    }
+
+    _.set(this.configurationInput, configurationPathKeys, value);
+    const metaPathPrefix = configurationPathKeys.join('\0');
+    for (const key of this.variablesMeta.keys()) {
+      if (key === metaPathPrefix || key.startsWith(`${metaPathPrefix}\0`)) {
+        this.variablesMeta.delete(key);
+      }
+    }
+    if (!_.isObject(value)) {
+      const lastKey = configurationPathKeys.pop();
+      value = { [lastKey]: value };
+    }
+    parseEntries(Object.entries(value), configurationPathKeys, this.variablesMeta);
+  }
 }
 
 module.exports = Serverless;

scripts/serverless.js

@@ -319,6 +319,7 @@ processSpanPromise = (async () => {
               propertyPathsToResolve: new Set(['provider\0name', 'provider\0stage', 'useDotenv']),
               variableSourcesInConfig,
             };
+
             if (isInteractiveSetup) resolverConfiguration.fulfilledSources.add('opt');
             await resolveVariables(resolverConfiguration);
 
@@ -596,6 +597,7 @@ processSpanPromise = (async () => {
       configurationFilename,
       commands,
       options,
+      variablesMeta,
     });
 
     try {
@@ -642,6 +644,31 @@ processSpanPromise = (async () => {
         if (hasFinalCommandSchema) require('../lib/cli/ensure-supported-command')(configuration);
         if (isHelpRequest) return;
         if (!_.get(variablesMeta, 'size')) return;
+        if (!resolverConfiguration) {
+          // There were no variables in the initial configuration, yet it was extended by
+          // the plugins with ones.
+          // In this case we need to ensure `resolverConfiguration` which initially was not setup
+          resolverConfiguration = {
+            serviceDir,
+            configuration,
+            variablesMeta,
+            sources: {
+              env: require('../lib/configuration/variables/sources/env'),
+              file: require('../lib/configuration/variables/sources/file'),
+              opt: require('../lib/configuration/variables/sources/opt'),
+              self: require('../lib/configuration/variables/sources/self'),
+              strToBool: require('../lib/configuration/variables/sources/str-to-bool'),
+              sls: require('../lib/configuration/variables/sources/instance-dependent/get-sls')(),
+            },
+            options: filterSupportedOptions(options, { commandSchema, providerName }),
+            fulfilledSources: new Set(['env', 'file', 'self', 'strToBool']),
+            propertyPathsToResolve:
+              commands[0] === 'plugin'
+                ? new Set(['plugins', 'provider\0name', 'provider\0stage', 'useDotenv'])
+                : null,
+            variableSourcesInConfig,
+          };
+        }
 
         if (commandSchema) {
           resolverConfiguration.options = filterSupportedOptions(options, {

test/fixtures/programmatic/plugin/extend-config-plugin/index.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const pluginConfig = {
+  targetValuePath: ['custom', 'extend', 'value'],
+  overwriteValuePath: ['custom', 'extend', 'overwrite'],
+  afterInitValuePath: ['custom', 'extend', 'afterInit'],
+  refValuePath: ['custom', 'extend', 'ref'],
+};
+
+module.exports = class TestPlugin {
+  constructor(serverless, options, utils) {
+    this.serverless = serverless;
+    this.options = options;
+    this.utils = utils;
+
+    this.hooks = {
+      initialize: () => this.extendAfterInit(),
+    };
+  }
+
+  async asyncInit() {
+    const configExt = {
+      var: 'value',
+    };
+    this.serverless.extendConfiguration(pluginConfig.targetValuePath, configExt);
+    this.serverless.extendConfiguration(pluginConfig.overwriteValuePath, configExt);
+    this.serverless.extendConfiguration(pluginConfig.refValuePath, '${self:custom.extend.value}');
+
+    try {
+      this.serverless.extendConfiguration([], { custom: {} });
+    } catch (error) {
+      // ignore this
+    }
+
+    try {
+      this.serverless.extendConfiguration('custom.target.invalid', {});
+    } catch (error) {
+      // ignore this
+    }
+  }
+
+  extendAfterInit() {
+    try {
+      this.serverless.extendConfiguration(pluginConfig.afterInitValuePath, 'value');
+    } catch (error) {
+      // ignore this
+    }
+  }
+};
+
+module.exports.pluginConfig = pluginConfig;

test/unit/lib/serverless.test.js

@@ -17,6 +17,11 @@ const ConfigSchemaHandler = require('../../../lib/classes/config-schema-handler'
 const CLI = require('../../../lib/classes/cli');
 const ServerlessError = require('../../../lib/serverless-error');
 const runServerless = require('../../utils/run-serverless');
+const spawn = require('child-process-ext/spawn');
+const programmaticFixturesEngine = require('../../fixtures/programmatic');
+const path = require('path');
+const yaml = require('js-yaml');
+const _ = require('lodash');
 
 describe('Serverless', () => {
   let serverless;
@@ -174,4 +179,43 @@ describe('test/unit/lib/serverless.test.js', () => {
       expect(serverless.config).to.have.property('servicePath');
     });
   });
+
+  describe('Extend configuration', () => {
+    const pluginConfig =
+      require('../../fixtures/programmatic/plugin/extend-config-plugin').pluginConfig;
+
+    const serverlessPath = path.resolve(__dirname, '../../../scripts/serverless.js');
+
+    it('Extends configuration with given values', async () => {
+      const customExt = { custom: {} };
+      const configExt = {
+        plugins: ['./extend-config-plugin/index.js'],
+        provider: {
+          stage: 'dev',
+        },
+        custom: {},
+      };
+      _.set(customExt, pluginConfig.overwriteValuePath, 'test_value');
+
+      const { servicePath: serviceDir } = await programmaticFixturesEngine.setup('plugin', {
+        configExt,
+      });
+      const serverlessProcess = await spawn('node', [serverlessPath, 'print'], {
+        cwd: serviceDir,
+      });
+      const configuration = yaml.load(String(serverlessProcess.stdoutBuffer));
+
+      const targetValue = _.get(configuration, pluginConfig.targetValuePath);
+      expect(targetValue, 'Target value should not be undefined').to.not.be.undefined;
+
+      const afterInitValue = _.get(configuration, pluginConfig.afterInitValuePath);
+      expect(afterInitValue, 'afterInitValue should be undefined').to.be.undefined;
+
+      const refValue = _.get(configuration, pluginConfig.refValuePath);
+      expect(refValue).to.deep.equal(targetValue, 'refValue should equal targetValue');
+
+      const overwriteValue = _.get(configuration, pluginConfig.overwriteValuePath);
+      expect(overwriteValue).to.deep.equal(targetValue, 'overwriteValue should equal targetValue');
+    });
+  });
 });