feat: Support environent variables in config

.yarn/versions/7297fb1f.yml

@@ -0,0 +1,32 @@
+releases:
+  "@yarnpkg/core": prerelease
+  "@yarnpkg/parsers": prerelease
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-node-modules"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/cli"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/shell"

packages/gatsby/src/pages/configuration/yarnrc.json

@@ -1,7 +1,7 @@
 {
   "title": "JSON Schema for Yarnrc files",
   "$schema": "https://json-schema.org/draft/2019-09/schema#",
-  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). Starting from the v2, they **must** be written in valid Yaml and have the right extension (simply calling your file `.yarnrc` won't do).\n\nThose settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
+  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). Starting from the v2, they **must** be written in valid Yaml and have the right extension (simply calling your file `.yarnrc` won't do).\n\nEnvironment variables can be accessed from setting definitions by using the `${NAME}` syntax when defining the values. By default Yarn will require the variables to be present, but this can be turned off by using either `${NAME-fallback}` (which will return `fallback` if `NAME` isn't set) or `${NAME:-fallback}` (which will return `fallback` if `NAME` isn't set, or is an empty string).\n\nFinally, note that most settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
   "type": "object",
   "properties": {
     "bstatePath": {

packages/yarnpkg-core/sources/Configuration.ts

@@ -438,17 +438,21 @@ function parseSingleValue(configuration: Configuration, path: string, value: unk
     if (typeof value !== `string`)
       throw new Error(`Expected value (${value}) to be a string`);
 
+    const valueWithReplacedVariables = miscUtils.replaceEnvVariables(value, {
+      env: process.env,
+    });
+
     switch (definition.type) {
       case SettingsType.ABSOLUTE_PATH:
-        return ppath.resolve(folder, npath.toPortablePath(value));
+        return ppath.resolve(folder, npath.toPortablePath(valueWithReplacedVariables));
       case SettingsType.LOCATOR_LOOSE:
-        return structUtils.parseLocator(value, false);
+        return structUtils.parseLocator(valueWithReplacedVariables, false);
       case SettingsType.NUMBER:
-        return parseInt(value);
+        return parseInt(valueWithReplacedVariables);
       case SettingsType.LOCATOR:
-        return structUtils.parseLocator(value);
+        return structUtils.parseLocator(valueWithReplacedVariables);
       default:
-        return value;
+        return valueWithReplacedVariables;
     }
   };
 
@@ -1049,7 +1053,15 @@ export class Configuration {
       if (this.sources.has(key) && !overwrite)
         continue;
 
-      this.values.set(key, parseValue(this, key, value, definition, folder));
+      let parsed;
+      try {
+        parsed = parseValue(this, key, data[key], definition, folder);
+      } catch (error) {
+        error.message += ` in ${source}`;
+        throw error;
+      }
+
+      this.values.set(key, parsed);
       this.sources.set(key, source);
     }
   }

packages/yarnpkg-core/sources/miscUtils.ts

@@ -1,4 +1,5 @@
 import {PortablePath, npath} from '@yarnpkg/fslib';
+import {UsageError}          from 'clipanion';
 import micromatch            from 'micromatch';
 import {Readable, Transform} from 'stream';
 
@@ -263,7 +264,7 @@ export function sortMap<T>(values: Iterable<T>, mappers: ((value: T) => string)
  *
  * @returns A `string` representing a regular expression or `null` if no glob patterns are provided
  */
-export const buildIgnorePattern = (ignorePatterns: Array<string>) => {
+export function buildIgnorePattern(ignorePatterns: Array<string>) {
   if (ignorePatterns.length === 0)
     return null;
 
@@ -273,4 +274,27 @@ export const buildIgnorePattern = (ignorePatterns: Array<string>) => {
       windows: false,
     }).source})`;
   }).join(`|`);
-};
+}
+
+export function replaceEnvVariables(value: string, {env}: {env: {[key: string]: string | undefined}}) {
+  const regex = /\${(?<variableName>[\d\w_]+)(?<colon>:)?-?(?<fallback>[^}]+)?}/g;
+
+  return value.replace(regex, (...args) => {
+    const {variableName, colon, fallback} = args[args.length - 1];
+
+    const variableExist = Object.prototype.hasOwnProperty.call(env, variableName);
+    const variableValue = process.env[variableName];
+
+    if (variableValue)
+      return variableValue;
+    if (variableExist && !variableValue && colon)
+      return fallback;
+    if (variableExist)
+      return variableValue;
+    if (fallback)
+      return fallback;
+
+    throw new UsageError(`Environment variable not found (${variableName})`);
+  });
+}
+

packages/yarnpkg-core/tests/Configuration.test.ts

@@ -33,4 +33,75 @@ describe(`Configuration`, () => {
       expect(secondToken).toEqual(SECRET);
     });
   });
+
+  describe(`Environment variables`, () => {
+    it(`should replace env variables`, async () => {
+      process.env.ENV_AUTH_TOKEN = `AAA-BBB-CCC`;
+      process.env.EMPTY_VARIABLE = ``;
+
+      await initializeConfiguration({
+        npmScopes: {
+          onlyEnv: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN}`,
+          },
+          multipleEnvs: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN}-separator-\${ENV_AUTH_TOKEN}`,
+          },
+          envInString: {
+            npmAuthToken: `beforeEnv-\${ENV_AUTH_TOKEN}-after-env`,
+          },
+          envSetWithFallback: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN-fallback-value}`,
+          },
+          unsetEnvWithFallback: {
+            npmAuthToken: `\${NOT_EXISTING_ENV-fallback-value}`,
+          },
+          emptyEnvWithStrictFallback: {
+            npmAuthToken: `\${EMPTY_VARIABLE-fallback-value}`,
+          },
+          emptyEnvWithFallback: {
+            npmAuthToken: `\${EMPTY_VARIABLE:-fallback-for-empty-value}`,
+          },
+        },
+      }, async dir => {
+        const configuration = await Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        });
+
+        const getToken = (scope: string) => configuration.get(`npmScopes`).get(scope).get(`npmAuthToken`);
+
+        const onlyEnv = getToken(`onlyEnv`);
+        const multipleEnvs = getToken(`multipleEnvs`);
+        const envInString = getToken(`envInString`);
+        const envSetWithFallback = getToken(`envSetWithFallback`);
+        const unsetEnvWithFallback = getToken(`unsetEnvWithFallback`);
+        const emptyEnvWithStrictFallback = getToken(`emptyEnvWithStrictFallback`);
+        const emptyEnvWithFallback = getToken(`emptyEnvWithFallback`);
+
+        expect(onlyEnv).toEqual(`AAA-BBB-CCC`);
+        expect(multipleEnvs).toEqual(`AAA-BBB-CCC-separator-AAA-BBB-CCC`);
+        expect(envInString).toEqual(`beforeEnv-AAA-BBB-CCC-after-env`);
+        expect(envSetWithFallback).toEqual(`AAA-BBB-CCC`);
+        expect(unsetEnvWithFallback).toEqual(`fallback-value`);
+        expect(emptyEnvWithStrictFallback).toEqual(``);
+        expect(emptyEnvWithFallback).toEqual(`fallback-for-empty-value`);
+      });
+    });
+
+    it(`should forbid unset variables`, async () => {
+      await initializeConfiguration({
+        npmScopes: {
+          onlyEnv: {
+            npmAuthToken: `\${A_VARIABLE_THAT_DEFINITELY_DOESNT_EXIST}`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).rejects.toThrow();
+      });
+    });
+  });
 });