feat: secrets (#8070)

docs/usage/self-hosted-configuration.md

@@ -265,6 +265,49 @@ Warning: this is an experimental feature and may be modified or removed in a fut
 
 ## requireConfig
 
+## secrets
+
+Secrets may be configured by a bot admin in `config.js`, which will then make them available for templating within repository configs.
+For example, to configure a `GOOGLE_TOKEN` to be accessible by all repositories:
+
+```js
+module.exports = {
+  secrets: {
+    GOOGLE_TOKEN: 'abc123',
+  },
+};
+```
+
+They can also be configured per repository, e.g.
+
+```js
+module.exports = {
+  repositories: [
+    {
+      repository: 'abc/def',
+      secrets: {
+        GOOGLE_TOKEN: 'abc123',
+      },
+    },
+  ],
+};
+```
+
+It could then be used in a repository config or preset like so:
+
+```json
+{
+  "hostRules": [
+    {
+      "domainName": "google.com",
+      "token": "{{ secrets.GOOGLE_TOKEN }}"
+    }
+  ]
+}
+```
+
+Secret names must start with a upper or lower case character and can contain only characters, digits, or underscores.
+
 ## skipInstalls
 
 By default, Renovate will use the most efficient approach to updating package files and lock files, which in most cases skips the need to perform a full module install by the bot.

lib/config/__snapshots__/secrets.spec.ts.snap

@@ -0,0 +1,34 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`config/secrets applySecretsToConfig(config) replaces secrets in a array of objects 1`] = `
+Object {
+  "hostRules": Array [
+    Object {
+      "hostType": "npm",
+      "token": "abc123==",
+    },
+  ],
+}
+`;
+
+exports[`config/secrets applySecretsToConfig(config) replaces secrets in a array of strings 1`] = `
+Object {
+  "allowedManagers": Array [
+    "npm",
+  ],
+}
+`;
+
+exports[`config/secrets applySecretsToConfig(config) replaces secrets in a subobject 1`] = `
+Object {
+  "npm": Object {
+    "npmToken": "abc123==",
+  },
+}
+`;
+
+exports[`config/secrets applySecretsToConfig(config) replaces secrets in the top level 1`] = `
+Object {
+  "npmToken": "abc123==",
+}
+`;

lib/config/definitions.ts

@@ -106,6 +106,17 @@ const options: RenovateOptions[] = [
       format: 'uri',
     },
   },
+  {
+    name: 'secrets',
+    description: 'Object containing secret name/value pairs',
+    type: 'object',
+    admin: true,
+    mergeable: true,
+    default: {},
+    additionalProperties: {
+      type: 'string',
+    },
+  },
   {
     name: 'extends',
     description:

lib/config/secrets.spec.ts

@@ -0,0 +1,101 @@
+import { defaultConfig, getName } from '../../test/util';
+import {
+  CONFIG_SECRETS_INVALID,
+  CONFIG_VALIDATION,
+} from '../constants/error-messages';
+import { applySecretsToConfig, validateConfigSecrets } from './secrets';
+
+describe(getName(__filename), () => {
+  describe('validateConfigSecrets(config)', () => {
+    it('works with default config', () => {
+      expect(() => validateConfigSecrets(defaultConfig)).not.toThrow();
+    });
+    it('returns if no secrets', () => {
+      expect(validateConfigSecrets({})).toBeUndefined();
+    });
+    it('throws if secrets is not an object', () => {
+      expect(() => validateConfigSecrets({ secrets: 'hello' } as any)).toThrow(
+        CONFIG_SECRETS_INVALID
+      );
+    });
+    it('throws for invalid secret names', () => {
+      expect(() =>
+        validateConfigSecrets({ secrets: { '123': 'abc' } })
+      ).toThrow(CONFIG_SECRETS_INVALID);
+    });
+    it('throws for non-string secret', () => {
+      expect(() =>
+        validateConfigSecrets({ secrets: { abc: 123 } } as any)
+      ).toThrow(CONFIG_SECRETS_INVALID);
+    });
+    it('throws for secrets inside repositories', () => {
+      expect(() =>
+        validateConfigSecrets({
+          repositories: [
+            { repository: 'abc/def', secrets: { abc: 123 } },
+          ] as any,
+        })
+      ).toThrow(CONFIG_SECRETS_INVALID);
+    });
+  });
+
+  describe('applySecretsToConfig(config)', () => {
+    it('works with default config', () => {
+      expect(() => applySecretsToConfig(defaultConfig)).not.toThrow();
+    });
+
+    it('throws if disallowed field is used', () => {
+      const config = {
+        prTitle: '{{ secrets.ARTIFACTORY_TOKEN }}',
+        secrets: {
+          ARTIFACTORY_TOKEN: 'abc123==',
+        },
+      };
+      expect(() => applySecretsToConfig(config)).toThrow(CONFIG_VALIDATION);
+    });
+    it('throws if an unknown secret is used', () => {
+      const config = {
+        npmToken: '{{ secrets.ARTIFACTORY_TOKEN }}',
+      };
+      expect(() => applySecretsToConfig(config)).toThrow(CONFIG_VALIDATION);
+    });
+    it('replaces secrets in the top level', () => {
+      const config = {
+        secrets: { ARTIFACTORY_TOKEN: 'abc123==' },
+        npmToken: '{{ secrets.ARTIFACTORY_TOKEN }}',
+      };
+      const res = applySecretsToConfig(config);
+      expect(res).toMatchSnapshot();
+      expect(Object.keys(res)).not.toContain('secrets');
+    });
+    it('replaces secrets in a subobject', () => {
+      const config = {
+        secrets: { ARTIFACTORY_TOKEN: 'abc123==' },
+        npm: { npmToken: '{{ secrets.ARTIFACTORY_TOKEN }}' },
+      };
+      const res = applySecretsToConfig(config);
+      expect(res).toMatchSnapshot();
+      expect(Object.keys(res)).not.toContain('secrets');
+    });
+    it('replaces secrets in a array of objects', () => {
+      const config = {
+        secrets: { ARTIFACTORY_TOKEN: 'abc123==' },
+        hostRules: [
+          { hostType: 'npm', token: '{{ secrets.ARTIFACTORY_TOKEN }}' },
+        ],
+      };
+      const res = applySecretsToConfig(config);
+      expect(res).toMatchSnapshot();
+      expect(Object.keys(res)).not.toContain('secrets');
+    });
+    it('replaces secrets in a array of strings', () => {
+      const config = {
+        secrets: { SECRET_MANAGER: 'npm' },
+        allowedManagers: ['{{ secrets.SECRET_MANAGER }}'],
+      };
+      const res = applySecretsToConfig(config);
+      expect(res).toMatchSnapshot();
+      expect(Object.keys(res)).not.toContain('secrets');
+    });
+  });
+});

lib/config/secrets.ts

@@ -0,0 +1,124 @@
+import is from '@sindresorhus/is';
+import {
+  CONFIG_SECRETS_INVALID,
+  CONFIG_VALIDATION,
+} from '../constants/error-messages';
+import { logger } from '../logger';
+import { regEx } from '../util/regex';
+import { add } from '../util/sanitize';
+import { GlobalConfig, RenovateConfig } from './types';
+
+const secretNamePattern = '[A-Za-z][A-Za-z0-9_]*';
+
+const secretNameRegex = regEx(`^${secretNamePattern}$`);
+const secretTemplateRegex = regEx(`{{ secrets\\.(${secretNamePattern}) }}`);
+
+function validateSecrets(secrets_: unknown): void {
+  if (!secrets_) {
+    return;
+  }
+  const validationErrors: string[] = [];
+  if (is.plainObject(secrets_)) {
+    for (const [secretName, secretValue] of Object.entries(secrets_)) {
+      if (!secretNameRegex.test(secretName)) {
+        validationErrors.push(`Invalid secret name "${secretName}"`);
+      }
+      if (!is.string(secretValue)) {
+        validationErrors.push(
+          `Secret values must be strings. Found type ${typeof secretValue} for secret ${secretName}`
+        );
+      }
+    }
+  } else {
+    validationErrors.push(
+      `Config secrets must be a plain object. Found: ${typeof secrets_}`
+    );
+  }
+  if (validationErrors.length) {
+    logger.error({ validationErrors }, 'Invalid secrets configured');
+    throw new Error(CONFIG_SECRETS_INVALID);
+  }
+}
+
+export function validateConfigSecrets(config: GlobalConfig): void {
+  validateSecrets(config.secrets);
+  if (config.repositories) {
+    for (const repository of config.repositories) {
+      if (is.plainObject(repository)) {
+        validateSecrets(repository.secrets);
+      }
+    }
+  }
+}
+
+function replaceSecretsInString(
+  key: string,
+  value: string,
+  secrets: Record<string, string>
+): string {
+  // do nothing if no secret template found
+  if (!secretTemplateRegex.test(value)) {
+    return value;
+  }
+
+  const disallowedPrefixes = ['branch', 'commit', 'group', 'pr', 'semantic'];
+  if (disallowedPrefixes.some((prefix) => key.startsWith(prefix))) {
+    const error = new Error(CONFIG_VALIDATION);
+    error.configFile = 'config';
+    error.validationError = 'Disallowed secret substitution';
+    error.validationMessage = `The field ${key} may not use secret substitution`;
+    throw error;
+  }
+  return value.replace(secretTemplateRegex, (_, secretName) => {
+    if (secrets[secretName]) {
+      return secrets[secretName];
+    }
+    const error = new Error(CONFIG_VALIDATION);
+    error.configFile = 'config';
+    error.validationError = 'Unknown secret name';
+    error.validationMessage = `The following secret name was not found in config: ${String(
+      secretName
+    )}`;
+    throw error;
+  });
+}
+
+function replaceSecretsinObject(
+  config_: RenovateConfig,
+  secrets: Record<string, string> = {}
+): RenovateConfig {
+  const config = { ...config_ };
+  delete config.secrets;
+  for (const [key, value] of Object.entries(config)) {
+    if (is.plainObject(value)) {
+      config[key] = replaceSecretsinObject(value, secrets);
+    }
+    if (is.string(value)) {
+      config[key] = replaceSecretsInString(key, value, secrets);
+    }
+    if (is.array(value)) {
+      for (const [arrayIndex, arrayItem] of value.entries()) {
+        if (is.plainObject(arrayItem)) {
+          config[key][arrayIndex] = replaceSecretsinObject(arrayItem, secrets);
+        } else if (is.string(arrayItem)) {
+          config[key][arrayIndex] = replaceSecretsInString(
+            key,
+            arrayItem,
+            secrets
+          );
+        }
+      }
+    }
+  }
+  return config;
+}
+
+export function applySecretsToConfig(config: RenovateConfig): RenovateConfig {
+  // Add all secrets to be sanitized
+  if (is.plainObject(config.secrets)) {
+    for (const secret of Object.values(config.secrets)) {
+      add(String(secret));
+    }
+  }
+  return replaceSecretsinObject(config, config.secrets);
+}

lib/config/types.ts

@@ -127,6 +127,7 @@ export type RenovateRepository =
   | string
   | {
       repository: string;
+      secrets?: Record<string, string>;
     };
 
 export interface CustomManager {
@@ -189,6 +190,7 @@ export interface RenovateConfig
   regexManagers?: CustomManager[];
 
   fetchReleaseNotes?: boolean;
+  secrets?: Record<string, string>;
 }
 
 export interface GlobalConfig extends RenovateConfig, GlobalOnlyConfig {}

lib/constants/error-messages.ts

@@ -13,6 +13,7 @@ export const PLATFORM_RATE_LIMIT_EXCEEDED = 'rate-limit-exceeded';
 // Config Error
 export const CONFIG_VALIDATION = 'config-validation';
 export const CONFIG_SECRETS_EXPOSED = 'config-secrets-exposed';
+export const CONFIG_SECRETS_INVALID = 'config-secrets-invalid';
 
 // Repository Errors - causes repo to be considered as disabled
 export const REPOSITORY_ACCESS_FORBIDDEN = 'forbidden';

lib/logger/index.spec.ts

@@ -154,6 +154,9 @@ describe('logger', () => {
       buffer: Buffer.from('test'),
       content: 'test',
       prBody: 'test',
+      secrets: {
+        foo: 'barsecret',
+      },
     });
 
     expect(logged.foo).not.toEqual('secret"password');
@@ -164,5 +167,6 @@ describe('logger', () => {
     expect(logged.buffer).toEqual('[content]');
     expect(logged.content).toEqual('[content]');
     expect(logged.prBody).toEqual('[Template]');
+    expect(logged.secrets.foo).toEqual('***********');
   });
 });

lib/logger/utils.ts

@@ -135,6 +135,11 @@ export function sanitizeValue(value: unknown, seen = new WeakMap()): any {
         curValue = '[content]';
       } else if (templateFields.includes(key)) {
         curValue = '[Template]';
+      } else if (key === 'secrets') {
+        curValue = {};
+        Object.keys(val).forEach((secretKey) => {
+          curValue[secretKey] = '***********';
+        });
       } else {
         curValue = seen.has(val) ? seen.get(val) : sanitizeValue(val, seen);
       }