feat(internal): generate dynamic manager imports (#5818)

lib/config/definitions.spec.ts

@@ -2,7 +2,7 @@ import { getOptions } from './definitions';
 import { getName } from '../../test/util';
 
 jest.mock('../manager', () => ({
-  getManagers: jest.fn(() => ({ testManager: {} })),
+  getManagers: jest.fn(() => new Map().set('testManager', {})),
 }));
 
 describe(getName(__filename), () => {

lib/config/definitions.ts

@@ -1738,7 +1738,7 @@ export function getOptions(): RenovateOptions[] {
 }
 
 function loadManagerOptions(): void {
-  for (const [name, config] of Object.entries(getManagers())) {
+  for (const [name, config] of getManagers().entries()) {
     if (config.defaultConfig) {
       const managerConfig: RenovateOptions = {
         name,

lib/manager/index.spec.ts

@@ -1,4 +1,6 @@
 import * as manager from '.';
+import { ManagerApi } from './common';
+import { loadModules } from '../util/modules';
 
 describe('manager', () => {
   describe('get()', () => {
@@ -16,28 +18,127 @@ describe('manager', () => {
       expect(manager.getManagerList()).not.toBeNull();
     });
   });
+
+  it('validates', () => {
+    function validate(module: ManagerApi): boolean {
+      if (!module.defaultConfig) {
+        return false;
+      }
+      if (!module.updateDependency && !module.autoReplace) {
+        return false;
+      }
+      if (!module.extractPackageFile && !module.extractAllPackageFiles) {
+        return false;
+      }
+      return true;
+    }
+    const mgrs = manager.getManagers();
+
+    const loadedMgr = loadModules(__dirname, validate);
+    expect(Array.from(mgrs.keys())).toEqual(Object.keys(loadedMgr));
+
+    for (const name of mgrs.keys()) {
+      const mgr = mgrs.get(name);
+      expect(validate(mgr)).toBe(true);
+    }
+  });
+
   describe('extractAllPackageFiles()', () => {
     it('returns null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+      });
       expect(
-        manager.extractAllPackageFiles('dockerfile', {} as any, [])
+        manager.extractAllPackageFiles('unknown', {} as any, [])
       ).toBeNull();
+      expect(manager.extractAllPackageFiles('dummy', {} as any, [])).toBeNull();
     });
     it('returns non-null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+        extractAllPackageFiles: () => Promise.resolve([]),
+      });
       expect(
-        manager.extractAllPackageFiles('npm', {} as any, [])
+        manager.extractAllPackageFiles('dummy', {} as any, [])
       ).not.toBeNull();
     });
+    afterEach(() => {
+      manager.getManagers().delete('dummy');
+    });
   });
 
   describe('extractPackageFile()', () => {
     it('returns null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+      });
       expect(manager.extractPackageFile('unknown', null)).toBeNull();
+      expect(manager.extractPackageFile('dummy', null)).toBeNull();
+    });
+    it('returns non-null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+        extractPackageFile: () => Promise.resolve({ deps: [] }),
+      });
+
+      expect(manager.extractPackageFile('dummy', null)).not.toBeNull();
+    });
+    afterEach(() => {
+      manager.getManagers().delete('dummy');
     });
   });
 
   describe('getPackageUpdates', () => {
     it('returns null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+      });
       expect(manager.getPackageUpdates('unknown', null)).toBeNull();
+      expect(manager.getPackageUpdates('dummy', null)).toBeNull();
+    });
+    it('returns non-null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+        getPackageUpdates: () => Promise.resolve([]),
+      });
+      expect(manager.getPackageUpdates('dummy', {} as any)).not.toBeNull();
+    });
+    afterEach(() => {
+      manager.getManagers().delete('dummy');
+    });
+  });
+
+  describe('getRangeStrategy', () => {
+    it('returns null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+      });
+      expect(
+        manager.getRangeStrategy({ manager: 'unknown', rangeStrategy: 'auto' })
+      ).toBeNull();
+    });
+    it('returns non-null', () => {
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+        getRangeStrategy: () => 'replace',
+      });
+      expect(
+        manager.getRangeStrategy({ manager: 'dummy', rangeStrategy: 'auto' })
+      ).not.toBeNull();
+
+      manager.getManagers().set('dummy', {
+        defaultConfig: {},
+      });
+      expect(
+        manager.getRangeStrategy({ manager: 'dummy', rangeStrategy: 'auto' })
+      ).not.toBeNull();
+
+      expect(
+        manager.getRangeStrategy({ manager: 'dummy', rangeStrategy: 'bump' })
+      ).not.toBeNull();
+    });
+    afterEach(() => {
+      manager.getManagers().delete('dummy');
     });
   });
 });

lib/manager/index.ts

@@ -21,29 +21,9 @@ import {
   LANGUAGE_RUBY,
   LANGUAGE_RUST,
 } from '../constants/languages';
-import { loadModules } from '../util/modules';
-import { logger } from '../logger';
+import managers from './api.generated';
 
-// istanbul ignore next
-function validateManager(manager): boolean {
-  if (!manager.defaultConfig) {
-    logger.fatal(`manager is missing defaultConfig`);
-    return false;
-  }
-  if (!manager.updateDependency && !manager.autoReplace) {
-    logger.fatal(`manager is missing updateDependency`);
-    return false;
-  }
-  if (!manager.extractPackageFile && !manager.extractAllPackageFiles) {
-    logger.fatal(
-      `manager must support extractPackageFile or extractAllPackageFiles`
-    );
-  }
-  return true;
-}
-
-const managers = loadModules<ManagerApi>(__dirname, validateManager);
-const managerList = Object.keys(managers);
+const managerList = Array.from(managers.keys());
 
 const languageList = [
   LANGUAGE_DART,
@@ -59,31 +39,39 @@ const languageList = [
   LANGUAGE_RUST,
 ];
 
-export const get = <T extends keyof ManagerApi>(
+export function get<T extends keyof ManagerApi>(
   manager: string,
   name: T
-): ManagerApi[T] => managers[manager][name];
+): ManagerApi[T] | null {
+  return managers.get(manager)?.[name];
+}
 export const getLanguageList = (): string[] => languageList;
 export const getManagerList = (): string[] => managerList;
-export const getManagers = (): Record<string, ManagerApi> => managers;
+export const getManagers = (): Map<string, ManagerApi> => managers;
 
 export function extractAllPackageFiles(
   manager: string,
   config: ExtractConfig,
   files: string[]
 ): Result<PackageFile[] | null> {
-  return managers[manager] && managers[manager].extractAllPackageFiles
-    ? managers[manager].extractAllPackageFiles(config, files)
+  if (!managers.has(manager)) {
+    return null;
+  }
+  const m = managers.get(manager);
+  return m.extractAllPackageFiles
+    ? m.extractAllPackageFiles(config, files)
     : null;
 }
 
 export function getPackageUpdates(
   manager: string,
   config: PackageUpdateConfig
 ): Result<PackageUpdateResult[]> | null {
-  return managers[manager] && managers[manager].getPackageUpdates
-    ? managers[manager].getPackageUpdates(config)
-    : null;
+  if (!managers.has(manager)) {
+    return null;
+  }
+  const m = managers.get(manager);
+  return m.getPackageUpdates ? m.getPackageUpdates(config) : null;
 }
 
 export function extractPackageFile(
@@ -92,16 +80,24 @@ export function extractPackageFile(
   fileName?: string,
   config?: ExtractConfig
 ): Result<PackageFile | null> {
-  return managers[manager] && managers[manager].extractPackageFile
-    ? managers[manager].extractPackageFile(content, fileName, config)
+  if (!managers.has(manager)) {
+    return null;
+  }
+  const m = managers.get(manager);
+  return m.extractPackageFile
+    ? m.extractPackageFile(content, fileName, config)
     : null;
 }
 
 export function getRangeStrategy(config: RangeConfig): RangeStrategy {
   const { manager, rangeStrategy } = config;
-  if (managers[manager].getRangeStrategy) {
+  if (!managers.has(manager)) {
+    return null;
+  }
+  const m = managers.get(manager);
+  if (m.getRangeStrategy) {
     // Use manager's own function if it exists
-    return managers[manager].getRangeStrategy(config);
+    return m.getRangeStrategy(config);
   }
   if (rangeStrategy === 'auto') {
     // default to 'replace' for auto

lib/util/exec/index.ts

@@ -17,7 +17,7 @@ import {
   Opt,
   DockerOptions,
 } from './common';
-import { RenovateConfig } from '../../config';
+import { RenovateConfig } from '../../config/common';
 
 const execConfig: ExecConfig = {
   binarySource: null,

package.json

@@ -14,7 +14,7 @@
     "compile:ts": "tsc -p tsconfig.app.json",
     "compile:dts": "tsc -p tsconfig.dts.json",
     "generate": "run-s generate:*",
-    "generate:imports": "node --experimental-modules tools/generate-imports.mjs",
+    "generate:imports": "node tmp/tools/generate-imports.js",
     "copy-static-files": "copyfiles -u 1 -e **/__fixtures__/**  -e **/__mocks__/** lib/**/*.json lib/**/*.py dist/",
     "create-json-schema": "babel-node --extensions \".ts,.js\" -- bin/create-json-schema.js && prettier --write \"renovate-schema.json\"",
     "debug": "babel-node --inspect-brk  --extensions \".ts,.js\" -- lib/renovate.ts",

tools/generate-imports.ts

@@ -0,0 +1,68 @@
+import shell from 'shelljs';
+import fs from 'fs-extra';
+import _ from 'lodash';
+
+shell.echo('generating imports');
+const newFiles = new Set();
+
+if (!fs.existsSync('lib')) {
+  shell.echo('> missing sources');
+  shell.exit(0);
+}
+
+function findModules(dirname: string): string[] {
+  return fs
+    .readdirSync(dirname, { withFileTypes: true })
+    .filter(dirent => dirent.isDirectory())
+    .map(dirent => dirent.name)
+    .filter(name => !name.startsWith('__'))
+    .sort();
+}
+async function updateFile(file: string, code: string): Promise<void> {
+  const oldCode = fs.existsSync(file) ? await fs.readFile(file, 'utf8') : null;
+  if (code !== oldCode) {
+    await fs.writeFile(file, code);
+  }
+  newFiles.add(file);
+}
+(async () => {
+  try {
+    shell.echo('> datasources');
+    let code = `
+import { Datasource } from './common';
+const api = new Map<string, Promise<Datasource>>();
+export default api;
+`;
+    for (const ds of findModules('lib/datasource')) {
+      code += `api.set('${ds}', import('./${ds}'));\n`;
+    }
+    await updateFile('lib/datasource/api.generated.ts', code);
+
+    shell.echo('> managers');
+    let imports = '';
+    let maps = '';
+    for (const ds of findModules('lib/manager')) {
+      const name = _.camelCase(ds);
+      imports += `import * as ${name} from './${ds}';\n`;
+      maps += `api.set('${ds}', ${name});\n`;
+    }
+
+    code = `import { ManagerApi } from './common';
+${imports}
+const api = new Map<string, ManagerApi>();
+export default api;
+${maps}`;
+
+    await updateFile('lib/manager/api.generated.ts', code);
+
+    await Promise.all(
+      shell
+        .find('lib/**/*.generated.ts')
+        .filter(f => !newFiles.has(f))
+        .map(file => fs.remove(file))
+    );
+  } catch (e) {
+    shell.echo(e.toString());
+    shell.exit(1);
+  }
+})();