feat(ci): add configPatterns as optional performance optimization

Author: matejchalk

packages/ci/README.md

@@ -103,11 +103,12 @@ Optionally, you can override default options for further customization:
 | `nxProjectsFilter` | `string \| string[]`      | `'--with-target={task}'`         | Arguments passed to [`nx show projects`](https://nx.dev/nx-api/nx/documents/show#projects), only relevant for Nx in [monorepo mode](#monorepo-mode) [^2] |
 | `directory`        | `string`                  | `process.cwd()`                  | Directory in which Code PushUp CLI should run                                                                                                            |
 | `config`           | `string \| null`          | `null` [^1]                      | Path to config file (`--config` option)                                                                                                                  |
-| `silent`           | `boolean`                 | `false`                          | Hides logs from CLI commands (erros will be printed)                                                                                                     |
+| `silent`           | `boolean`                 | `false`                          | Hides logs from CLI commands (errors will be printed)                                                                                                    |
 | `bin`              | `string`                  | `'npx --no-install code-pushup'` | Command for executing Code PushUp CLI                                                                                                                    |
 | `detectNewIssues`  | `boolean`                 | `true`                           | Toggles if new issues should be detected and returned in `newIssues` property                                                                            |
 | `logger`           | `Logger`                  | `console`                        | Logger for reporting progress and encountered problems                                                                                                   |
 | `skipComment`      | `boolean`                 | `false`                          | Toggles if comparison comment is posted to PR                                                                                                            |
+| `configPatterns`   | `ConfigPatterns \| null`  | `null`                           | Additional configuration which enables [faster CI runs](#faster-ci-runs-with-configpatterns)                                                             |
 
 [^1]: By default, the `code-pushup.config` file is autodetected as described in [`@code-pushup/cli` docs](../cli/README.md#configuration).
 
@@ -216,6 +217,32 @@ await runInCI(refs, api, {
 });
 ```
 
+### Faster CI runs with `configPatterns`
+
+By default, the `print-config` command is run sequentially for each project in order to reliably detect how `code-pushup` is configured - specifically, where to read output files from (`persist` config) and whether portal may be used as a cache (`upload` config). This allows for each project to be configured in its own way without breaking anything, but for large monorepos these extra `code-pushup print-config` executions can accumulate and significantly slow down CI pipelines.
+
+As a more scalable alternative, `configPatterns` may be provided. A user declares upfront how every project is configured, which allows `print-config` to be skipped. It's the user's responsibility to ensure this configuration holds for every project (it won't be checked). The `configPatterns` support string interpolation, substituting `{projectName}` with each project's name. Other than that, each project's `code-pushup.config` must have exactly the same `persist` and `upload` configurations.
+
+```ts
+await runInCI(refs, api, {
+  monorepo: true,
+  configPatterns: {
+    persist: {
+      outputDir: '.code-pushup/{projectName}',
+      filename: 'report',
+      format: ['json', 'md'],
+    },
+    // optional: will use portal as cache when comparing reports in PRs
+    upload: {
+      server: 'https://api.code-pushup.example.com/graphql',
+      apiKey: 'cp_...',
+      organization: 'example',
+      project: '{projectName}',
+    },
+  },
+});
+```
+
 ### Monorepo result
 
 In monorepo mode, the resolved object includes the merged diff at the top-level, as well as a list of projects.

packages/ci/src/lib/cli/context.unit.test.ts

@@ -24,6 +24,7 @@ describe('createCommandContext', () => {
           silent: false,
           task: 'code-pushup',
           skipComment: false,
+          configPatterns: null,
         },
         null,
       ),
@@ -52,6 +53,7 @@ describe('createCommandContext', () => {
           silent: false,
           task: 'code-pushup',
           skipComment: false,
+          configPatterns: null,
         },
         {
           name: 'ui',

packages/ci/src/lib/constants.ts

@@ -14,4 +14,5 @@ export const DEFAULT_SETTINGS: Settings = {
   logger: console,
   nxProjectsFilter: '--with-target={task}',
   skipComment: false,
+  configPatterns: null,
 };

packages/ci/src/lib/models.ts

@@ -1,4 +1,4 @@
-import type { Format } from '@code-pushup/models';
+import type { Format, PersistConfig, UploadConfig } from '@code-pushup/models';
 import type { SourceFileIssue } from './issues.js';
 import type { MonorepoTool } from './monorepo/index.js';
 
@@ -20,6 +20,7 @@ export type Options = {
   detectNewIssues?: boolean;
   logger?: Logger;
   skipComment?: boolean;
+  configPatterns?: ConfigPatterns | null;
 };
 
 /**
@@ -74,6 +75,15 @@ export type Logger = {
   debug: (message: string) => void;
 };
 
+/**
+ * Code PushUp config patterns which hold for every project in monorepo.
+ * Providing this information upfront makes CI runs faster (skips print-config).
+ */
+export type ConfigPatterns = {
+  persist: Required<PersistConfig>;
+  upload?: UploadConfig;
+};
+
 /**
  * Resolved return value of {@link runInCI}
  */

packages/ci/src/lib/run-monorepo.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines */
 import { readFile } from 'node:fs/promises';
 import {
   type ExcludeNullableProps,
@@ -32,6 +33,7 @@ import {
   type RunEnv,
   checkPrintConfig,
   compareReports,
+  configFromPatterns,
   hasDefaultPersistFormats,
   loadCachedBaseReport,
   printPersistConfig,
@@ -85,13 +87,16 @@ export async function runInMonorepoMode(
   };
 }
 
-type ProjectReport = {
+type ProjectEnv = {
   project: ProjectConfig;
-  reports: OutputFiles;
   config: EnhancedPersistConfig;
   ctx: CommandContext;
 };
 
+type ProjectReport = ProjectEnv & {
+  reports: OutputFiles;
+};
+
 function runProjectsIndividually(
   projects: ProjectConfig[],
   env: RunEnv,
@@ -117,25 +122,12 @@ async function runProjectsInBulk(
     `Running on ${projects.length} projects in bulk (parallel: ${settings.parallel})`,
   );
 
-  const currProjectConfigs = await asyncSequential(projects, async project => {
-    const ctx = createCommandContext(settings, project);
-    const config = await printPersistConfig(ctx);
-    return { project, config, ctx };
-  });
-  const hasFormats = allProjectsHaveDefaultPersistFormats(currProjectConfigs);
-  logger.debug(
-    [
-      `Loaded ${currProjectConfigs.length} persist and upload configs by running print-config command for each project.`,
-      hasFormats
-        ? 'Every project has default persist formats.'
-        : 'Not all projects have default persist formats.',
-    ].join(' '),
-  );
+  const { projectEnvs, hasFormats } = await loadProjectEnvs(projects, settings);
 
   await collectMany(runManyCommand, env, { hasFormats });
 
   const currProjectReports = await Promise.all(
-    currProjectConfigs.map(
+    projectEnvs.map(
       async ({ project, config, ctx }): Promise<ProjectReport> => {
         const reports = await saveOutputFiles({
           project,
@@ -155,6 +147,45 @@ async function runProjectsInBulk(
   return compareProjectsInBulk(currProjectReports, base, runManyCommand, env);
 }
 
+async function loadProjectEnvs(
+  projects: ProjectConfig[],
+  settings: Settings,
+): Promise<{
+  projectEnvs: ProjectEnv[];
+  hasFormats: boolean;
+}> {
+  const { logger, configPatterns } = settings;
+
+  const projectEnvs: ProjectEnv[] = configPatterns
+    ? projects.map(
+        (project): ProjectEnv => ({
+          project,
+          config: configFromPatterns(configPatterns, project),
+          ctx: createCommandContext(settings, project),
+        }),
+      )
+    : await asyncSequential(projects, async (project): Promise<ProjectEnv> => {
+        const ctx = createCommandContext(settings, project);
+        const config = await printPersistConfig(ctx);
+        return { project, config, ctx };
+      });
+
+  const hasFormats = allProjectsHaveDefaultPersistFormats(projectEnvs);
+
+  logger.debug(
+    [
+      configPatterns
+        ? `Parsed ${projectEnvs.length} persist and upload configs by interpolating configPatterns option for each project.`
+        : `Loaded ${projectEnvs.length} persist and upload configs by running print-config command for each project.`,
+      hasFormats
+        ? 'Every project has default persist formats.'
+        : 'Not all projects have default persist formats.',
+    ].join(' '),
+  );
+
+  return { projectEnvs, hasFormats };
+}
+
 async function compareProjectsInBulk(
   currProjectReports: ProjectReport[],
   base: GitBranch,
@@ -228,27 +259,29 @@ async function collectPreviousReports(
   env: RunEnv,
 ): Promise<Record<string, ReportData<'previous'>>> {
   const { settings } = env;
-  const { logger } = settings;
+  const { logger, configPatterns } = settings;
 
   if (uncachedProjectReports.length === 0) {
     return {};
   }
 
   return runInBaseBranch(base, env, async () => {
-    const uncachedProjectConfigs = await asyncSequential(
-      uncachedProjectReports,
-      async args => ({
-        name: args.project.name,
-        ctx: args.ctx,
-        config: await checkPrintConfig(args),
-      }),
-    );
+    const uncachedProjectConfigs = configPatterns
+      ? uncachedProjectReports.map(({ project, ctx }) => {
+          const config = configFromPatterns(configPatterns, project);
+          return { project, ctx, config };
+        })
+      : await asyncSequential(uncachedProjectReports, async args => ({
+          project: args.project,
+          ctx: args.ctx,
+          config: await checkPrintConfig(args),
+        }));
 
     const validProjectConfigs =
       uncachedProjectConfigs.filter(hasNoNullableProps);
-    const onlyProjects = validProjectConfigs.map(({ name }) => name);
-    const invalidProjects = uncachedProjectConfigs
-      .map(({ name }) => name)
+    const onlyProjects = validProjectConfigs.map(({ project }) => project.name);
+    const invalidProjects: string[] = uncachedProjectConfigs
+      .map(({ project }) => project.name)
       .filter(name => !onlyProjects.includes(name));
     if (invalidProjects.length > 0) {
       logger.debug(
@@ -277,19 +310,19 @@ async function collectPreviousReports(
 }
 
 async function savePreviousProjectReport(args: {
-  name: string;
+  project: ProjectConfig;
   ctx: CommandContext;
   config: EnhancedPersistConfig;
   settings: Settings;
 }): Promise<[string, ReportData<'previous'>]> {
-  const { name, ctx, config, settings } = args;
+  const { project, ctx, config, settings } = args;
   const files = await saveReportFiles({
-    project: { name },
+    project,
     type: 'previous',
     files: persistedFilesFromConfig(config, ctx),
     settings,
   });
-  return [name, files];
+  return [project.name, files];
 }
 
 async function collectMany(

packages/ci/src/lib/run-utils.ts

@@ -7,6 +7,7 @@ import {
   type ReportsDiff,
 } from '@code-pushup/models';
 import {
+  interpolate,
   removeUndefinedAndEmptyProps,
   stringifyError,
 } from '@code-pushup/utils';
@@ -24,6 +25,7 @@ import { DEFAULT_SETTINGS } from './constants.js';
 import { listChangedFiles, normalizeGitRef } from './git.js';
 import { type SourceFileIssue, filterRelevantIssues } from './issues.js';
 import type {
+  ConfigPatterns,
   GitBranch,
   GitRefs,
   Options,
@@ -76,9 +78,8 @@ export async function createRunEnv(
   options: Options | undefined,
   git: SimpleGit,
 ): Promise<RunEnv> {
-  const inferredVerbose: boolean = Boolean(
-    options?.debug === true || options?.silent === false,
-  );
+  const inferredVerbose: boolean =
+    options?.debug === true || options?.silent === false;
   // eslint-disable-next-line functional/immutable-data
   process.env['CP_VERBOSE'] = `${inferredVerbose}`;
 
@@ -114,9 +115,13 @@ export async function runOnProject(
     logger.info(`Running Code PushUp on monorepo project ${project.name}`);
   }
 
-  const config = await printPersistConfig(ctx);
+  const config = settings.configPatterns
+    ? configFromPatterns(settings.configPatterns, project)
+    : await printPersistConfig(ctx);
   logger.debug(
-    `Loaded persist and upload configs from print-config command - ${JSON.stringify(config)}`,
+    settings.configPatterns
+      ? `Parsed persist and upload configs from configPatterns option - ${JSON.stringify(config)}`
+      : `Loaded persist and upload configs from print-config command - ${JSON.stringify(config)}`,
   );
 
   await runCollect(ctx, { hasFormats: hasDefaultPersistFormats(config) });
@@ -216,15 +221,17 @@ export async function collectPreviousReport(
 ): Promise<ReportData<'previous'> | null> {
   const { ctx, env, base, project } = args;
   const { settings } = env;
-  const { logger } = settings;
+  const { logger, configPatterns } = settings;
 
   const cachedBaseReport = await loadCachedBaseReport(args);
   if (cachedBaseReport) {
     return cachedBaseReport;
   }
 
   return runInBaseBranch(base, env, async () => {
-    const config = await checkPrintConfig(args);
+    const config = configPatterns
+      ? configFromPatterns(configPatterns, project)
+      : await checkPrintConfig(args);
     if (!config) {
       return null;
     }
@@ -399,6 +406,32 @@ export function hasDefaultPersistFormats(
   );
 }
 
+export function configFromPatterns(
+  configPatterns: ConfigPatterns,
+  project: ProjectConfig | null,
+): ConfigPatterns {
+  const { persist, upload } = configPatterns;
+  const variables = {
+    projectName: project?.name ?? '',
+  };
+  return {
+    persist: {
+      outputDir: interpolate(persist.outputDir, variables),
+      filename: interpolate(persist.filename, variables),
+      format: persist.format,
+    },
+    ...(upload && {
+      upload: {
+        server: upload.server,
+        apiKey: upload.apiKey,
+        organization: interpolate(upload.organization, variables),
+        project: interpolate(upload.project, variables),
+        ...(upload.timeout != null && { timeout: upload.timeout }),
+      },
+    }),
+  };
+}
+
 export async function findNewIssues(
   args: CompareReportsArgs & { diffFiles: OutputFiles },
 ): Promise<SourceFileIssue[]> {