fix(backend): Limit concurrent git operations to prevent resource exhaustion

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed "The account is already associated with another user" errors with GitLab oauth provider. [#584](https://github.com/sourcebot-dev/sourcebot/pull/584)
 - Fixed error when viewing a generic git connection in `/settings/connections`. [#588](https://github.com/sourcebot-dev/sourcebot/pull/588)
 - Fixed issue with an unbounded `Promise.allSettled(...)` when retrieving details from the GitHub API about a large number of repositories (or orgs or users). [#591](https://github.com/sourcebot-dev/sourcebot/pull/591)
+- Fixed resource exhaustion (EAGAIN errors) when syncing generic-git-host connections with thousands of repositories. [#593](https://github.com/sourcebot-dev/sourcebot/pull/593)
 
 ## Removed
 - Removed built-in secret manager. [#592](https://github.com/sourcebot-dev/sourcebot/pull/592)

packages/backend/src/repoCompileUtils.ts

@@ -20,11 +20,17 @@ import assert from 'assert';
 import GitUrlParse from 'git-url-parse';
 import { RepoMetadata } from '@sourcebot/shared';
 import { SINGLE_TENANT_ORG_ID } from './constants.js';
+import pLimit from 'p-limit';
 
 export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
 
 const logger = createLogger('repo-compile-utils');
 
+// Limit concurrent git operations to prevent resource exhaustion (EAGAIN errors)
+// when processing thousands of repositories simultaneously
+const MAX_CONCURRENT_GIT_OPERATIONS = 100;
+const gitOperationLimit = pLimit(MAX_CONCURRENT_GIT_OPERATIONS);
+
 type CompileResult = {
     repoData: RepoData[],
     warnings: string[],
@@ -472,7 +478,7 @@ export const compileGenericGitHostConfig_file = async (
     const repos: RepoData[] = [];
     const warnings: string[] = [];
 
-    await Promise.all(repoPaths.map(async (repoPath) => {
+    await Promise.all(repoPaths.map((repoPath) => gitOperationLimit(async () => {
         const isGitRepo = await isPathAValidGitRepoRoot({
             path: repoPath,
         });
@@ -526,7 +532,7 @@ export const compileGenericGitHostConfig_file = async (
         }
 
         repos.push(repo);
-    }));
+    })));
 
     return {
         repoData: repos,