From 005303efef2672523e5a2279ffe2502b14ebc811 Mon Sep 17 00:00:00 2001 From: David First Date: Wed, 15 Apr 2026 10:24:45 -0400 Subject: [PATCH 1/5] fix(ci): retry ci pr export on lane hash mismatch from concurrent pushes --- scopes/git/ci/ci.main.runtime.ts | 45 +++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/scopes/git/ci/ci.main.runtime.ts b/scopes/git/ci/ci.main.runtime.ts index 0a7b9ebb9699..dbce8cff9091 100644 --- a/scopes/git/ci/ci.main.runtime.ts +++ b/scopes/git/ci/ci.main.runtime.ts @@ -385,6 +385,7 @@ export class CiMain { this.logger.console(chalk.blue(`Creating temporary lane ${laneId.scope}/${tempLaneName}`)); let foundErr: Error | undefined; + let renamedToFinalName = false; try { await this.lanes.createLane(tempLaneName, { scope: laneId.scope, @@ -452,10 +453,12 @@ export class CiMain { // Rename temp lane to original name this.logger.console(chalk.blue(`Renaming lane from ${tempLaneName} to ${laneId.name}`)); await this.lanes.rename(laneId.name, tempLaneName); + renamedToFinalName = true; - // Export with the correct name + // Export with the correct name. Retry on hash-mismatch, which indicates a concurrent CI job + // pushed the same lane id between our pre-export delete and our merge on the hub. this.logger.console(chalk.blue(`Exporting ${snappedComponents.length} components`)); - const exportResults = await this.exporter.export(); + const exportResults = await this.exportWithRetryOnLaneHashMismatch(laneId.toString()); this.logger.console(chalk.green(`Exported ${exportResults.componentsIds.length} components`)); } catch (e: any) { foundErr = e; @@ -477,8 +480,10 @@ export class CiMain { await this.lanes.checkout.checkout({ head: true, skipNpmInstall: true }); } - // Clean up orphaned temporary lane on error - if (foundErr) { + // Clean up orphaned temporary lane on error. Skip if the rename to the final name already + // happened - in that case the temp name no longer exists locally, and the lane under the + // final name may have been partially exported; leave it alone rather than wipe evidence. + if (foundErr && !renamedToFinalName) { const tempLaneFullName = `${laneId.scope}/${tempLaneName}`; this.logger.console(chalk.blue(`Cleaning up temporary lane ${tempLaneFullName}`)); try { @@ -828,6 +833,38 @@ export class CiMain { } } + /** + * Run `exporter.export()` and retry when the remote rejects the push because a lane with the same + * id already exists with a different hash. This race happens when a concurrent `bit ci pr` run + * (same PR, different workflow or re-run) pushed the same lane id between our pre-export + * existence check and our merge on the hub. Between "Exporting" and the hub's merge check there + * can be ~1-2 minutes of upload/processing - plenty of time for another run to persist a new lane. + * + * On retry we delete the remote lane (now populated by the winning concurrent push) and try + * the export again with our local hash. Bounded attempts guard against a tight push loop between + * two runners. + */ + private async exportWithRetryOnLaneHashMismatch(laneIdStr: string, maxAttempts = 3) { + const isHashMismatchErr = (err: any) => + (err?.message || err?.toString() || '').includes('a lane with the same id already exists with a different hash'); + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + return await this.exporter.export(); + } catch (e: any) { + if (!isHashMismatchErr(e) || attempt === maxAttempts) throw e; + this.logger.console( + chalk.yellow( + `Export attempt ${attempt}/${maxAttempts} failed with lane hash mismatch on "${laneIdStr}" (likely a concurrent CI push). Deleting remote lane and retrying.` + ) + ); + await this.archiveLane(laneIdStr, true); + } + } + // Unreachable: the loop either returns on success or throws on the last attempt. + throw new Error(`exportWithRetryOnLaneHashMismatch: exhausted ${maxAttempts} attempts for lane ${laneIdStr}`); + } + /** * Archives (deletes) a lane with proper error handling and logging. * @param throwOnError - if true, throws on failure (use for critical operations like pre-export cleanup) From 6e5161bcb653ef514bddb246d2998935addf8231 Mon Sep 17 00:00:00 2001 From: David First Date: Wed, 15 Apr 2026 10:43:09 -0400 Subject: [PATCH 2/5] fix(ci): skip hash-mismatch retry when PR branch has advanced (stale run) --- scopes/git/ci/ci.main.runtime.ts | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scopes/git/ci/ci.main.runtime.ts b/scopes/git/ci/ci.main.runtime.ts index dbce8cff9091..2e2f12e010cb 100644 --- a/scopes/git/ci/ci.main.runtime.ts +++ b/scopes/git/ci/ci.main.runtime.ts @@ -843,6 +843,11 @@ export class CiMain { * On retry we delete the remote lane (now populated by the winning concurrent push) and try * the export again with our local hash. Bounded attempts guard against a tight push loop between * two runners. + * + * Before each retry we verify we're not a stale run: if the PR branch has advanced past our + * commit on the remote, a newer CI run is on the way and clobbering its lane with our older + * snaps would regress the PR preview. In that case we surface the original error and let the + * newer run publish the correct lane. */ private async exportWithRetryOnLaneHashMismatch(laneIdStr: string, maxAttempts = 3) { const isHashMismatchErr = (err: any) => @@ -853,6 +858,14 @@ export class CiMain { return await this.exporter.export(); } catch (e: any) { if (!isHashMismatchErr(e) || attempt === maxAttempts) throw e; + if (await this.isStaleCiRun()) { + this.logger.console( + chalk.yellow( + `Export failed with lane hash mismatch on "${laneIdStr}" and the PR branch has advanced past our commit. Not retrying - a newer CI run will publish the correct lane.` + ) + ); + throw e; + } this.logger.console( chalk.yellow( `Export attempt ${attempt}/${maxAttempts} failed with lane hash mismatch on "${laneIdStr}" (likely a concurrent CI push). Deleting remote lane and retrying.` @@ -865,6 +878,28 @@ export class CiMain { throw new Error(`exportWithRetryOnLaneHashMismatch: exhausted ${maxAttempts} attempts for lane ${laneIdStr}`); } + /** + * Returns true when the PR branch on the remote has advanced past our local HEAD, meaning a + * newer commit was pushed to the branch while this CI run was in flight. Best-effort: when we + * can't determine the branch or reach the remote we return false (don't block retry). + */ + private async isStaleCiRun(): Promise { + try { + const branch = await this.getBranchName(); + if (!branch) return false; + const localSha = (await git.revparse(['HEAD'])).trim(); + await git.fetch('origin', branch); + const remoteSha = (await git.revparse([`origin/${branch}`])).trim(); + if (!remoteSha || !localSha || remoteSha === localSha) return false; + // If remote has a commit we don't, we're stale. + const mergeBase = (await git.raw(['merge-base', localSha, remoteSha])).trim(); + return mergeBase === localSha && mergeBase !== remoteSha; + } catch (err: any) { + this.logger.console(chalk.yellow(`Unable to verify CI run freshness (assuming fresh): ${err?.message || err}`)); + return false; + } + } + /** * Archives (deletes) a lane with proper error handling and logging. * @param throwOnError - if true, throws on failure (use for critical operations like pre-export cleanup) From 1e8b77092a98d6394b91c55d34242863749cf927 Mon Sep 17 00:00:00 2001 From: David First Date: Wed, 15 Apr 2026 11:29:09 -0400 Subject: [PATCH 3/5] fix(ci): harden hash-mismatch retry against injection and error masking --- scopes/git/ci/ci.main.runtime.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scopes/git/ci/ci.main.runtime.ts b/scopes/git/ci/ci.main.runtime.ts index 2e2f12e010cb..93da63104fb2 100644 --- a/scopes/git/ci/ci.main.runtime.ts +++ b/scopes/git/ci/ci.main.runtime.ts @@ -871,7 +871,21 @@ export class CiMain { `Export attempt ${attempt}/${maxAttempts} failed with lane hash mismatch on "${laneIdStr}" (likely a concurrent CI push). Deleting remote lane and retrying.` ) ); - await this.archiveLane(laneIdStr, true); + try { + await this.archiveLane(laneIdStr, true); + } catch (archiveErr: any) { + // Preserve the original export error - rethrowing the archive error would hide the real + // reason the push was rejected. + this.logger.console( + chalk.yellow( + `Failed to delete remote lane "${laneIdStr}" while recovering from hash mismatch: ${archiveErr?.message || archiveErr}. Rethrowing the original export error.` + ) + ); + if (e && typeof e === 'object' && !('cause' in e)) { + (e as any).cause = archiveErr; + } + throw e; + } } } // Unreachable: the loop either returns on success or throws on the last attempt. @@ -888,8 +902,10 @@ export class CiMain { const branch = await this.getBranchName(); if (!branch) return false; const localSha = (await git.revparse(['HEAD'])).trim(); - await git.fetch('origin', branch); - const remoteSha = (await git.revparse([`origin/${branch}`])).trim(); + // Fetch with fully-qualified ref and `--` separator so a branch name starting with `-` + // cannot be interpreted as a git option (defense in depth for untrusted PR branches). + await git.raw(['fetch', 'origin', '--', `refs/heads/${branch}`]); + const remoteSha = (await git.revparse([`refs/remotes/origin/${branch}`])).trim(); if (!remoteSha || !localSha || remoteSha === localSha) return false; // If remote has a commit we don't, we're stale. const mergeBase = (await git.raw(['merge-base', localSha, remoteSha])).trim(); From c2369a4b5224872303fb8b8b60fd2d3cf45b9672 Mon Sep 17 00:00:00 2001 From: David First Date: Wed, 15 Apr 2026 11:36:15 -0400 Subject: [PATCH 4/5] refactor(ci): simplify hash-mismatch retry helpers --- scopes/git/ci/ci.main.runtime.ts | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/scopes/git/ci/ci.main.runtime.ts b/scopes/git/ci/ci.main.runtime.ts index 93da63104fb2..0da6cf1ce277 100644 --- a/scopes/git/ci/ci.main.runtime.ts +++ b/scopes/git/ci/ci.main.runtime.ts @@ -834,20 +834,11 @@ export class CiMain { } /** - * Run `exporter.export()` and retry when the remote rejects the push because a lane with the same - * id already exists with a different hash. This race happens when a concurrent `bit ci pr` run - * (same PR, different workflow or re-run) pushed the same lane id between our pre-export - * existence check and our merge on the hub. Between "Exporting" and the hub's merge check there - * can be ~1-2 minutes of upload/processing - plenty of time for another run to persist a new lane. - * - * On retry we delete the remote lane (now populated by the winning concurrent push) and try - * the export again with our local hash. Bounded attempts guard against a tight push loop between - * two runners. - * - * Before each retry we verify we're not a stale run: if the PR branch has advanced past our - * commit on the remote, a newer CI run is on the way and clobbering its lane with our older - * snaps would regress the PR preview. In that case we surface the original error and let the - * newer run publish the correct lane. + * Export with retry on lane hash-mismatch, caused by a concurrent `bit ci pr` run pushing the + * same lane id between our pre-export delete and the hub's merge (the export takes 1-2 minutes, + * plenty of time to race). Before each retry we skip if the PR branch has advanced past our + * commit - in that case a newer run will publish the correct lane, and retrying with our older + * snaps would regress the PR preview. */ private async exportWithRetryOnLaneHashMismatch(laneIdStr: string, maxAttempts = 3) { const isHashMismatchErr = (err: any) => @@ -888,7 +879,6 @@ export class CiMain { } } } - // Unreachable: the loop either returns on success or throws on the last attempt. throw new Error(`exportWithRetryOnLaneHashMismatch: exhausted ${maxAttempts} attempts for lane ${laneIdStr}`); } @@ -902,14 +892,14 @@ export class CiMain { const branch = await this.getBranchName(); if (!branch) return false; const localSha = (await git.revparse(['HEAD'])).trim(); - // Fetch with fully-qualified ref and `--` separator so a branch name starting with `-` - // cannot be interpreted as a git option (defense in depth for untrusted PR branches). + // `--` separator and fully-qualified ref so a branch name starting with `-` can't be + // interpreted as a git option (defense in depth for untrusted PR branches). await git.raw(['fetch', 'origin', '--', `refs/heads/${branch}`]); const remoteSha = (await git.revparse([`refs/remotes/origin/${branch}`])).trim(); - if (!remoteSha || !localSha || remoteSha === localSha) return false; - // If remote has a commit we don't, we're stale. + if (remoteSha === localSha) return false; const mergeBase = (await git.raw(['merge-base', localSha, remoteSha])).trim(); - return mergeBase === localSha && mergeBase !== remoteSha; + // local is strictly behind remote - remote has commits we don't. + return mergeBase === localSha; } catch (err: any) { this.logger.console(chalk.yellow(`Unable to verify CI run freshness (assuming fresh): ${err?.message || err}`)); return false; From 74e665b2b3b29cf887927df56fb2ddfef899d04a Mon Sep 17 00:00:00 2001 From: David First Date: Wed, 15 Apr 2026 11:40:19 -0400 Subject: [PATCH 5/5] refactor(ci): harden retry - explicit fetch refspec, nullish cause, marker constant --- scopes/git/ci/ci.main.runtime.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scopes/git/ci/ci.main.runtime.ts b/scopes/git/ci/ci.main.runtime.ts index 0da6cf1ce277..83fd17af9c81 100644 --- a/scopes/git/ci/ci.main.runtime.ts +++ b/scopes/git/ci/ci.main.runtime.ts @@ -27,6 +27,14 @@ import type { Version, LaneComponent } from '@teambit/objects'; import { SourceBranchDetector } from './source-branch-detector'; import { generateRandomStr } from '@teambit/toolbox.string.random'; +/** + * Sentinel substring emitted by the hub when a lane push is rejected because a lane with the same + * id already exists with a different hash. Thrown as `BitError` from + * `components/legacy/scope/repositories/sources.ts` and wrapped in `UnexpectedNetworkError` across + * the wire, so we match on the message text rather than an error class. + */ +const LANE_HASH_MISMATCH_MARKER = 'a lane with the same id already exists with a different hash'; + export interface CiWorkspaceConfig { /** * Path to a custom script that generates commit messages for `bit ci merge` operations. @@ -841,8 +849,7 @@ export class CiMain { * snaps would regress the PR preview. */ private async exportWithRetryOnLaneHashMismatch(laneIdStr: string, maxAttempts = 3) { - const isHashMismatchErr = (err: any) => - (err?.message || err?.toString() || '').includes('a lane with the same id already exists with a different hash'); + const isHashMismatchErr = (err: any) => (err?.message || err?.toString() || '').includes(LANE_HASH_MISMATCH_MARKER); for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { @@ -872,7 +879,7 @@ export class CiMain { `Failed to delete remote lane "${laneIdStr}" while recovering from hash mismatch: ${archiveErr?.message || archiveErr}. Rethrowing the original export error.` ) ); - if (e && typeof e === 'object' && !('cause' in e)) { + if (e && typeof e === 'object' && (e as any).cause == null) { (e as any).cause = archiveErr; } throw e; @@ -894,7 +901,7 @@ export class CiMain { const localSha = (await git.revparse(['HEAD'])).trim(); // `--` separator and fully-qualified ref so a branch name starting with `-` can't be // interpreted as a git option (defense in depth for untrusted PR branches). - await git.raw(['fetch', 'origin', '--', `refs/heads/${branch}`]); + await git.raw(['fetch', 'origin', '--', `refs/heads/${branch}:refs/remotes/origin/${branch}`]); const remoteSha = (await git.revparse([`refs/remotes/origin/${branch}`])).trim(); if (remoteSha === localSha) return false; const mergeBase = (await git.raw(['merge-base', localSha, remoteSha])).trim();