diff --git a/azure-pipelines/dotnet-vscode-csharp-insertion.yml b/azure-pipelines/dotnet-vscode-csharp-insertion.yml new file mode 100644 index 0000000000..533176b6c5 --- /dev/null +++ b/azure-pipelines/dotnet-vscode-csharp-insertion.yml @@ -0,0 +1,98 @@ +trigger: none +pr: none + +parameters: + - name: targetBranch + displayName: Target Branch for PR + type: string + default: main + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: RoslynMirror + type: git + name: internal/dotnet-roslyn + pipelines: + - pipeline: officialBuildCI + source: dotnet-roslyn-official + project: internal + branch: main + trigger: none + +variables: + - name: RoslynEndSHA + value: $(resources.pipeline.officialBuildCI.sourceCommit) + - name: RoslynBuildNumber + value: $(resources.pipeline.officialBuildCI.runName) + - name: RoslynBuildId + value: $(resources.pipeline.officialBuildCI.runID) + - template: dotnet-variables.yml + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: netcore1espool-internal + image: 1es-ubuntu-2204 + os: linux + stages: + - stage: BumpRoslyn + displayName: Bump Roslyn Version + jobs: + - job: ProcessBump + displayName: Process Roslyn Bump + pool: + name: netcore1espool-internal + image: 1es-ubuntu-2204 + os: linux + steps: + - checkout: self + persistCredentials: true + + - task: UseDotNet@2 + displayName: Install .NET SDK + inputs: + version: $(defaultDotnetVersion) + + - template: install-node.yml + + - task: DownloadPipelineArtifact@2 + displayName: Download Asset Manifests + inputs: + source: specific + project: internal + pipeline: dotnet-roslyn-official + runVersion: specific + runId: $(RoslynBuildId) + artifact: AssetManifests + path: $(Pipeline.Workspace)/AssetManifests + + - pwsh: | + npm ci + npm install + npm install -g gulp + gulp installDependencies + displayName: 'Install npm dependencies and gulp' + - checkout: RoslynMirror + displayName: 'Checkout Roslyn repository' + path: roslyn + + - script: | + echo "Installing roslyn-tools..." + dotnet tool install -g Microsoft.RoslynTools --prerelease --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json + echo "Verifying roslyn-tools installation..." + dotnet tool list -g + roslyn-tools --version + displayName: 'Install roslyn-tools CLI' + + - task: Npm@1 + displayName: Run Roslyn insertion + inputs: + command: custom + customCommand: 'run gulp -- insertion:roslyn --assetManifestPath=$(Pipeline.Workspace)/AssetManifests --roslynRepoPath=$(Pipeline.Workspace)/roslyn --roslynEndSHA=$(RoslynEndSHA) --roslynBuildNumber=$(RoslynBuildNumber) --roslynBuildId=$(RoslynBuildId) --targetBranch=${{ parameters.targetBranch }} --githubPAT=$(BotAccount-dotnet-bot-repo-PAT) --dryRun=false' + + diff --git a/gulpfile.ts b/gulpfile.ts index dedaf17128..8e29d1ea70 100644 --- a/gulpfile.ts +++ b/gulpfile.ts @@ -12,4 +12,5 @@ require('./tasks/debuggerTasks'); require('./tasks/snapTasks'); require('./tasks/signingTasks'); require('./tasks/profilingTasks'); +require('./tasks/insertionTasks'); require('./tasks/componentUpdateTasks'); diff --git a/tasks/createTagsTasks.ts b/tasks/createTagsTasks.ts index a5589e9d74..606c524900 100644 --- a/tasks/createTagsTasks.ts +++ b/tasks/createTagsTasks.ts @@ -4,12 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import * as gulp from 'gulp'; -import * as fs from 'fs'; import minimist from 'minimist'; import { Octokit } from '@octokit/rest'; -import { allNugetPackages, NugetPackageInfo, platformSpecificPackages } from './offlinePackagingTasks'; -import { PlatformInformation } from '../src/shared/platform'; -import path from 'path'; +import { allNugetPackages } from './offlinePackagingTasks'; +import { getCommitFromNugetAsync } from './gitTasks'; interface CreateTagsOptions { releaseVersion: string; @@ -194,53 +192,3 @@ function logWarning(message: string): void { function logError(message: string): void { console.log(`##vso[task.logissue type=error]${message}`); } - -async function getCommitFromNugetAsync(packageInfo: NugetPackageInfo): Promise { - const packageJsonString = fs.readFileSync('./package.json').toString(); - const packageJson = JSON.parse(packageJsonString); - const packageVersion = packageJson['defaults'][packageInfo.packageJsonName]; - if (!packageVersion) { - logError(`Can't find ${packageInfo.packageJsonName} version in package.json`); - return null; - } - - const platform = await PlatformInformation.GetCurrent(); - const vsixPlatformInfo = platformSpecificPackages.find( - (p) => p.platformInfo.platform === platform.platform && p.platformInfo.architecture === platform.architecture - )!; - - const packageName = packageInfo.getPackageName(vsixPlatformInfo); - console.log(`${packageName} version is ${packageVersion}`); - - // Nuget package should exist under out/.nuget/ since we have run the install dependencies task. - // Package names are always lower case in the .nuget folder. - const packageDir = path.join('out', '.nuget', packageName.toLowerCase(), packageVersion); - const nuspecFiles = fs.readdirSync(packageDir).filter((file) => file.endsWith('.nuspec')); - - if (nuspecFiles.length === 0) { - logError(`No .nuspec file found in ${packageDir}`); - return null; - } - - if (nuspecFiles.length > 1) { - logError(`Multiple .nuspec files found in ${packageDir}`); - return null; - } - - const nuspecFilePath = path.join(packageDir, nuspecFiles[0]); - const nuspecFile = fs.readFileSync(nuspecFilePath).toString(); - const results = /commit="(.*)"/.exec(nuspecFile); - if (results == null || results.length == 0) { - logError('Failed to find commit number from nuspec file'); - return null; - } - - if (results.length != 2) { - logError('Unexpected regex match result from nuspec file.'); - return null; - } - - const commitNumber = results[1]; - console.log(`commitNumber is ${commitNumber}`); - return commitNumber; -} diff --git a/tasks/gitTasks.ts b/tasks/gitTasks.ts index 9230729e2e..781455df99 100644 --- a/tasks/gitTasks.ts +++ b/tasks/gitTasks.ts @@ -4,7 +4,29 @@ *--------------------------------------------------------------------------------------------*/ import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; import { Octokit } from '@octokit/rest'; +import { NugetPackageInfo, platformSpecificPackages } from './offlinePackagingTasks'; +import { PlatformInformation } from '../src/shared/platform'; + +export interface GitOptions { + commitSha: string; + targetRemoteRepo: string; + baseBranch: string; +} + +export interface BranchAndPROptions extends GitOptions { + githubPAT: string; + dryRun: boolean; + newBranchName: string; + userName?: string; + email?: string; +} + +export function logError(message: string): void { + console.log(`##vso[task.logissue type=error]${message}`); +} /** * Execute a git command with optional logging @@ -55,7 +77,7 @@ export async function createCommit(branch: string, files: string[], commitMessag * Check if a branch exists on the remote repository */ export async function doesBranchExist(remoteAlias: string, branch: string): Promise { - const lsRemote = await git(['ls-remote', remoteAlias, 'refs/head/' + branch]); + const lsRemote = await git(['ls-remote', remoteAlias, 'refs/heads/' + branch]); return lsRemote.trim() !== ''; } @@ -146,3 +168,113 @@ export async function createPullRequest( return null; } } + +export async function getCommitFromNugetAsync(packageInfo: NugetPackageInfo): Promise { + try { + const packageJsonString = fs.readFileSync('./package.json').toString(); + const packageJson = JSON.parse(packageJsonString); + const packageVersion = packageJson['defaults'][packageInfo.packageJsonName]; + if (!packageVersion) { + logError(`Can't find ${packageInfo.packageJsonName} version in package.json`); + return null; + } + + const platform = await PlatformInformation.GetCurrent(); + const vsixPlatformInfo = platformSpecificPackages.find( + (p) => p.platformInfo.platform === platform.platform && p.platformInfo.architecture === platform.architecture + )!; + + const packageName = packageInfo.getPackageName(vsixPlatformInfo); + console.log(`${packageName} version is ${packageVersion}`); + + // Nuget package should exist under out/.nuget/ since we have run the install dependencies task. + // Package names are always lower case in the .nuget folder. + const packageDir = path.join('out', '.nuget', packageName.toLowerCase(), packageVersion); + const nuspecFiles = fs.readdirSync(packageDir).filter((file) => file.endsWith('.nuspec')); + + if (nuspecFiles.length === 0) { + logError(`No .nuspec file found in ${packageDir}`); + return null; + } + + if (nuspecFiles.length > 1) { + logError(`Multiple .nuspec files found in ${packageDir}`); + return null; + } + + const nuspecFilePath = path.join(packageDir, nuspecFiles[0]); + const nuspecFile = fs.readFileSync(nuspecFilePath).toString(); + const results = /commit="(.*?)"/.exec(nuspecFile); + if (results == null || results.length === 0) { + logError('Failed to find commit number from nuspec file'); + return null; + } + + if (results.length !== 2) { + logError('Unexpected regex match result from nuspec file.'); + return null; + } + + const commitNumber = results[1]; + console.log(`commitNumber is ${commitNumber}`); + return commitNumber; + } catch (error) { + logError(`Error getting commit from NuGet package: ${error}`); + if (error instanceof Error && error.stack) { + console.log(`##[debug]${error.stack}`); + } + throw error; + } +} + +export async function createBranchAndPR( + options: BranchAndPROptions, + title: string, + commitMessage: string, + body?: string +): Promise { + const { githubPAT, targetRemoteRepo, baseBranch, dryRun, userName, email, newBranchName } = options; + + // Configure git user credentials + await configureGitUser(userName, email); + + // Create branch and commit changes + await createCommit(newBranchName, ['.'], commitMessage); + + if (dryRun !== true) { + // Push branch to remote + await pushBranch(newBranchName, githubPAT, 'dotnet', targetRemoteRepo); + } else { + console.log('[DRY RUN] Would have pushed branch to remote'); + } + + // Check for existing PR and create new one if needed + const existingPRUrl = await findPRByTitle(githubPAT, 'dotnet', targetRemoteRepo, title); + if (existingPRUrl) { + console.log('Pull request with the same name already exists. Skip creation.'); + return null; + } + + if (dryRun !== true) { + const prUrl = await createPullRequest( + githubPAT, + 'dotnet', + targetRemoteRepo, + newBranchName, + title, + body || title, + baseBranch + ); + + if (prUrl) { + console.log(`Created pull request: ${prUrl}.`); + // Extract PR number from URL (format: https://github.com/owner/repo/pull/123) + const prNumberMatch = prUrl.match(/\/pull\/(\d+)$/); + return prNumberMatch ? parseInt(prNumberMatch[1], 10) : null; + } + return null; + } else { + console.log(`[DRY RUN] Would have created PR with title: "${title}" and body: "${body || title}"`); + return null; + } +} diff --git a/tasks/insertionTasks.ts b/tasks/insertionTasks.ts new file mode 100644 index 0000000000..fdb110ec74 --- /dev/null +++ b/tasks/insertionTasks.ts @@ -0,0 +1,264 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as gulp from 'gulp'; +import * as fs from 'fs'; +import * as path from 'path'; +import minimist from 'minimist'; +import { exec } from 'child_process'; +import { promisify } from 'util'; +import * as xml2js from 'xml2js'; +import { allNugetPackages} from './offlinePackagingTasks'; +import {getCommitFromNugetAsync, createBranchAndPR, git } from './gitTasks'; +import * as os from 'os'; + +const execAsync = promisify(exec); + +interface InsertionOptions { + roslynVersion?: string; + roslynEndSHA?: string; + roslynBuildId?: string; + roslynBuildNumber?: string; + assetManifestPath?: string; + roslynRepoPath?: string; + targetBranch?: string; + githubPAT?: string; + dryRun: boolean; +} + +function logWarning(message: string, error?: unknown): void { + console.log(`##vso[task.logissue type=warning]${message}`); + if (error instanceof Error && error.stack) { + console.log(`##[debug]${error.stack}`); + } +} + +function logError(message: string, error?: unknown): void { + console.log(`##vso[task.logissue type=error]${message}`); + if (error instanceof Error && error.stack) { + console.log(`##[debug]${error.stack}`); + } +} + +gulp.task('insertion:roslyn', async (): Promise => { + const options = minimist(process.argv.slice(2), { + boolean: ['dryRun'], + default: { + dryRun: false, + }, + }); + + console.log('Starting Roslyn insertion process...'); + + try { + // Step 1: Extract Roslyn version from AssetManifest + if (!options.assetManifestPath) { + throw new Error('assetManifestPath is required'); + } + + const newVersion = await extractRoslynVersionFromManifest(options.assetManifestPath); + if (!newVersion) { + throw new Error('Failed to extract Roslyn version from asset manifest'); + } + options.roslynVersion = newVersion; + console.log(`New Roslyn version: ${newVersion}`); + + // Step 2: Get current SHA from package + const currentSHA = await getCommitFromNugetAsync(allNugetPackages.roslyn); + if (!currentSHA) { + throw new Error('Could not determine current Roslyn SHA from package'); + } + console.log(`Current Roslyn SHA: ${currentSHA}`); + + // Step 3: Check if update needed + if (!options.roslynEndSHA) { + throw new Error('roslynEndSHA is required'); + } + + if (currentSHA === options.roslynEndSHA) { + console.log('No new commits to process - versions are identical'); + return; + } + + console.log(`Update needed: ${currentSHA}..${options.roslynEndSHA}`); + + // Step 4: Verify Roslyn repo exists + if (!options.roslynRepoPath) { + throw new Error('roslynRepoPath is required'); + } + await verifyRoslynRepo(options.roslynRepoPath); + + // Step 5: Generate PR list + const prList = await generatePRList(currentSHA, options.roslynEndSHA, options.roslynRepoPath, options); + console.log('PR List generated:', prList); + + // Check if PR list is null or empty (generation failed or no matching PRs) + if (!prList) { + console.log('No PRs with required labels found or PR list generation failed. Skipping insertion.'); + logWarning('No PRs with VSCode label found between the commits or PR list generation failed. Skipping insertion.'); + return; + } + + // Step 6: Update files + await updatePackageJson(options.roslynVersion); + + // Step 7: Create branch and PR + const prTitle = `Bump Roslyn to ${options.roslynVersion} (${options.roslynEndSHA?.substring(0, 8)})`; + + // Include build information in the PR description + let prBody = `This PR updates Roslyn to version ${options.roslynVersion} (${options.roslynEndSHA}).\n\n`; + + // Add build link if build information is available + if (options.roslynBuildNumber && options.roslynBuildId) { + prBody += `Build: [#${options.roslynBuildNumber}](https://dev.azure.com/dnceng/internal/_build/results?buildId=${options.roslynBuildId})\n\n`; + } + + // Add PR list + prBody += prList; + + const commitMessage = `Bump Roslyn to ${options.roslynVersion} (${options.roslynEndSHA?.substring(0, 8)})`; + + // Create the PR + const prNumber = await createBranchAndPR({ + ...options, + commitSha: options.roslynEndSHA!, + targetRemoteRepo: 'vscode-csharp', + baseBranch: options.targetBranch || 'main', + newBranchName: `insertion/${options.roslynEndSHA}`, + githubPAT: options.githubPAT!, + dryRun: options.dryRun, + userName: options.userName, + email: options.email + }, prTitle, commitMessage, prBody); + + // If PR was created and we're not in dry run mode, update the changelog with the PR number + if (prNumber && !options.dryRun) { + console.log(`PR #${prNumber} created. Updating changelog with PR link...`); + + // Update changelog with PR number (single call) + await updateChangelog(options.roslynVersion, prList, prNumber); + + // Create a second commit to include the updated changelog and push to update the PR + await git(['add', 'CHANGELOG.md']); + await git(['commit', '-m', `Update changelog with PR #${prNumber}`]); + await git(['push', 'target', `insertion/${options.roslynEndSHA}`]); + + console.log(`Changelog updated with PR #${prNumber} link.`); + } + + } catch (error) { + logError(`Insertion failed: ${error instanceof Error ? error.message : String(error)}`, error); + throw error; + } +}); + +async function extractRoslynVersionFromManifest(manifestPath: string): Promise { + const xmlFile = path.join(manifestPath, 'OfficialBuild.xml'); + + if (!fs.existsSync(xmlFile)) { + logError(`OfficialBuild.xml not found at ${xmlFile}`); + return null; + } + + const xmlContent = fs.readFileSync(xmlFile, 'utf8'); + const parser = new xml2js.Parser(); + const result = await parser.parseStringPromise(xmlContent); + + const packages = result?.Build?.Package || []; + for (const pkg of packages) { + const attrs = pkg.$; + if (attrs?.Id === 'Microsoft.CodeAnalysis.Common') { + return attrs.Version; + } + } + + logError('Microsoft.CodeAnalysis.Common package not found in the asset manifest.'); + return null; +} + +async function verifyRoslynRepo(roslynRepoPath: string): Promise { + if (!fs.existsSync(roslynRepoPath)) { + throw new Error(`Roslyn repository not found at ${roslynRepoPath}`); + } + console.log(`Using Roslyn repository at ${roslynRepoPath}`); +} + +async function generatePRList(startSHA: string, endSHA: string, roslynRepoPath: string, _options: InsertionOptions): Promise { + console.log(`Generating PR list from ${startSHA} to ${endSHA}...`); + + try { + const { stdout } = await execAsync( + `cd "${roslynRepoPath}" && roslyn-tools pr-finder -s "${startSHA}" -e "${endSHA}" --format changelog --label VSCode`, + { maxBuffer: 10 * 1024 * 1024 } // 10MB buffer + ); + return stdout && stdout.trim() ? stdout : null; + } catch (error) { + logWarning(`PR finder failed: ${error instanceof Error ? error.message : String(error)}`, error); + return null; + } +} + +async function updatePackageJson(newVersion: string): Promise { + console.log(`Updating package.json with Roslyn version ${newVersion}...`); + const packageJsonPath = 'package.json'; + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')); + + if (!packageJson.defaults) { + throw new Error('Could not find defaults section in package.json'); + } + packageJson.defaults.roslyn = newVersion; + + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2) + '\n'); +} + +async function updateChangelog(version: string, prList: string | null, prNumber?: number): Promise { + console.log('Updating CHANGELOG.md...'); + const changelogPath = 'CHANGELOG.md'; + const text = fs.readFileSync(changelogPath, 'utf8'); + const NL = os.EOL; + + // Find the first top-level header "# ..." + const topHeaderRegex = /^(# .*?)(\r?\n|$)/m; + const headerMatch = topHeaderRegex.exec(text); + if (!headerMatch) { + throw new Error('CHANGELOG.md must contain at least one top-level header (#)'); + } + + const headerEndLineIndex = headerMatch.index + headerMatch[0].length; + + // Prepare new Roslyn block + let newRoslynBlock = `* Bump Roslyn to ${version}`; + + // Add PR number if available + if (prNumber) { + newRoslynBlock = `* Bump Roslyn to ${version} (PR: [#${prNumber}](https://github.com/dotnet/vscode-csharp/pull/${prNumber}))`; + } + + // Add PR list as sub-items if available + if (prList) { + const prLines = prList + .split(/\r?\n/) + .filter((l) => l.trim() && !l.includes('View Complete Diff')) + .map((line) => line.trim()); + + const formattedPRList = prLines.length > 0 + ? prLines.map((line) => ` ${line}`).join(NL) + : ''; + + if (formattedPRList) { + newRoslynBlock += NL + formattedPRList; + } + } + + // Insert the new block right after the header + const newText = + text.slice(0, headerEndLineIndex) + + newRoslynBlock + + (text.length > headerEndLineIndex ? NL + text.slice(headerEndLineIndex) : ''); + + // Write the updated content back to the file + fs.writeFileSync(changelogPath, newText, 'utf8'); + console.log('CHANGELOG.md updated successfully'); +} diff --git a/tasks/localizationTasks.ts b/tasks/localizationTasks.ts index bdb3d13c67..e3da516156 100644 --- a/tasks/localizationTasks.ts +++ b/tasks/localizationTasks.ts @@ -10,7 +10,7 @@ import { spawnSync } from 'node:child_process'; import * as path from 'path'; import * as util from 'node:util'; import { EOL } from 'node:os'; -import { Octokit } from '@octokit/rest'; +import { createBranchAndPR } from './gitTasks'; type Options = { userName?: string; @@ -76,70 +76,29 @@ gulp.task('publish localization content', async () => { } console.log(`Changed files going to be staged: ${diff}`); - const newBranchName = `localization/${parsedArgs.commitSha}`; - // Make this optional so it can be tested locally by using dev's information. In real CI user name and email are always supplied. - if (parsedArgs.userName) { - await git(['config', '--local', 'user.name', parsedArgs.userName]); - } - if (parsedArgs.email) { - await git(['config', '--local', 'user.email', parsedArgs.email]); - } - - await git(['checkout', '-b', newBranchName]); - await git(['commit', '-m', `Localization result of ${parsedArgs.commitSha}.`]); - + const title = `Localization result based on ${parsedArgs.commitSha}`; + const commitMessage = `Localization result of ${parsedArgs.commitSha}`; const pat = process.env['GitHubPAT']; if (!pat) { throw 'No GitHub Pat found.'; } - - const remoteRepoAlias = 'targetRepo'; - await git( - [ - 'remote', - 'add', - remoteRepoAlias, - `https://${parsedArgs.userName}:${pat}@github.com/dotnet/${parsedArgs.targetRemoteRepo}.git`, - ], - // Note: don't print PAT to console - false - ); - await git(['fetch', remoteRepoAlias]); - - const lsRemote = await git(['ls-remote', remoteRepoAlias, 'refs/head/' + newBranchName]); - if (lsRemote.trim() !== '') { - // If the localization branch of this commit already exists, don't try to create another one. - console.log( - `##vso[task.logissue type=error]${newBranchName} already exists in ${parsedArgs.targetRemoteRepo}. Skip pushing.` + try { + await createBranchAndPR( + { + commitSha: parsedArgs.commitSha, + targetRemoteRepo: parsedArgs.targetRemoteRepo, + baseBranch: parsedArgs.baseBranch, + githubPAT: process.env['GitHubPAT'] || '', + dryRun: false, + newBranchName: `localization/${parsedArgs.commitSha}`, + userName: parsedArgs.userName, + email: parsedArgs.email + }, + title, + commitMessage ); - } else { - await git(['push', '-u', remoteRepoAlias]); - } - - const octokit = new Octokit({ auth: pat }); - const listPullRequest = await octokit.rest.pulls.list({ - owner: 'dotnet', - repo: parsedArgs.targetRemoteRepo, - }); - - if (listPullRequest.status != 200) { - throw `Failed get response from GitHub, http status code: ${listPullRequest.status}`; - } - - const title = `Localization result based on ${parsedArgs.commitSha}`; - if (listPullRequest.data.some((pr) => pr.title === title)) { - console.log('Pull request with the same name already exists. Skip creation.'); - return; + } catch (error) { + console.error('Error creating branch and PR:', error); + throw error; } - - const pullRequest = await octokit.rest.pulls.create({ - body: title, - owner: 'dotnet', - repo: parsedArgs.targetRemoteRepo, - title: title, - head: newBranchName, - base: parsedArgs.baseBranch, - }); - - console.log(`Created pull request: ${pullRequest.data.html_url}.`); });