From 66bee75d7c20fd4d3a91f676fb2e86aa2d1f6b30 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Mon, 16 Sep 2019 11:12:22 -0400 Subject: [PATCH 1/2] ci: use circleci 2.1 --- .circleci/config.yml | 114 +++++++++++++++++++++++++---- script/release/ci-release-build.js | 78 ++++++++++++++++---- 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e7c31eadd19cc..991d672e968fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,46 @@ +version: 2.1 + +parameters: + upload-to-s3: + type: string + default: '1' + + run-lint: + type: boolean + default: true + + run-build-linux: + type: boolean + default: true + + run-build-mac: + type: boolean + default: true + + run-linux-x64-publish: + type: boolean + default: false + + run-linux-ia32-publish: + type: boolean + default: false + + run-linux-arm-publish: + type: boolean + default: false + + run-linux-arm64-publish: + type: boolean + default: false + + run-osx-publish: + type: boolean + default: false + + run-mas-publish: + type: boolean + default: false + # The config expects the following environment variables to be set: # - "SLACK_WEBHOOK" Slack hook URL to send notifications. # @@ -651,7 +694,7 @@ steps-lint: &steps-lint chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)" gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)" - cipd ensure -ensure-file - -root . <<-CIPD + cipd ensure -ensure-file - -root . \<<-CIPD \$ServiceURL https://chrome-infra-packages.appspot.com/ @Subdir src/buildtools/linux64 gn/gn/linux-amd64 $gn_version @@ -1008,7 +1051,6 @@ chromium-upgrade-branches: &chromium-upgrade-branches /chromium\-upgrade\/[0-9]+/ # List of all jobs. -version: 2 jobs: # Layer 0: Lint. Standalone. lint: @@ -1128,6 +1170,7 @@ jobs: <<: *env-linux-2xlarge-release GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True' <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish linux-ia32-debug: @@ -1178,6 +1221,7 @@ jobs: GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True' <<: *env-ia32 <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish linux-arm-debug: @@ -1229,6 +1273,7 @@ jobs: <<: *env-arm <<: *env-release-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish linux-arm64-debug: @@ -1296,6 +1341,7 @@ jobs: <<: *env-arm64 <<: *env-release-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish osx-testing: @@ -1354,6 +1400,7 @@ jobs: <<: *env-mac-large-release <<: *env-release-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish mas-testing: @@ -1418,6 +1465,7 @@ jobs: <<: *env-mas <<: *env-release-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> <<: *steps-electron-build-for-publish # Layer 3: Tests. @@ -1673,11 +1721,44 @@ jobs: workflows: version: 2 + + publish-x64-linux: + when: << pipeline.parameters.run-linux-x64-publish >> + jobs: + - linux-x64-publish + + publish-ia32-linux: + when: << pipeline.parameters.run-linux-ia32-publish >> + jobs: + - linux-ia32-publish + + publish-arm-linux: + when: << pipeline.parameters.run-linux-arm-publish >> + jobs: + - linux-arm-publish + + publish-arm64-linux: + when: << pipeline.parameters.run-linux-arm64-publish >> + jobs: + - linux-arm64-publish + + publish-osx: + when: << pipeline.parameters.run-osx-publish >> + jobs: + - osx-publish + + publish-mas: + when: << pipeline.parameters.run-mas-publish >> + jobs: + - mas-publish + lint: + when: << pipeline.parameters.run-lint >> jobs: - lint build-linux: + when: << pipeline.parameters.run-build-linux >> jobs: - linux-checkout-fast - linux-checkout-and-save-cache @@ -1744,6 +1825,7 @@ workflows: - linux-checkout-fast build-mac: + when: << pipeline.parameters.run-build-mac >> jobs: - mac-checkout-fast - mac-checkout-and-save-cache @@ -1798,11 +1880,11 @@ workflows: - master - *chromium-upgrade-branches jobs: - - linux-checkout + - linux-checkout-fast - linux-x64-release: requires: - - linux-checkout + - linux-checkout-fast - linux-x64-release-tests: requires: - linux-x64-release @@ -1814,7 +1896,7 @@ workflows: - linux-x64-release - linux-x64-chromedriver: requires: - - linux-checkout + - linux-checkout-fast - linux-x64-release-summary: requires: - linux-x64-release @@ -1824,7 +1906,7 @@ workflows: - linux-ia32-release: requires: - - linux-checkout + - linux-checkout-fast - linux-ia32-release-tests: requires: - linux-ia32-release @@ -1836,7 +1918,7 @@ workflows: - linux-ia32-release - linux-ia32-chromedriver: requires: - - linux-checkout + - linux-checkout-fast - linux-ia32-release-summary: requires: - linux-ia32-release @@ -1846,10 +1928,10 @@ workflows: - linux-arm-release: requires: - - linux-checkout + - linux-checkout-fast - linux-arm-chromedriver: requires: - - linux-checkout + - linux-checkout-fast - linux-arm-release-summary: requires: - linux-arm-release @@ -1858,10 +1940,10 @@ workflows: - linux-arm64-release: requires: - - linux-checkout + - linux-checkout-fast - linux-arm64-chromedriver: requires: - - linux-checkout + - linux-checkout-fast - linux-arm64-release-summary: requires: - linux-arm64-release @@ -1877,11 +1959,11 @@ workflows: - master - *chromium-upgrade-branches jobs: - - mac-checkout + - mac-checkout-fast - osx-release: requires: - - mac-checkout + - mac-checkout-fast - osx-release-tests: requires: - osx-release @@ -1893,7 +1975,7 @@ workflows: - osx-release - osx-chromedriver: requires: - - mac-checkout + - mac-checkout-fast - osx-release-summary: requires: - osx-release @@ -1903,7 +1985,7 @@ workflows: - mas-release: requires: - - mac-checkout + - mac-checkout-fast - mas-release-tests: requires: - mas-release @@ -1915,7 +1997,7 @@ workflows: - mas-release - mas-chromedriver: requires: - - mac-checkout + - mac-checkout-fast - mas-release-summary: requires: - mas-release diff --git a/script/release/ci-release-build.js b/script/release/ci-release-build.js index b0ffe20b06689..5305ce0cc94e7 100644 --- a/script/release/ci-release-build.js +++ b/script/release/ci-release-build.js @@ -3,6 +3,7 @@ if (!process.env.CI) require('dotenv-safe').load() const assert = require('assert') const request = require('request') const buildAppVeyorURL = 'https://ci.appveyor.com/api/builds' +const circleCIPipelineURL = 'https://circleci.com/api/v2/project/gh/electron/electron/pipeline' const vstsURL = 'https://github.visualstudio.com/electron/_apis/build' const appVeyorJobs = { @@ -55,30 +56,80 @@ async function makeRequest (requestOptions, parseResponse) { }) } -async function circleCIcall (buildUrl, targetBranch, job, options) { +async function circleCIcall (targetBranch, job, options) { console.log(`Triggering CircleCI to run build job: ${job} on branch: ${targetBranch} with release flag.`) const buildRequest = { - 'build_parameters': { - 'CIRCLE_JOB': job + 'branch': targetBranch, + 'parameters': { + 'run-lint': false, + 'run-build-linux': false, + 'run-build-mac': false } } - - if (!options.ghRelease) { - buildRequest.build_parameters.UPLOAD_TO_S3 = 1 + if (options.ghRelease) { + buildRequest.parameters['upload-to-s3'] = '0' + } else { + buildRequest.parameters['upload-to-s3'] = '1' } + buildRequest.parameters[`run-${job}`] = true jobRequestedCount++ - const circleResponse = await makeRequest({ - method: 'POST', - url: buildUrl, + try { + const circleResponse = await circleCIRequest(circleCIPipelineURL, 'POST', buildRequest) + console.log(`CircleCI release build pipeline ${circleResponse.id} for ${job} triggered.`) + const pipelineInfoUrl = `https://circleci.com/api/v2/pipeline/${circleResponse.id}` + const workflowId = await getCircleCIWorkflowId(circleResponse.id) + if (workflowId === -1) { + return + } + console.log(`CircleCI release build workflow running at https://circleci.com/workflow-run/${workflowId} for ${job}.`) + const jobInfoUrl = `https://circleci.com/api/v2/workflow/${workflowId}/jobs` + const jobInfo = await circleCIRequest(jobInfoUrl, 'GET') + if (!jobInfo.items || jobInfo.items.length !== 1) { + console.log('Error retrieving job for workflow, response was:', jobInfo) + return + } + const jobUrl = `https://circleci.com/gh/electron/electron/${jobInfo.items[0].job_number}` + console.log(`CircleCI release build request for ${job} successful. Check ${jobUrl} for status.`) + } catch (err) { + console.log('Error calling CircleCI: ', err) + } +} + +async function getCircleCIWorkflowId (pipelineId) { + const pipelineInfoUrl = `https://circleci.com/api/v2/pipeline/${pipelineId}` + for (let i = 0; i < 5; i++) { + const pipelineInfo = await circleCIRequest(pipelineInfoUrl, 'GET') + if (pipelineInfo.workflows.length !== 1) { + if (pipelineInfo.workflows.length > 1) { + console.log('Error retrieving workflows, response was:', pipelineInfo) + return -1 + } else { + // Workflows not available yet; wait 5 seconds before checking for workflows + await new Promise(resolve => setTimeout(resolve, 5000)) + } + } else { + return pipelineInfo.workflows[0].id + } + } + return -1 +} + +async function circleCIRequest (url, method, requestBody) { + return makeRequest({ + auth: { + username: process.env.CIRCLE_TOKEN, + password: '' + }, + method, + url, headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' }, - body: JSON.stringify(buildRequest) + body: requestBody ? JSON.stringify(requestBody) : null }, true).catch(err => { console.log('Error calling CircleCI:', err) }) - console.log(`CircleCI release build request for ${job} successful. Check ${circleResponse.build_url} for status.`) } function buildAppVeyor (targetBranch, options) { @@ -126,12 +177,11 @@ async function callAppVeyor (targetBranch, job, options) { } function buildCircleCI (targetBranch, options) { - const circleBuildUrl = `https://circleci.com/api/v1.1/project/github/electron/electron/tree/${targetBranch}?circle-token=${process.env.CIRCLE_TOKEN}` if (options.job) { assert(circleCIJobs.includes(options.job), `Unknown CircleCI job name: ${options.job}. Valid values are: ${circleCIJobs}.`) - circleCIcall(circleBuildUrl, targetBranch, options.job, options) + circleCIcall(targetBranch, options.job, options) } else { - circleCIJobs.forEach((job) => circleCIcall(circleBuildUrl, targetBranch, job, options)) + circleCIJobs.forEach((job) => circleCIcall(targetBranch, job, options)) } } From ebb53fe71cbdf97bf64bbb1a54c6ae2df48448b8 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Tue, 17 Sep 2019 10:08:48 -0400 Subject: [PATCH 2/2] Check for pipeline status and add comments --- .circleci/config.yml | 6 ++++++ script/release/ci-release-build.js | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 991d672e968fb..47fe4d344b7c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1722,6 +1722,12 @@ jobs: workflows: version: 2 + # The publish workflows below each contain one job so that they are + # compatible with how sudowoodo works today. If these workflows are + # changed to have multiple jobs, then scripts/release/ci-release-build.js + # will need to be updated and there will most likely need to be changes to + # sudowoodo + publish-x64-linux: when: << pipeline.parameters.run-linux-x64-publish >> jobs: diff --git a/script/release/ci-release-build.js b/script/release/ci-release-build.js index 5305ce0cc94e7..66f6b44ae6326 100644 --- a/script/release/ci-release-build.js +++ b/script/release/ci-release-build.js @@ -73,6 +73,10 @@ async function circleCIcall (targetBranch, job, options) { } buildRequest.parameters[`run-${job}`] = true jobRequestedCount++ + // The logic below expects that the CircleCI workflows for releases each + // contain only one job in order to maintain compatibility with sudowoodo. + // If the workflows are changed in the CircleCI config.yml, this logic will + // also need to be changed as well as possibly changing sudowoodo. try { const circleResponse = await circleCIRequest(circleCIPipelineURL, 'POST', buildRequest) console.log(`CircleCI release build pipeline ${circleResponse.id} for ${job} triggered.`) @@ -99,17 +103,20 @@ async function getCircleCIWorkflowId (pipelineId) { const pipelineInfoUrl = `https://circleci.com/api/v2/pipeline/${pipelineId}` for (let i = 0; i < 5; i++) { const pipelineInfo = await circleCIRequest(pipelineInfoUrl, 'GET') - if (pipelineInfo.workflows.length !== 1) { - if (pipelineInfo.workflows.length > 1) { + switch (pipelineInfo.state) { + case 'created': { + if (pipelineInfo.workflows.length === 1) { + return pipelineInfo.workflows[0].id + } + console.log('Unxpected number of workflows, response was:', pipelineInfo) + return -1 + } + case 'error': { console.log('Error retrieving workflows, response was:', pipelineInfo) return -1 - } else { - // Workflows not available yet; wait 5 seconds before checking for workflows - await new Promise(resolve => setTimeout(resolve, 5000)) } - } else { - return pipelineInfo.workflows[0].id } + await new Promise(resolve => setTimeout(resolve, 5000)) } return -1 }