From dcadd8b241abd5f1d98e17695497835b0e0e4104 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 07:43:38 +0000 Subject: [PATCH 1/7] Move CLI version tests into a separate workflow --- .github/workflows/cli-test.yml | 103 +++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 51 ++++------------ 2 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/cli-test.yml diff --git a/.github/workflows/cli-test.yml b/.github/workflows/cli-test.yml new file mode 100644 index 00000000000..ff1f244751d --- /dev/null +++ b/.github/workflows/cli-test.yml @@ -0,0 +1,103 @@ +name: Run CLI tests +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + find-nightly: + name: Find Nightly Release + runs-on: ubuntu-latest + outputs: + url: ${{ steps.get-url.outputs.nightly-url }} + steps: + - name: Get Nightly Release URL + id: get-url + env: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + shell: bash + # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests. + run: | + LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'` + echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT" + + set-matrix: + name: Set Matrix for cli-test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set the variables + id: set-variables + run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT + outputs: + cli-versions: ${{ steps.set-variables.outputs.cli-versions }} + + cli-test: + name: CLI Test + runs-on: ${{ matrix.os }} + needs: [find-nightly, set-matrix] + timeout-minutes: 30 + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }} + fail-fast: false + env: + CLI_VERSION: ${{ matrix.version }} + NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }} + TEST_CODEQL_PATH: '${{ github.workspace }}/codeql' + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: '16.17.1' + cache: 'npm' + cache-dependency-path: extensions/ql-vscode/package-lock.json + + - name: Install dependencies + working-directory: extensions/ql-vscode + run: | + npm ci + shell: bash + + - name: Build + working-directory: extensions/ql-vscode + run: | + npm run build + shell: bash + + - name: Decide on ref of CodeQL repo + id: choose-ref + shell: bash + run: | + if [[ "${{ matrix.version }}" == "nightly" ]] + then + REF="codeql-cli/latest" + else + REF="codeql-cli/${{ matrix.version }}" + fi + echo "ref=$REF" >> "$GITHUB_OUTPUT" + + - name: Checkout QL + uses: actions/checkout@v3 + with: + repository: github/codeql + ref: ${{ steps.choose-ref.outputs.ref }} + path: codeql + + - name: Run CLI tests (Linux) + working-directory: extensions/ql-vscode + if: matrix.os == 'ubuntu-latest' + run: | + unset DBUS_SESSION_BUS_ADDRESS + /usr/bin/xvfb-run npm run test:cli-integration + + - name: Run CLI tests (Windows) + working-directory: extensions/ql-vscode + if: matrix.os == 'windows-latest' + run: | + npm run test:cli-integration diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a61d97e2577..b4ddc60c4d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,22 +53,6 @@ jobs: name: vscode-codeql-extension path: artifacts - find-nightly: - name: Find Nightly Release - runs-on: ubuntu-latest - outputs: - url: ${{ steps.get-url.outputs.nightly-url }} - steps: - - name: Get Nightly Release URL - id: get-url - env: - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' - shell: bash - # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests. - run: | - LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'` - echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT" - lint: name: Lint runs-on: ubuntu-latest @@ -198,31 +182,32 @@ jobs: run: | npm run test:vscode-integration - set-matrix: - name: Set Matrix for cli-test + get-latest-cli-version: + name: Get latest CLI version runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - - name: Set the variables - id: set-variables - run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT + - name: Set the variable + id: set-variable + run: | + echo "cli-version=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc '.[0]')" >> $GITHUB_OUTPUT + echo "$cli-version" outputs: - cli-versions: ${{ steps.set-variables.outputs.cli-versions }} + cli-version: ${{ steps.set-variable.outputs.cli-version }} cli-test: name: CLI Test runs-on: ${{ matrix.os }} - needs: [find-nightly, set-matrix] + needs: [get-latest-cli-version] timeout-minutes: 30 strategy: matrix: os: [ubuntu-latest, windows-latest] - version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }} + version: ['${{ needs.get-latest-cli-version.outputs.cli-version }}'] fail-fast: false env: - CLI_VERSION: ${{ matrix.version }} - NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }} + CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }} TEST_CODEQL_PATH: '${{ github.workspace }}/codeql' steps: @@ -247,23 +232,11 @@ jobs: npm run build shell: bash - - name: Decide on ref of CodeQL repo - id: choose-ref - shell: bash - run: | - if [[ "${{ matrix.version }}" == "nightly" ]] - then - REF="codeql-cli/latest" - else - REF="codeql-cli/${{ matrix.version }}" - fi - echo "ref=$REF" >> "$GITHUB_OUTPUT" - - name: Checkout QL uses: actions/checkout@v3 with: repository: github/codeql - ref: ${{ steps.choose-ref.outputs.ref }} + ref: 'codeql-cli/${{ needs.get-latest-cli-version.outputs.cli-version }}' path: codeql - name: Run CLI tests (Linux) From c3182764e33869667f1488b65d11666f31475c9f Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 07:46:03 +0000 Subject: [PATCH 2/7] Add on-push to test new workflow --- .github/workflows/cli-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cli-test.yml b/.github/workflows/cli-test.yml index ff1f244751d..be74d7a2440 100644 --- a/.github/workflows/cli-test.yml +++ b/.github/workflows/cli-test.yml @@ -3,6 +3,9 @@ on: workflow_dispatch: schedule: - cron: '0 0 * * *' + push: + branches: + - charisk/split-out-cli-tests jobs: find-nightly: From 4aa1f84f2688a6ff3b8697c354d89b8f0536c650 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 08:49:00 +0000 Subject: [PATCH 3/7] Revert "Add on-push to test new workflow" This reverts commit c3182764e33869667f1488b65d11666f31475c9f. --- .github/workflows/cli-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/cli-test.yml b/.github/workflows/cli-test.yml index be74d7a2440..ff1f244751d 100644 --- a/.github/workflows/cli-test.yml +++ b/.github/workflows/cli-test.yml @@ -3,9 +3,6 @@ on: workflow_dispatch: schedule: - cron: '0 0 * * *' - push: - branches: - - charisk/split-out-cli-tests jobs: find-nightly: From 3af268366763bb53e608a825210b652471e46ac5 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 08:55:43 +0000 Subject: [PATCH 4/7] Update releasing instructions --- docs/releasing.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/releasing.md b/docs/releasing.md index 86706f72094..d300799a45f 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,5 +1,6 @@ # Releasing (write access required) +1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step. 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top. * Go through all recent PRs and make sure they are properly accounted for. * Make sure all changelog entries have links back to their PR(s) if appropriate. From d5492c391d9453745147a96d6d5cf3143aa4e0eb Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 09:47:46 +0000 Subject: [PATCH 5/7] Don't include the CLI version in the matrix --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b4ddc60c4d8..1b3acbbac29 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -204,7 +204,6 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - version: ['${{ needs.get-latest-cli-version.outputs.cli-version }}'] fail-fast: false env: CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }} From 24eedb0bcbfd72370f9945bfd39e7924980364af Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 21 Aug 2023 15:26:56 +0000 Subject: [PATCH 6/7] Automatically run CLI tests if certain files change --- .github/workflows/cli-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cli-test.yml b/.github/workflows/cli-test.yml index ff1f244751d..715a16a79df 100644 --- a/.github/workflows/cli-test.yml +++ b/.github/workflows/cli-test.yml @@ -3,6 +3,11 @@ on: workflow_dispatch: schedule: - cron: '0 0 * * *' + push: + paths: + - extensions/ql-vscode/src/codeql-cli/** + - extensions/ql-vscode/src/language-support/** + - extensions/ql-vscode/src/query-server/** jobs: find-nightly: From 5c09913b61c075ee9639699620bda54cf9d22a09 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Tue, 22 Aug 2023 07:40:47 +0000 Subject: [PATCH 7/7] Switch to use pull request trigger rather than push --- .github/workflows/cli-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-test.yml b/.github/workflows/cli-test.yml index 715a16a79df..42fe20759c4 100644 --- a/.github/workflows/cli-test.yml +++ b/.github/workflows/cli-test.yml @@ -3,7 +3,8 @@ on: workflow_dispatch: schedule: - cron: '0 0 * * *' - push: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths: - extensions/ql-vscode/src/codeql-cli/** - extensions/ql-vscode/src/language-support/**