diff --git a/.github/workflows/test262.yml b/.github/workflows/test262.yml new file mode 100644 index 00000000000..02ea6d5ebbc --- /dev/null +++ b/.github/workflows/test262.yml @@ -0,0 +1,90 @@ +name: test262 + +on: + pull_request: + branches: + - main + - releases/** + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + run_test262: + name: Run the test262 test suite + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + path: boa + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable + with: + toolchain: stable + + - name: Cache cargo + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 + with: + path: | + boa/target + ~/.cargo/git + ~/.cargo/registry + key: ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Checkout the data repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + repository: boa-dev/data + path: data + persist-credentials: false + + - name: Run the test262 test suite + run: | + cd boa + mkdir -p ../results/test262 + cargo run --release --bin boa_tester -- run -v -o ../results/test262 + cd .. + + - name: Compare results + shell: bash + run: | + cd boa + + base_results="../data/test262/refs/heads/main/latest.json" + pr_results="../results/test262/pull/latest.json" + output_dir="../results/outputs" + + test -f "$base_results" + test -f "$pr_results" + + comment="$(./target/release/boa_tester compare "$base_results" "$pr_results" -m)" + maincommit="$(jq -r '.c' "$base_results")" + + mkdir -p "$output_dir" + { + echo "" + echo "### Test262 conformance changes" + echo + echo "$comment" + echo + echo "Tested main commit: [\`${maincommit}\`](${{ github.event.pull_request.base.repo.html_url }}/commit/${maincommit})" + echo "Tested PR commit: [\`${{ github.event.pull_request.head.sha }}\`](${{ github.event.pull_request.head.repo.html_url }}/commit/${{ github.event.pull_request.head.sha }})" + echo "Compare commits: ${{ github.event.pull_request.base.repo.html_url }}/compare/${maincommit}...${{ github.event.pull_request.head.sha }}" + } > "$output_dir/comment.md" + + echo "${{ github.event.pull_request.number }}" > "$output_dir/pr_number.txt" + + - name: Upload results + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: test262-results + path: results/outputs + retention-days: 1 diff --git a/.github/workflows/test262_comment.yml b/.github/workflows/test262_comment.yml new file mode 100644 index 00000000000..ccd538234e8 --- /dev/null +++ b/.github/workflows/test262_comment.yml @@ -0,0 +1,46 @@ +name: test262_comment + +on: + workflow_run: + workflows: ["test262"] + types: + - completed + +permissions: + contents: read + pull-requests: write + +jobs: + comment: + name: Post results to PR + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Download results + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: test262-results + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + path: downloaded-results + + - name: Read results + id: results + shell: bash + run: | + echo "pr_number=$(cat downloaded-results/pr_number.txt)" >> $GITHUB_OUTPUT + + - name: Find Previous Comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4 + id: previous-comment + with: + issue-number: ${{ steps.results.outputs.pr_number }} + body-includes: "" + + - name: Update or create comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5 + with: + comment-id: ${{ steps.previous-comment.outputs.comment-id }} + issue-number: ${{ steps.results.outputs.pr_number }} + body-path: downloaded-results/comment.md + edit-mode: replace diff --git a/.github/workflows/test262_pr.yml b/.github/workflows/test262_pr.yml deleted file mode 100644 index 4fd1422481b..00000000000 --- a/.github/workflows/test262_pr.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: EcmaScript official test suite (test262) -on: - # SECURITY WARNING: Using `pull_request_target` can be risky! - # It runs workflows with full repo permissions and secrets on code from forks. - # If not handled carefully, malicious PRs can exploit this to steal secrets or manipulate the repo. - # - # Before merging any PR that changes this file, verify that the jobs don't contain - # any commands that could expose credentials or secrets. - # - # GitHub Documentation: - pull_request_target: - branches: - - main - - releases/** - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - run_test262: - name: Run the test262 test suite - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - path: boa - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable - with: - toolchain: stable - - - name: Cache cargo - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 - with: - path: | - target - ~/.cargo/git - ~/.cargo/registry - key: ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Checkout the data repo - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - repository: boa-dev/data - path: data - # Run the test suite. - - name: Run the test262 test suite - run: | - cd boa - mkdir ../results - cargo run --release --bin boa_tester -- run -v -o ../results/test262 - cd .. - - # Run the results comparison - - name: Compare results - id: compare - shell: bash - run: | - cd boa - comment="$(./target/release/boa_tester compare ../data/test262/refs/heads/main/latest.json ../results/test262/refs/heads/main/latest.json -m)" - maincommit="$(jq -r '.c' ../results/test262/refs/heads/main/latest.json)" - echo "comment<> $GITHUB_OUTPUT - echo "$comment" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - echo "Tested main commit: [\`${maincommit}\`](${{ github.event.pull_request.base.repo.html_url }}/commit/${maincommit})" >> $GITHUB_OUTPUT - echo "Tested PR commit: [\`${{ github.event.pull_request.head.sha }}\`](${{ github.event.pull_request.head.repo.html_url }}/commit/${{ github.event.pull_request.head.sha }})" >> $GITHUB_OUTPUT - echo "Compare commits: ${{ github.event.pull_request.base.repo.html_url }}/compare/${maincommit}...${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Get the PR number - id: pr-number - uses: kkak10/pr-number-action@8f5358941366822cd0825e04dfe68437f2f5f15b # v1.3 - - - name: Find Previous Comment - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4 - id: previous-comment - with: - issue-number: ${{ steps.pr-number.outputs.pr }} - body-includes: Test262 conformance changes - - - name: Update comment - if: steps.previous-comment.outputs.comment-id - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5 - continue-on-error: true - with: - comment-id: ${{ steps.previous-comment.outputs.comment-id }} - body: | - ### Test262 conformance changes - - ${{ steps.compare.outputs.comment }} - edit-mode: replace - - - name: Write a new comment - if: ${{ !steps.previous-comment.outputs.comment-id }} - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5 - continue-on-error: true - with: - issue-number: ${{ steps.pr-number.outputs.pr }} - body: | - ### Test262 conformance changes - - ${{ steps.compare.outputs.comment }}