Skip to content

Update CodeQL CLI dependency to 2.12.7. #48

Update CodeQL CLI dependency to 2.12.7.

Update CodeQL CLI dependency to 2.12.7. #48

name: CodeQL Standard Library Upgrade tests
# Run this workflow every time the "supported_codeql_configs.json" file is changed
on:
pull_request:
branches:
- main
- "rc/**"
- next
paths:
- "supported_codeql_configs.json"
workflow_dispatch:
jobs:
prepare-unit-test-matrix:
name: Prepare CodeQL unit test matrix
runs-on: ubuntu-22.04
outputs:
matrix: ${{ steps.export-unit-test-matrix.outputs.matrix }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Export unit test matrix
id: export-unit-test-matrix
run: |
echo "::set-output name=matrix::$(
jq --compact-output \
'.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \
supported_codeql_configs.json
)"
run-test-suites:
name: Run unit tests
needs: prepare-unit-test-matrix
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix: ${{fromJSON(needs.prepare-unit-test-matrix.outputs.matrix)}}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Setup Python 3
uses: actions/setup-python@v2
with:
python-version: "3.x"
- name: Cache CodeQL
id: cache-codeql
uses: actions/cache@v2.1.3
with:
# A list of files, directories, and wildcard patterns to cache and restore
path: ${{github.workspace}}/codeql_home
# An explicit key for restoring and saving the cache
key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}}
- name: Install CodeQL
if: steps.cache-codeql.outputs.cache-hit != 'true'
uses: ./.github/actions/install-codeql
with:
codeql-cli-version: ${{matrix.codeql_cli}}
codeql-stdlib-version: ${{matrix.codeql_standard_library}}
codeql-home: ${{ github.workspace }}/codeql_home
- name: Run test suites
id: run-test-suites
env:
RUNNER_OS: ${{runner.os}}
RUNNER_TEMP: ${{runner.temp}}
CODEQL_CLI: ${{matrix.codeql_cli}}
CODEQL_STDLIB: ${{matrix.codeql_standard_library}}
CODEQL_STDLIB_IDENT: ${{matrix.codeql_standard_library_ident}}
shell: python
run: |
import os
import sys
import subprocess
import re
import json
from pathlib import Path
def print_error(fmt, *args):
print(f"::error::{fmt}", *args)
def print_error_and_fail(fmt, *args):
print_error(fmt, args)
sys.exit(1)
def print_debug(fmt, *args):
print(f"::debug::{fmt}", *args)
def print_warning(fmt, *args):
print(f"::warning::{fmt}", *args)
def set_output(key, value):
print(f"::set-output name={key}::{value}")
cli_version = os.environ['CODEQL_CLI']
stdlib_ref = os.environ['CODEQL_STDLIB']
stdlib_ref_ident = os.environ['CODEQL_STDLIB_IDENT']
workspace = os.environ['GITHUB_WORKSPACE']
runner_os = os.environ['RUNNER_OS']
runner_temp = os.environ['RUNNER_TEMP']
codeql_home = os.path.join(workspace, 'codeql_home')
codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql')
if runner_os == "Windows":
codeql_bin += ".exe"
test_report_path = os.path.join(runner_temp, f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}.json")
os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
with open(test_report_path, 'w') as test_report_file:
codeql_home = os.path.join(workspace, 'codeql_home')
stdlib_path = os.path.join(codeql_home, 'codeql-stdlib')
cpp_test_root = Path(stdlib_path, 'cpp/ql/test')
print(f"Executing tests found (recursively) in the directory '{cpp_test_root}'")
cp = subprocess.run([codeql_bin, "test", "run", "--format=json", cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE)
if cp.returncode != 0:
print_error_and_fail(f"Failed to run tests with return code {cp.returncode} and error {cp.stderr}")
test_summary = cp.stderr.decode('utf-8')
m = re.match(r"Executing (?P<tests>\d+) tests in (?P<dirs>\d+) directories\.(?P<progress>.*?)\r?\nAll (?P<passed>\d+) tests passed\.", test_summary, re.DOTALL)
if m == None:
print_error_and_fail(f"Unable to parse test summary: {test_summary}")
tests = int(m.group('tests'))
dirs = int(m.group('dirs'))
passed = int(m.group('passed'))
print(f"Executed {tests} tests in {dirs} dirs and {passed} passed.")
test_summary_path = os.path.join(runner_temp, f"test_summary_{runner_os}_{cli_version}_{stdlib_ref_ident}.json")
with open(test_summary_path, 'w') as test_summary_file:
json.dump({
"os": runner_os,
"cli_version": cli_version,
"stdlib_ref": stdlib_ref,
"tests": tests,
"dirs": dirs,
"passed": passed
}, test_summary_file)
- name: Upload test results
uses: actions/upload-artifact@v2
with:
name: test-results-${{runner.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library_ident}}
path: |
${{runner.temp}}/test_report_${{runner.os}}_${{matrix.codeql_cli}}_${{matrix.codeql_standard_library_ident}}.json
${{runner.temp}}/test_summary_${{runner.os}}_${{matrix.codeql_cli}}_${{matrix.codeql_standard_library_ident}}.json
if-no-files-found: error
validate-test-results:
name: Validate test results
needs: [run-test-suites]
runs-on: ubuntu-22.04
steps:
- name: Install Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Collect test results
uses: actions/download-artifact@v2
- name: Validate test results
shell: python
run: |
import os
import sys
import json
from pathlib import Path
def print_error(fmt, *args):
print(f"::error::{fmt}", *args)
def print_error_and_fail(fmt, *args):
print_error(fmt, args)
sys.exit(1)
workspace = Path(os.environ['GITHUB_WORKSPACE'])
test_summary_files = workspace.glob("*/test_summary_*.json")
total_tests = 0
total_dirs = 0
total_passed = 0
for test_summary_file_path in test_summary_files:
with open(test_summary_file_path, 'r') as test_summary_file:
test_summary = json.load(test_summary_file)
print(f"os: {test_summary['os']} cli: {test_summary['cli_version']} stdlib: {test_summary['stdlib_ref']} tests: {test_summary['tests']} dirs: {test_summary['dirs']} passed: {test_summary['passed']}")
total_tests += test_summary['tests']
total_dirs += test_summary['dirs']
total_passed += test_summary['passed']
print(f"Total tests: {total_tests} total dirs: {total_dirs} total passed: {total_passed}")
if total_passed != total_tests:
print_error_and_fail(f"{total_tests - total_passed} out of ${total_tests} failed!")
print("All tests passed!")