From 725d6e74fb0f5683e2a9d2cc8099d49070431c3d Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 16 Oct 2025 18:04:35 +0200 Subject: [PATCH 1/2] (DO NOT MERGE) ci: only run instrumentation integration tests --- .github/workflows/ci.yml | 115 +++------------------------------------ 1 file changed, 8 insertions(+), 107 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e174912..fd6ddb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,59 +7,16 @@ on: workflow_dispatch: jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.1 - with: - extra_args: --all-files - - tests: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - - - name: Cache build - uses: actions/cache@v3 - with: - path: core/build-tests - key: ${{ runner.os }}-build-tests-${{ hashFiles('**/CMakeLists.txt') }} - - - name: Create build directory - run: mkdir -p core/build-tests - - - name: Build tests - run: | - cd core/build-tests - cmake .. -DENABLE_TESTS=ON - make -j - - - name: Run tests - run: | - cd core/build-tests - GTEST_OUTPUT=json:test-results/ ctest - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test_results - path: ${{runner.workspace}}/core/build-tests/test/test-results/**/*.json - cmake-integration-tests: strategy: matrix: include: - codspeed-mode: "instrumentation" runner: "ubuntu-latest" - - codspeed-mode: "walltime" - runner: "codspeed-macro" - - codspeed-mode: "off" - runner: "ubuntu-latest" + # - codspeed-mode: "walltime" + # runner: "codspeed-macro" + # - codspeed-mode: "off" + # runner: "ubuntu-latest" runs-on: ${{ matrix.runner }} steps: - name: Checkout code @@ -96,10 +53,10 @@ jobs: include: - codspeed-mode: "instrumentation" runner: "ubuntu-latest" - - codspeed-mode: "walltime" - runner: "codspeed-macro" - - codspeed-mode: "off" - runner: "ubuntu-latest" + # - codspeed-mode: "walltime" + # runner: "codspeed-macro" + # - codspeed-mode: "off" + # runner: "ubuntu-latest" runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -127,59 +84,3 @@ jobs: mode: ${{ matrix.codspeed-mode }} run: bazel run //examples/google_benchmark_bazel:my_benchmark --//core:codspeed_mode=${{ matrix.codspeed-mode }} token: ${{ secrets.CODSPEED_TOKEN }} - - windows-cmake-build: - strategy: - matrix: - codspeed-mode: ["off", "walltime"] - runs-on: windows-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: "recursive" - - - name: Cache build - uses: actions/cache@v3 - with: - path: examples/google_benchmark_cmake/build - key: ${{ runner.os }}-build-${{ matrix.codspeed-mode }}-${{ hashFiles('**/CMakeLists.txt') }} - - - name: Create build directory - run: | - if (-not (Test-Path examples\google_benchmark_cmake\build)) { - mkdir examples\google_benchmark_cmake\build - } - shell: pwsh - - - name: Build benchmark example - run: | - cd examples\google_benchmark_cmake\build - cmake -DCODSPEED_MODE=${{ matrix.codspeed-mode }} .. - cmake --build . --config Release - shell: pwsh - - windows-bazel-build: - strategy: - matrix: - codspeed-mode: ["off", "walltime"] - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - name: Set up Bazel - uses: bazel-contrib/setup-bazel@0.14.0 - with: - # Avoid downloading Bazel every time. - bazelisk-cache: true - # Store build cache per workflow. - disk-cache: ${{ github.workflow }} - # Share repository cache between workflows. - repository-cache: true - - - name: Build benchmark example - run: | - bazel build //examples/google_benchmark_bazel:my_benchmark --//core:codspeed_mode=${{ matrix.codspeed-mode }} - shell: pwsh From 86ad31dd16c27fec9dce77b0ef19f05186cc0ae3 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 16 Oct 2025 18:35:50 +0200 Subject: [PATCH 2/2] --wip-- [skip ci] --- compare_workflow_durations.py | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 compare_workflow_durations.py diff --git a/compare_workflow_durations.py b/compare_workflow_durations.py new file mode 100755 index 0000000..909ef71 --- /dev/null +++ b/compare_workflow_durations.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +Compare workflow durations between two branches. +""" + +import json +import subprocess +import statistics +from datetime import datetime +from typing import List, Dict + + +def get_workflow_runs(branch: str, limit: int = 20) -> List[Dict]: + """Fetch workflow runs for a specific branch.""" + cmd = [ + "gh", "run", "list", + "--workflow", "ci.yml", + "--branch", branch, + "--limit", str(limit), + "--json", "startedAt,updatedAt,status,conclusion,number" + ] + + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + runs = json.loads(result.stdout) + + # Filter only completed runs and calculate duration + completed_runs = [] + for run in runs: + if run['status'] == 'completed' and run['startedAt'] and run['updatedAt']: + started = datetime.fromisoformat(run['startedAt'].replace('Z', '+00:00')) + updated = datetime.fromisoformat(run['updatedAt'].replace('Z', '+00:00')) + duration_ms = (updated - started).total_seconds() * 1000 + run['durationMs'] = duration_ms + completed_runs.append(run) + + return completed_runs + + +def calculate_stats(durations: List[float]) -> Dict[str, float]: + """Calculate statistics for durations.""" + if not durations: + return {"count": 0, "avg_seconds": 0, "median_seconds": 0, "avg_minutes": 0, "median_minutes": 0} + + avg_ms = statistics.mean(durations) + median_ms = statistics.median(durations) + + return { + "count": len(durations), + "avg_seconds": avg_ms / 1000, + "median_seconds": median_ms / 1000, + "avg_minutes": avg_ms / 60000, + "median_minutes": median_ms / 60000, + } + + +def main(): + branches = ["test-action-no-cache", "test-action-cache-effects"] + + print("Fetching workflow runs...\n") + + results = {} + for branch in branches: + print(f"Fetching runs for branch: {branch}") + runs = get_workflow_runs(branch, limit=20) + durations = [run['durationMs'] for run in runs] + stats = calculate_stats(durations) + results[branch] = stats + + print(f" Found {len(runs)} completed runs") + + # Display results + print("\n" + "=" * 80) + print("WORKFLOW DURATION COMPARISON") + print("=" * 80 + "\n") + + for branch in branches: + stats = results[branch] + print(f"Branch: {branch}") + print(f" Completed runs: {stats['count']}") + print(f" Average: {stats['avg_minutes']:.2f} minutes ({stats['avg_seconds']:.1f} seconds)") + print(f" Median: {stats['median_minutes']:.2f} minutes ({stats['median_seconds']:.1f} seconds)") + print() + + # Calculate difference + if results[branches[0]]["count"] > 0 and results[branches[1]]["count"] > 0: + print("=" * 80) + print("COMPARISON") + print("=" * 80 + "\n") + + cache_stats = results["test-action-cache-effects"] + no_cache_stats = results["test-action-no-cache"] + + avg_diff = no_cache_stats["avg_seconds"] - cache_stats["avg_seconds"] + median_diff = no_cache_stats["median_seconds"] - cache_stats["median_seconds"] + + avg_pct = (avg_diff / no_cache_stats["avg_seconds"]) * 100 if no_cache_stats["avg_seconds"] > 0 else 0 + median_pct = (median_diff / no_cache_stats["median_seconds"]) * 100 if no_cache_stats["median_seconds"] > 0 else 0 + + print(f"Average difference: {abs(avg_diff):.1f} seconds ({abs(avg_pct):.1f}%)") + if avg_diff > 0: + print(f" → Cache makes workflow FASTER") + else: + print(f" → Cache makes workflow SLOWER") + + print(f"\nMedian difference: {abs(median_diff):.1f} seconds ({abs(median_pct):.1f}%)") + if median_diff > 0: + print(f" → Cache makes workflow FASTER") + else: + print(f" → Cache makes workflow SLOWER") + + +if __name__ == "__main__": + main()