Fix failures in the cron ci. (RustPython#5002)

Author: DimitrisJim

.github/workflows/cron-ci.yaml

@@ -6,50 +6,33 @@ on:
 name: Periodic checks/tasks
 
 env:
-  CARGO_ARGS: --features ssl,jit
+  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
 
 jobs:
+  # codecov collects code coverage data from the rust tests, python snippets and python test suite.
+  # This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
   codecov:
     name: Collect code coverage data
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: llvm-tools-preview
-      - run: sudo apt-get update && sudo apt-get -y install lcov
-      - run: cargo build --release --verbose ${{ env.CARGO_ARGS }}
-        env:
-          RUSTC_WRAPPER: './scripts/codecoverage-rustc-wrapper.sh'
+      - uses: taiki-e/install-action@cargo-llvm-cov
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
-      - run: python -m pip install pytest
-        working-directory: ./extra_tests
-      - name: run snippets
-        run: LLVM_PROFILE_FILE="$PWD/snippet-%p.profraw" pytest -v
-        working-directory: ./extra_tests
+      - run: sudo apt-get update && sudo apt-get -y install lcov
+      - name: Run cargo-llvm-cov with Rust tests.
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+      - name: Run cargo-llvm-cov with Python snippets.
+        run: python scripts/cargo-llvm-cov.py
         continue-on-error: true
-      - name: run cpython tests
-        run: |
-          alltests=($(target/release/rustpython -c 'from test.libregrtest.runtest import findtests; print(*findtests())'))
-          i=0
-          # chunk into chunks of 10 tests each. idk at this point
-          while subtests=("${alltests[@]:$i:10}"); [[ ${#subtests[@]} -ne 0 ]]; do
-            LLVM_PROFILE_FILE="$PWD/regrtest-%p.profraw" target/release/rustpython -m test -v "${subtests[@]}" || true
-            ((i+=10))
-          done
+      - name: Run cargo-llvm-cov with Python test suite.
+        run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed 
         continue-on-error: true
-      - name: prepare code coverage data
-        run: |
-          rusttool() {
-            local tool=$1; shift; "$(rustc --print target-libdir)/../bin/llvm-$tool" "$@"
-          }
-          rusttool profdata merge extra_tests/snippet-*.profraw regrtest-*.profraw --output codecov.profdata
-          rusttool cov export --instr-profile codecov.profdata target/release/rustpython --format lcov > codecov_tmp.lcov
-          lcov -e codecov_tmp.lcov "$PWD"/'*' -o codecov_tmp2.lcov
-          lcov -r codecov_tmp2.lcov "$PWD"/target/'*' -o codecov.lcov  # remove LALRPOP-generated parser
-      - name: upload to Codecov
+      - name: Prepare code coverage data
+        run: cargo llvm-cov report --lcov --output-path='codecov.cov'
+      - name: Upload to Codecov
         uses: codecov/codecov-action@v3
         with:
           file: ./codecov.lcov
@@ -89,6 +72,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
       - name: build rustpython
         run: cargo build --release --verbose
       - name: Collect what is left data

benches/microbenchmarks.rs

@@ -9,6 +9,25 @@ use std::{
     path::{Path, PathBuf},
 };
 
+// List of microbenchmarks to skip.
+//
+// These result in excessive memory usage, some more so than others. For example, while
+// exception_context.py consumes a lot of memory, it still finishes. On the other hand,
+// call_kwargs.py seems like it performs an excessive amount of allocations and results in
+// a system freeze.
+// In addition, the fact that we don't yet have a GC means that benchmarks which might consume
+// a bearable amount of memory accumulate. As such, best to skip them for now.
+const SKIP_MICROBENCHMARKS: [&str; 8] = [
+    "call_simple.py",
+    "call_kwargs.py",
+    "construct_object.py",
+    "define_function.py",
+    "define_class.py",
+    "exception_nested.py",
+    "exception_simple.py",
+    "exception_context.py",
+];
+
 pub struct MicroBenchmark {
     name: String,
     setup: String,
@@ -211,6 +230,9 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         .collect();
 
     for benchmark in benchmarks {
+        if SKIP_MICROBENCHMARKS.contains(&benchmark.name.as_str()) {
+            continue;
+        }
         run_micro_benchmark(c, benchmark);
     }
 }

scripts/cargo-llvm-cov.py

@@ -0,0 +1,20 @@
+import os
+import subprocess
+
+TARGET = "extra_tests/snippets"
+
+def run_llvm_cov(file_path: str):
+    """ Run cargo llvm-cov on a file. """
+    if file_path.endswith(".py"):
+        command = ["cargo", "llvm-cov", "--no-report", "run", "--", file_path]
+        subprocess.call(command)
+
+def iterate_files(folder: str):
+    """ Iterate over all files in a folder. """
+    for root, _, files in os.walk(folder):
+        for file in files:
+            file_path = os.path.join(root, file)
+            run_llvm_cov(file_path)
+
+if __name__ == "__main__":
+    iterate_files(TARGET)