Start parsing the chunks file with serde

.github/workflows/bench.yml

@@ -16,7 +16,8 @@ jobs:
         with:
           lfs: true
 
-      - run: rustup toolchain install nightly --profile minimal --no-self-update
+      - run: rustup toolchain install stable --profile minimal --no-self-update
+      - uses: Swatinem/rust-cache@v2
       - uses: cargo-bins/cargo-binstall@main
       - run: cargo binstall cargo-codspeed
 

.github/workflows/ci.yml

@@ -20,7 +20,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - run: rustup toolchain install nightly --profile minimal --component rustfmt --component clippy --no-self-update
+      - run: rustup toolchain install stable --profile minimal --component rustfmt --component clippy --no-self-update
+      - uses: Swatinem/rust-cache@v2
 
       - run: cargo fmt --all -- --check
       - run: cargo clippy --all-features --workspace --tests --examples -- -D clippy::all
@@ -46,7 +47,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - run: rustup toolchain install nightly --profile minimal --no-self-update
+      - run: rustup toolchain install stable --profile minimal --no-self-update
+      - uses: Swatinem/rust-cache@v2
 
       - run: cargo test --workspace --all-features --doc
       - run: cargo doc --workspace --all-features --document-private-items --no-deps
@@ -56,15 +58,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          lfs: true
 
-      - run: rustup toolchain install nightly --profile minimal --no-self-update
+      - run: rustup toolchain install stable --profile minimal --no-self-update
+      - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@cargo-llvm-cov
       - uses: taiki-e/install-action@nextest
 
-      # FIXME(swatinem): We should pass `--all-targets` to also compile and tests benchmarks
-      # Though currently `divan` does not support all CLI arguments as used by `nextest`,
-      # and benchmarks are unbearably slow anyway, so its not feasible to run in debug builds.
-      - run: cargo llvm-cov nextest --lcov --output-path core.lcov --workspace --all-features
+      - run: cargo llvm-cov nextest --lcov --output-path core.lcov --workspace --all-features --all-targets
       - run: mv target/nextest/default/core-test-results.xml .
 
       - uses: actions/setup-python@v5

README.md

@@ -17,17 +17,19 @@ All details (e.g. SQLite schema, code interfaces) subject to breaking changes un
 ## Developing
 
 Set up your development environment:
-- Install the nightly compiler via [rustup](https://rustup.rs/). At time of writing, `codecov-rs` requires the nightly compiler for niceties such as `#[feature(trait_alias)]`.
+
 - To work on the Python bindings, run `source .envrc` (or use `direnv`) to set up a virtual environment. Update development dependencies with `pip install -r python/requirements.dev.txt`
 - Install lint hooks with `pip install pre-commit && pre-commit install`.
 - Large sample test reports are checked in using [Git LFS](https://git-lfs.com/) in `test_utils/fixtures/**/large` directories (e.g. `test_utils/fixtures/pyreport/large`). Tests and benchmarks may reference them so installing it yourself is recommended.
 
 `codecov-rs` aims to serve as effective documentation for every flavor of every format it supports. To that end, the following are greatly appreciated in submissions:
+
 - Thorough doc comments (`///` / `/**`). For parsers, include snippets that show what inputs look like
 - Granular, in-module unit tests
 - Integration tests with real-world samples (that are safe to distribute; don't send us data from your private repo)
 
 The `core/examples/` directory contains runnable commands for developers including:
+
 - `parse_pyreport`: converts a given pyreport into a SQLite report
 - `sql_to_pyreport`: converts a given SQLite report into a pyreport (report JSON + chunks file)
 
@@ -53,9 +55,8 @@ New parsers should be optional via Cargo features. Adding them to the default fe
 Where possible, parsers should not load their entire input or output into RAM. On the input side, you can avoid that with a _streaming_ parser or by using `memmap2` to map the input file into virtual memory. SQLite makes it straightforward enough to stream outputs to the database.
 
 Coverage formats really run the gamut so there's no one-size-fits-all framework we can use. Some options:
+
 - [`quick_xml`](https://crates.io/crates/quick_xml), a streaming XML parser
-- [`winnow`](https://crates.io/crates/winnow), a parser combinator framework (fork of [`nom`](https://crates.io/crates/nom))
-  - `winnow`'s docs illustrate [how one can write a streaming parser](https://docs.rs/winnow/latest/winnow/_topic/partial/index.html)
 - [`serde`](https://serde.rs/), a popular serialization/deserialization framework
   - `serde`'s docs illustrate [how one can write a streaming parser](https://serde.rs/stream-array.html)
 
@@ -64,6 +65,7 @@ Non-XML formats lack clean OOTB support for streaming so `codecov-rs` currently
 ### Testing
 
 Run tests with:
+
 ```
 # Rust tests
 $ cargo test
@@ -75,6 +77,7 @@ $ pytest
 ### Benchmarks
 
 Run benchmarks with:
+
 ```
 $ cargo bench --features testing
 ```

bindings/src/error.rs

@@ -1,5 +1,6 @@
 pub use codecov_rs::error::CodecovError as RsCodecovError;
-use pyo3::{exceptions::PyRuntimeError, prelude::*};
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::*;
 
 pub struct PyCodecovError(RsCodecovError);
 

bindings/src/lib.rs

@@ -1,4 +1,5 @@
-use std::{fs::File, path::PathBuf};
+use std::fs::File;
+use std::path::PathBuf;
 
 use codecov_rs::{parsers, report};
 use pyo3::prelude::*;

core/Cargo.toml

@@ -10,6 +10,7 @@ testing = []
 
 [dependencies]
 include_dir = "0.7.3"
+memchr = "2.7.4"
 memmap2 = "0.9.5"
 rand = "0.8.5"
 rusqlite = { version = "0.31.0", features = [
@@ -22,7 +23,6 @@ seahash = "4.1.0"
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 thiserror = "1.0.64"
-winnow = "0.5.34"
 
 [dev-dependencies]
 criterion = { version = "2.7.2", package = "codspeed-criterion-compat" }

core/benches/pyreport.rs

@@ -1,12 +1,11 @@
 use std::collections::HashMap;
 
-use codecov_rs::{
-    parsers::pyreport::{chunks, report_json},
-    test_utils::test_report::{TestReport, TestReportBuilder},
-};
+use codecov_rs::parsers::pyreport::{chunks, report_json};
+use codecov_rs::test_utils::test_report::TestReportBuilder;
 use criterion::{criterion_group, criterion_main, Criterion};
-use test_utils::fixtures::{read_fixture, FixtureFormat::Pyreport, FixtureSize::Large};
-use winnow::Parser as _;
+use test_utils::fixtures::read_fixture;
+use test_utils::fixtures::FixtureFormat::Pyreport;
+use test_utils::fixtures::FixtureSize::Large;
 
 criterion_group!(
     benches,
@@ -55,24 +54,25 @@ fn parse_report_json(input: &[u8]) -> report_json::ParsedReportJson {
 }
 
 fn simple_chunks(c: &mut Criterion) {
-    let chunks = &[
+    let chunks: &[&[u8]] = &[
         // Header and one chunk with an empty line
-        "{}\n<<<<< end_of_header >>>>>\n{}\n",
-        // No header, one chunk with a populated line and an  empty line
-        "{}\n[1, null, [[0, 1]]]\n",
+        b"{}\n<<<<< end_of_header >>>>>\n{}\n",
+        // No header, one chunk with a populated line and an empty line
+        b"{}\n[1, null, [[0, 1]]]\n",
         // No header, two chunks, the second having just one empty line
-        "{}\n[1, null, [[0, 1]]]\n\n<<<<< end_of_chunk >>>>>\n{}\n",
+        b"{}\n[1, null, [[0, 1]]]\n\n<<<<< end_of_chunk >>>>>\n{}\n",
         // Header, two chunks, the second having multiple data lines and an empty line
-        "{}\n<<<<< end_of_header >>>>>\n{}\n[1, null, [[0, 1]]]\n\n<<<<< end_of_chunk >>>>>\n{}\n[1, null, [[0, 1]]]\n[1, null, [[0, 1]]]\n",
+        b"{}\n<<<<< end_of_header >>>>>\n{}\n[1, null, [[0, 1]]]\n\n<<<<< end_of_chunk >>>>>\n{}\n[1, null, [[0, 1]]]\n[1, null, [[0, 1]]]\n",
     ];
 
     let files = HashMap::from([(0, 0), (1, 1), (2, 2)]);
     let sessions = HashMap::from([(0, 0), (1, 1), (2, 2)]);
 
+    let report_json = report_json::ParsedReportJson { files, sessions };
     c.bench_function("simple_chunks", |b| {
         b.iter(|| {
             for input in chunks {
-                parse_chunks_file(input, files.clone(), sessions.clone())
+                parse_chunks_file_serde(input, report_json.clone());
             }
         })
     });
@@ -87,7 +87,6 @@ fn complex_chunks(c: &mut Criterion) {
         "worker-c71ddfd4cb1753c7a540e5248c2beaa079fc3341-chunks.txt",
     )
     .unwrap();
-    let chunks = std::str::from_utf8(&chunks).unwrap();
 
     // parsing the chunks depends on having loaded the `report_json`
     let report = read_fixture(
@@ -96,23 +95,14 @@ fn complex_chunks(c: &mut Criterion) {
         "worker-c71ddfd4cb1753c7a540e5248c2beaa079fc3341-report_json.json",
     )
     .unwrap();
-    let report_json::ParsedReportJson { files, sessions } = parse_report_json(&report);
+    let report_json = parse_report_json(&report);
 
     c.bench_function("complex_chunks", |b| {
-        b.iter(|| parse_chunks_file(chunks, files.clone(), sessions.clone()))
+        b.iter(|| parse_chunks_file_serde(&chunks, report_json.clone()))
     });
 }
 
-fn parse_chunks_file(input: &str, files: HashMap<usize, i64>, sessions: HashMap<usize, i64>) {
+fn parse_chunks_file_serde(input: &[u8], report_json: report_json::ParsedReportJson) {
     let report_builder = TestReportBuilder::default();
-
-    let chunks_ctx = chunks::ParseCtx::new(report_builder, files, sessions);
-    let mut chunks_stream = chunks::ReportOutputStream::<&str, TestReport, TestReportBuilder> {
-        input,
-        state: chunks_ctx,
-    };
-
-    chunks::parse_chunks_file
-        .parse_next(&mut chunks_stream)
-        .unwrap();
+    chunks::parse_chunks_file(input, report_json, report_builder).unwrap();
 }

core/examples/parse_pyreport.rs

@@ -1,6 +1,10 @@
-use std::{env, fs::File, path::PathBuf};
+use std::env;
+use std::fs::File;
+use std::path::PathBuf;
 
-use codecov_rs::{error::Result, parsers::pyreport::parse_pyreport, report::SqliteReportBuilder};
+use codecov_rs::error::Result;
+use codecov_rs::parsers::pyreport::parse_pyreport;
+use codecov_rs::report::SqliteReportBuilder;
 
 fn usage_error() -> ! {
     println!("Usage:");

core/examples/sql_to_pyreport.rs

@@ -1,9 +1,9 @@
-use std::{env, fs::File};
+use std::env;
+use std::fs::File;
 
-use codecov_rs::{
-    error::Result,
-    report::{pyreport::ToPyreport, SqliteReport},
-};
+use codecov_rs::error::Result;
+use codecov_rs::report::pyreport::ToPyreport;
+use codecov_rs::report::SqliteReport;
 
 fn usage_error() -> ! {
     println!("Usage:");

core/src/error.rs

@@ -1,5 +1,7 @@
 use thiserror::Error;
 
+use crate::parsers::pyreport::chunks::ChunksFileParseError;
+
 pub type Result<T, E = CodecovError> = std::result::Result<T, E>;
 
 #[derive(Error, Debug)]
@@ -13,10 +15,6 @@ pub enum CodecovError {
     #[error("report builder error: '{0}'")]
     ReportBuilderError(String),
 
-    // Can't use #[from]
-    #[error("parser error: '{0}'")]
-    ParserError(winnow::error::ContextError),
-
     #[error("parser error: '{0}'")]
     Json(#[from] serde_json::Error),
 
@@ -26,4 +24,7 @@ pub enum CodecovError {
     #[cfg(feature = "pyreport")]
     #[error("failed to convert sqlite to pyreport: '{0}'")]
     PyreportConversionError(String),
+
+    #[error(transparent)]
+    ChunksFileParseError(#[from] ChunksFileParseError),
 }

core/src/lib.rs

@@ -1,5 +1,3 @@
-#![feature(trait_alias)]
-
 pub mod report;
 
 pub mod parsers;