Benchmarks (#16)

* Make target for running benches
* Move testing out to a script
* Moved fuzz check to stable

.github/workflows/python-release.yml

@@ -14,7 +14,7 @@ jobs:
       with:
         manylinux: auto
         command: build
-        args: --release -o dist --cargo-extra-args="--all-features"
+        args: --release -o dist --cargo-extra-args="--features=python"
     - name: Upload wheels
       uses: actions/upload-artifact@v2
       with:
@@ -29,7 +29,7 @@ jobs:
     - uses: messense/maturin-action@v1
       with:
         command: build
-        args: --release --no-sdist -o dist --universal2 --cargo-extra-args="--all-features"
+        args: --release --no-sdist -o dist --universal2 --cargo-extra-args="--features=python"
     - name: Upload wheels
       uses: actions/upload-artifact@v2
       with:

.github/workflows/test.yml

@@ -33,24 +33,9 @@ jobs:
 
     - uses: Swatinem/rust-cache@v1
 
-    - name: Rust tests
-      run: cargo test
-    - name: Check fuzzer
-      run: cd fuzz && cargo check
-    - name: Clippy
-      run: cargo clippy
-    - name: Check formatting
-      run: cargo fmt -- --check
-
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
         python-version: "3.7"
-    - name: Install virtualenv
-      run: pip install virtualenv
-    - name: Setup virtualenv
-      run: make venv
-    - name: Build python library
-      run: source venv/bin/activate && maturin develop --cargo-extra-args="--all-features"
-    - name: Python tests
-      run: source venv/bin/activate && pytest --color=yes python_tests/
+
+    - run: ./scripts/ci.sh "${{ matrix.rust }}" "${{ matrix.os }}"

Cargo.toml

@@ -17,6 +17,7 @@ crate-type = ["cdylib", "rlib"]
 default = ["sqlite"]
 sqlite = ["rusqlite", "zstd", "r2d2", "r2d2_sqlite", "tempfile"]
 python = ["pyo3", "sqlite"]
+benches = []
 
 [dependencies]
 string_cache = "0.8.4"

Makefile

@@ -1,11 +1,14 @@
 export RUST_BACKTRACE=1
 
-.PHONY: test fuzz check fmt
+.PHONY: bench test fuzz check fmt
 
 venv:
 	virtualenv -v venv -p python3.7
 	. venv/bin/activate && pip install maturin pytest
 
+bench:
+	cargo +nightly bench --features=benches
+
 test:
 	cargo test --all-features
 	make venv
@@ -16,10 +19,10 @@ fuzz:
 	cargo +nightly fuzz run compare
 
 check:
-	cargo +stable check --all-features
+	cargo +stable check
 	cargo +nightly check --all-features
 	cd fuzz && cargo +stable check
-	cargo +stable clippy --all-features
+	cargo +nightly clippy --all-features
 	cargo fmt -- --check
 
 fmt:

scripts/ci.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+rust_variant=$1
+os=$2
+
+cargo test
+
+if [ "$os" == "ubuntu-latest" ]; then
+    if [ "$rust_variant" == "stable" ]; then
+        cargo fmt -- --check
+        (cd fuzz && cargo check)
+    else
+        cargo check --all-features
+        cargo clippy --all-features
+    fi
+fi
+
+pip install virtualenv
+make venv
+source venv/bin/activate
+maturin develop --cargo-extra-args="--features=python"
+pytest --color=yes python_tests/

src/benches.rs

@@ -0,0 +1,69 @@
+use std::borrow::Cow;
+use std::thread;
+use std::time::Duration;
+
+use crate::{Entry, Range, Store};
+
+use string_cache::Atom;
+use test::Bencher;
+
+/// Defines a benchmark function.
+#[macro_export]
+macro_rules! define_bench {
+    ($name:ident, $store_constructor:expr) => {
+        #[bench]
+        fn $name(b: &mut test::Bencher) {
+            let store = $store_constructor;
+            $crate::benches::$name(b, &store);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! bench_store_impl {
+    ($code:expr) => {
+        define_bench!(push, $code);
+        define_bench!(push_parallel, $code);
+        define_bench!(iter, $code);
+    };
+}
+
+pub fn push<S: Store>(b: &mut Bencher, store: &S) {
+    let entry = Entry::new_with_time(Duration::from_micros(1), Atom::from("bench_push"), vec![1, 2, 3]);
+    b.iter(|| {
+        store.push(Cow::Borrowed(&entry)).unwrap();
+    });
+}
+
+pub fn push_parallel<S: Store + Clone + 'static>(b: &mut Bencher, store: &S) {
+    b.iter(|| {
+        let mut threads = Vec::default();
+        for i in 1..11 {
+            let store = store.clone();
+            threads.push(thread::spawn(move || {
+                for j in 1..1001 {
+                    let idx = i * j;
+                    let entry = Entry::new_with_time(
+                        Duration::from_micros(idx),
+                        Atom::from("bench_push_parallel"),
+                        vec![1, 2, 3],
+                    );
+                    store.push(Cow::Owned(entry)).unwrap();
+                }
+            }));
+        }
+        for thread in threads.into_iter() {
+            thread.join().unwrap();
+        }
+    });
+}
+
+pub fn iter<S: Store>(b: &mut Bencher, store: &S) {
+    for i in 0..=255u8 {
+        let entry = Entry::new_with_time(Duration::from_micros(i.into()), Atom::from("bench_iter"), vec![i]);
+        store.push(Cow::Owned(entry)).unwrap();
+    }
+    b.iter(|| {
+        assert_eq!(store.range(.., None).unwrap().iter().unwrap().count(), 256);
+    });
+}

src/lib.rs

@@ -1,3 +1,7 @@
+#![cfg_attr(feature = "benches", feature(test))]
+#[cfg(feature = "benches")]
+extern crate test;
+
 mod entry;
 mod errors;
 mod memory;
@@ -10,6 +14,9 @@ pub mod tests;
 mod python;
 #[cfg(feature = "sqlite")]
 mod sqlite;
+#[cfg(feature = "benches")]
+#[macro_use]
+pub mod benches;
 
 pub use self::entry::Entry;
 pub use self::errors::Error;

src/memory.rs

@@ -94,7 +94,18 @@ impl Range for MemoryRange {
 
 #[cfg(test)]
 mod tests {
-    use super::MemoryStore;
     use crate::{define_test, test_store_impl};
-    test_store_impl!(MemoryStore::default());
+    test_store_impl!({
+        use super::MemoryStore;
+        MemoryStore::default()
+    });
+}
+
+#[cfg(feature = "benches")]
+mod benches {
+    use crate::{bench_store_impl, define_bench};
+    bench_store_impl!({
+        use crate::MemoryStore;
+        MemoryStore::default()
+    });
 }

src/sqlite.rs

@@ -242,10 +242,21 @@ impl Iterator for SqliteRangeIterator {
 
 #[cfg(test)]
 mod tests {
-    use super::SqliteStore;
-    use tempfile::NamedTempFile;
-
+    use crate::{define_test, test_store_impl};
     test_store_impl!({
+        use super::SqliteStore;
+        use tempfile::NamedTempFile;
+        let file = NamedTempFile::new().unwrap().into_temp_path();
+        SqliteStore::new(file, None).unwrap()
+    });
+}
+
+#[cfg(feature = "benches")]
+mod benches {
+    use crate::{bench_store_impl, define_bench};
+    bench_store_impl!({
+        use super::SqliteStore;
+        use tempfile::NamedTempFile;
         let file = NamedTempFile::new().unwrap().into_temp_path();
         SqliteStore::new(file, None).unwrap()
     });