wip

analyzeme/benches/serialization_bench.rs

@@ -3,12 +3,12 @@
 extern crate test;
 
 use analyzeme::testing_common;
-use measureme::{FileSerializationSink, MmapSerializationSink};
+use measureme::{FileSinkConfig, MmapSinkConfig, PagedSinkConfig};
 
 #[bench]
 fn bench_file_serialization_sink(bencher: &mut test::Bencher) {
     bencher.iter(|| {
-        testing_common::run_serialization_bench::<FileSerializationSink>(
+        testing_common::run_serialization_bench::<FileSinkConfig>(
             "file_serialization_sink_test",
             500_000,
             1,
@@ -19,18 +19,29 @@ fn bench_file_serialization_sink(bencher: &mut test::Bencher) {
 #[bench]
 fn bench_mmap_serialization_sink(bencher: &mut test::Bencher) {
     bencher.iter(|| {
-        testing_common::run_serialization_bench::<MmapSerializationSink>(
+        testing_common::run_serialization_bench::<MmapSinkConfig>(
             "mmap_serialization_sink_test",
             500_000,
             1,
         );
     });
 }
 
+#[bench]
+fn bench_paged_serialization_sink(bencher: &mut test::Bencher) {
+    bencher.iter(|| {
+        testing_common::run_serialization_bench::<PagedSinkConfig>(
+            "paged_serialization_sink_test",
+            500_000,
+            1,
+        );
+    });
+}
+
 #[bench]
 fn bench_file_serialization_sink_8_threads(bencher: &mut test::Bencher) {
     bencher.iter(|| {
-        testing_common::run_serialization_bench::<FileSerializationSink>(
+        testing_common::run_serialization_bench::<FileSinkConfig>(
             "file_serialization_sink_test",
             50_000,
             8,
@@ -41,10 +52,21 @@ fn bench_file_serialization_sink_8_threads(bencher: &mut test::Bencher) {
 #[bench]
 fn bench_mmap_serialization_sink_8_threads(bencher: &mut test::Bencher) {
     bencher.iter(|| {
-        testing_common::run_serialization_bench::<MmapSerializationSink>(
+        testing_common::run_serialization_bench::<MmapSinkConfig>(
             "mmap_serialization_sink_test",
             50_000,
             8,
         );
     });
 }
+
+#[bench]
+fn bench_paged_serialization_sink_8_threads(bencher: &mut test::Bencher) {
+    bencher.iter(|| {
+        testing_common::run_serialization_bench::<PagedSinkConfig>(
+            "paged_serialization_sink_test",
+            50_000,
+            8,
+        );
+    });
+}

analyzeme/src/profiling_data.rs

@@ -45,14 +45,49 @@ pub struct ProfilingData {
 
 impl ProfilingData {
     pub fn new(path_stem: &Path) -> Result<ProfilingData, Box<dyn Error>> {
-        let paths = ProfilerFiles::new(path_stem);
+        if path_stem.with_extension("rspd").exists() {
+            use std::convert::TryInto;
+            use std::io::Read;
 
-        let string_data = fs::read(paths.string_data_file).expect("couldn't read string_data file");
-        let index_data =
-            fs::read(paths.string_index_file).expect("couldn't read string_index file");
-        let event_data = fs::read(paths.events_file).expect("couldn't read events file");
+            let page_size = measureme::PagedSinkConfig::PAGE_SIZE as u64;
 
-        ProfilingData::from_buffers(string_data, index_data, event_data)
+            let mut file = fs::File::open(path_stem.with_extension("rspd"))?;
+            let file_len = file.metadata()?.len();
+            if file_len % page_size != 0 {
+                panic!("TODO: should be an error");
+            }
+            let num_pages = file_len / page_size;
+
+            let mut event_data = Vec::new();
+            let mut string_data = Vec::new();
+            let mut index_data = Vec::new();
+
+            let mut page = vec![0u8; page_size as usize];
+
+            for _ in 0 .. num_pages {
+                file.read_exact(&mut page[..])?;
+
+                let bytes_used = u32::from_be_bytes(page[1..5].try_into().unwrap()) as usize;
+
+                match page[0] {
+                    1 => event_data.extend_from_slice(&page[5 .. 5 + bytes_used]),
+                    2 => string_data.extend_from_slice(&page[5 .. 5 + bytes_used]),
+                    3 => index_data.extend_from_slice(&page[5 .. 5 + bytes_used]),
+                    _ => unreachable!(),
+                }
+            }
+
+            ProfilingData::from_buffers(string_data, index_data, event_data)
+        } else {
+            let paths = ProfilerFiles::new(path_stem);
+
+            let string_data = fs::read(paths.string_data_file).expect("couldn't read string_data file");
+            let index_data =
+                fs::read(paths.string_index_file).expect("couldn't read string_index file");
+            let event_data = fs::read(paths.events_file).expect("couldn't read events file");
+
+            ProfilingData::from_buffers(string_data, index_data, event_data)
+        }
     }
 
     pub fn from_buffers(

analyzeme/src/testing_common.rs

@@ -1,6 +1,6 @@
 use crate::timestamp::Timestamp;
 use crate::{Event, ProfilingData};
-use measureme::{EventId, EventIdBuilder, Profiler, SerializationSink, StringId};
+use measureme::{EventId, EventIdBuilder, Profiler, ProfilerConfig, StringId};
 use rustc_hash::FxHashMap;
 use std::borrow::Cow;
 use std::default::Default;
@@ -36,12 +36,12 @@ impl ExpectedEvent {
 }
 
 // Generate some profiling data. This is the part that would run in rustc.
-fn generate_profiling_data<S: SerializationSink>(
+fn generate_profiling_data<C: ProfilerConfig>(
     filestem: &Path,
     num_stacks: usize,
     num_threads: usize,
 ) -> Vec<Event<'static>> {
-    let profiler = Arc::new(Profiler::<S>::new(Path::new(filestem)).unwrap());
+    let profiler = Arc::new(Profiler::<C>::new(Path::new(filestem)).unwrap());
 
     let event_id_virtual = EventId::from_label(StringId::new_virtual(42));
     let event_id_builder = EventIdBuilder::new(&profiler);
@@ -187,26 +187,26 @@ fn collect_events_per_thread<'a>(
     per_thread
 }
 
-pub fn run_serialization_bench<S: SerializationSink>(
+pub fn run_serialization_bench<C: ProfilerConfig>(
     file_name_stem: &str,
     num_events: usize,
     num_threads: usize,
 ) {
     let filestem = mk_filestem(file_name_stem);
-    generate_profiling_data::<S>(&filestem, num_events, num_threads);
+    generate_profiling_data::<C>(&filestem, num_events, num_threads);
 }
 
-pub fn run_end_to_end_serialization_test<S: SerializationSink>(
+pub fn run_end_to_end_serialization_test<C: ProfilerConfig>(
     file_name_stem: &str,
     num_threads: usize,
 ) {
     let filestem = mk_filestem(file_name_stem);
-    let expected_events = generate_profiling_data::<S>(&filestem, 10_000, num_threads);
+    let expected_events = generate_profiling_data::<C>(&filestem, 10_000, num_threads);
     process_profiling_data(&filestem, &expected_events);
 }
 
-fn pseudo_invocation<S: SerializationSink>(
-    profiler: &Profiler<S>,
+fn pseudo_invocation<C: ProfilerConfig>(
+    profiler: &Profiler<C>,
     random: usize,
     thread_id: u32,
     recursions_left: usize,

analyzeme/tests/serialization.rs

@@ -1,34 +1,43 @@
 use analyzeme::testing_common::run_end_to_end_serialization_test;
-use measureme::{FileSerializationSink, MmapSerializationSink};
+use measureme::{FileSinkConfig, MmapSinkConfig, PagedSinkConfig};
 
 #[test]
 fn test_file_serialization_sink_1_thread() {
-    run_end_to_end_serialization_test::<FileSerializationSink>(
-        "file_serialization_sink_test_1_thread",
-        1,
-    );
+    run_end_to_end_serialization_test::<FileSinkConfig>("file_serialization_sink_test_1_thread", 1);
 }
 
 #[test]
 fn test_file_serialization_sink_8_threads() {
-    run_end_to_end_serialization_test::<FileSerializationSink>(
+    run_end_to_end_serialization_test::<FileSinkConfig>(
         "file_serialization_sink_test_8_threads",
         8,
     );
 }
 
 #[test]
 fn test_mmap_serialization_sink_1_thread() {
-    run_end_to_end_serialization_test::<MmapSerializationSink>(
-        "mmap_serialization_sink_test_1_thread",
-        1,
-    );
+    run_end_to_end_serialization_test::<MmapSinkConfig>("mmap_serialization_sink_test_1_thread", 1);
 }
 
 #[test]
 fn test_mmap_serialization_sink_8_threads() {
-    run_end_to_end_serialization_test::<MmapSerializationSink>(
+    run_end_to_end_serialization_test::<MmapSinkConfig>(
         "mmap_serialization_sink_test_8_threads",
         8,
     );
 }
+
+
+
+#[test]
+fn test_paged_serialization_sink_1_thread() {
+    run_end_to_end_serialization_test::<PagedSinkConfig>("paged_serialization_sink_test_1_thread", 1);
+}
+
+#[test]
+fn test_paged_serialization_sink_8_threads() {
+    run_end_to_end_serialization_test::<PagedSinkConfig>(
+        "paged_serialization_sink_test_8_threads",
+        8,
+    );
+}

measureme/src/event_id.rs

@@ -1,4 +1,4 @@
-use crate::{Profiler, SerializationSink, StringComponent, StringId};
+use crate::{Profiler, ProfilerConfig, StringComponent, StringId};
 
 /// Event IDs are strings conforming to the following grammar:
 ///
@@ -53,12 +53,12 @@ impl EventId {
     }
 }
 
-pub struct EventIdBuilder<'p, S: SerializationSink> {
-    profiler: &'p Profiler<S>,
+pub struct EventIdBuilder<'p, C: ProfilerConfig> {
+    profiler: &'p Profiler<C>,
 }
 
-impl<'p, S: SerializationSink> EventIdBuilder<'p, S> {
-    pub fn new(profiler: &Profiler<S>) -> EventIdBuilder<'_, S> {
+impl<'p, C: ProfilerConfig> EventIdBuilder<'p, C> {
+    pub fn new(profiler: &Profiler<C>) -> EventIdBuilder<'_, C> {
         EventIdBuilder { profiler }
     }
 

measureme/src/file_serialization_sink.rs

@@ -1,9 +1,29 @@
 use crate::serialization::{Addr, SerializationSink};
+use crate::{GenericError, ProfilerConfig, ProfilerFiles, SerializationSinks};
 use parking_lot::Mutex;
 use std::error::Error;
 use std::fs;
 use std::io::Write;
-use std::path::Path;
+use std::{path::Path, sync::Arc};
+
+#[derive(Copy, Clone, Debug)]
+pub struct FileSinkConfig;
+
+impl ProfilerConfig for FileSinkConfig {
+    type SerializationSink = FileSerializationSink;
+
+    fn create_sinks<P: AsRef<Path>>(
+        path_stem: P,
+    ) -> Result<SerializationSinks<FileSerializationSink>, GenericError> {
+        let paths = ProfilerFiles::new(path_stem.as_ref());
+
+        Ok(SerializationSinks {
+            events: Arc::new(FileSerializationSink::from_path(&paths.events_file)?),
+            string_data: Arc::new(FileSerializationSink::from_path(&paths.string_data_file)?),
+            string_index: Arc::new(FileSerializationSink::from_path(&paths.string_index_file)?),
+        })
+    }
+}
 
 pub struct FileSerializationSink {
     data: Mutex<Inner>,

measureme/src/lib.rs

@@ -35,18 +35,17 @@
 //! [`Profiler::record_event()`]: Profiler::record_event
 //! [`Profiler::start_recording_interval_event()`]: Profiler::start_recording_interval_event
 //! [`StringId`]: StringId
-#![allow(renamed_and_removed_lints)] // intra_doc_link_resolution_failure is renamed on nightly
-#![deny(
-    warnings,
-    intra_doc_link_resolution_failure,
-)]
+#![allow(renamed_and_removed_lints)]
+// intra_doc_link_resolution_failure is renamed on nightly
+#![deny(warnings, intra_doc_link_resolution_failure)]
 
 pub mod event_id;
 pub mod file_header;
 #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
 mod file_serialization_sink;
 #[cfg(not(target_arch = "wasm32"))]
 mod mmap_serialization_sink;
+mod paged_serialization_sink;
 mod profiler;
 mod raw_event;
 mod serialization;
@@ -56,10 +55,16 @@ pub mod rustc;
 
 pub use crate::event_id::{EventId, EventIdBuilder};
 #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
-pub use crate::file_serialization_sink::FileSerializationSink;
+pub use crate::file_serialization_sink::{FileSerializationSink, FileSinkConfig};
 #[cfg(not(target_arch = "wasm32"))]
-pub use crate::mmap_serialization_sink::MmapSerializationSink;
-pub use crate::profiler::{Profiler, ProfilerFiles, TimingGuard};
+pub use crate::mmap_serialization_sink::{MmapSerializationSink, MmapSinkConfig};
+pub use crate::profiler::{
+    Profiler, ProfilerConfig, ProfilerFiles, SerializationSinks, TimingGuard,
+};
 pub use crate::raw_event::{RawEvent, MAX_INSTANT_TIMESTAMP, MAX_INTERVAL_TIMESTAMP};
 pub use crate::serialization::{Addr, ByteVecSink, SerializationSink};
 pub use crate::stringtable::{SerializableString, StringComponent, StringId, StringTableBuilder};
+
+pub use crate::paged_serialization_sink::{PagedSinkConfig, PagedWriter};
+
+pub type GenericError = Box<dyn std::error::Error + Send + Sync>;

measureme/src/mmap_serialization_sink.rs

@@ -1,10 +1,35 @@
-use crate::serialization::{Addr, SerializationSink};
+use crate::{
+    serialization::{Addr, SerializationSink},
+    GenericError, ProfilerConfig, ProfilerFiles, SerializationSinks,
+};
 use memmap::MmapMut;
 use std::error::Error;
 use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+#[derive(Copy, Clone, Debug)]
+pub struct MmapSinkConfig;
+
+impl ProfilerConfig for MmapSinkConfig {
+    type SerializationSink = MmapSerializationSink;
+
+    fn create_sinks<P: AsRef<Path>>(
+        path_stem: P,
+    ) -> Result<SerializationSinks<MmapSerializationSink>, GenericError> {
+        let paths = ProfilerFiles::new(path_stem.as_ref());
+
+        Ok(SerializationSinks {
+            events: Arc::new(MmapSerializationSink::from_path(&paths.events_file)?),
+            string_data: Arc::new(MmapSerializationSink::from_path(&paths.string_data_file)?),
+            string_index: Arc::new(MmapSerializationSink::from_path(&paths.string_index_file)?),
+        })
+    }
+}
 
 pub struct MmapSerializationSink {
     mapped_file: MmapMut,