diff --git a/filpreload/src/lib.rs b/filpreload/src/lib.rs index aa98d81..05ca54c 100644 --- a/filpreload/src/lib.rs +++ b/filpreload/src/lib.rs @@ -1,5 +1,6 @@ #![deny(unsafe_op_in_unsafe_fn)] use parking_lot::Mutex; +use pymemprofile_api::memorytracking::LineNumberInfo::LineNumber; use pymemprofile_api::memorytracking::{ AllocationTracker, CallSiteId, Callstack, FunctionId, VecFunctionLocations, PARENT_PROCESS, }; @@ -57,8 +58,10 @@ fn add_function(filename: String, function_name: String) -> FunctionId { /// Add to per-thread function stack: fn start_call(call_site: FunctionId, parent_line_number: u16, line_number: u16) { THREAD_CALLSTACK.with(|cs| { - cs.borrow_mut() - .start_call(parent_line_number, CallSiteId::new(call_site, line_number)); + cs.borrow_mut().start_call( + parent_line_number as u32, + CallSiteId::new(call_site, LineNumber(line_number as u32)), + ); }); } @@ -136,7 +139,7 @@ fn add_allocation( // Will fail during thread shutdown, but not much we can do at that point. let callstack_id = THREAD_CALLSTACK.try_with(|tcs| { let mut callstack = tcs.borrow_mut(); - callstack.id_for_new_allocation(line_number, |callstack| { + callstack.id_for_new_allocation(line_number as u32, |callstack| { allocations.get_callstack_id(callstack) }) })?; diff --git a/memapi/src/memorytracking.rs b/memapi/src/memorytracking.rs index 856042f..998a94f 100644 --- a/memapi/src/memorytracking.rs +++ b/memapi/src/memorytracking.rs @@ -82,19 +82,35 @@ impl FunctionLocations for VecFunctionLocations { } } -pub type LineNumber = u16; // TODO u32, newtype +/// Either the line number, or the bytecode index needed to get it. +#[derive(Copy, Clone, Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] +pub enum LineNumberInfo { + LineNumber(u32), + BytecodeIndex(i32), +} + +impl LineNumberInfo { + fn get_line_number(&self) -> u32 { + if let LineNumberInfo::LineNumber(lineno) = self { + *lineno + } else { + debug_assert!(false, "This should never happen."); + 0 + } + } +} /// A specific location: file + function + line number. #[derive(Clone, Debug, PartialEq, Eq, Copy, Hash, Serialize, Deserialize)] pub struct CallSiteId { /// The function + filename. We use IDs for performance reasons (faster hashing). - function: FunctionId, + pub function: FunctionId, /// Line number within the _file_, 1-indexed. - line_number: LineNumber, + pub line_number: LineNumberInfo, } impl CallSiteId { - pub fn new(function: FunctionId, line_number: u16) -> CallSiteId { + pub fn new(function: FunctionId, line_number: LineNumberInfo) -> CallSiteId { CallSiteId { function, line_number, @@ -108,7 +124,7 @@ impl CallSiteId { pub struct Callstack { calls: Vec, #[derivative(Hash = "ignore", PartialEq = "ignore")] - cached_callstack_id: Option<(u16, CallstackId)>, // first bit is line number + cached_callstack_id: Option<(u32, CallstackId)>, // first bit is line number } impl Callstack { @@ -126,10 +142,14 @@ impl Callstack { } } - pub fn start_call(&mut self, parent_line_number: u16, callsite_id: CallSiteId) { + pub fn to_vec(&self) -> Vec { + self.calls.clone() + } + + pub fn start_call(&mut self, parent_line_number: u32, callsite_id: CallSiteId) { if parent_line_number != 0 { if let Some(mut call) = self.calls.last_mut() { - call.line_number = parent_line_number; + call.line_number = LineNumberInfo::LineNumber(parent_line_number); } } self.calls.push(callsite_id); @@ -141,7 +161,7 @@ impl Callstack { self.cached_callstack_id = None; } - pub fn id_for_new_allocation(&mut self, line_number: u16, get_callstack_id: F) -> CallstackId + pub fn id_for_new_allocation(&mut self, line_number: u32, get_callstack_id: F) -> CallstackId where F: FnOnce(&Callstack) -> CallstackId, { @@ -156,7 +176,7 @@ impl Callstack { // Set the new line number: if line_number != 0 { if let Some(call) = self.calls.last_mut() { - call.line_number = line_number; + call.line_number = LineNumberInfo::LineNumber(line_number); } } @@ -194,7 +214,8 @@ impl Callstack { .map(|(id, (function, filename))| { if to_be_post_processed { // Get Python code. - let code = linecache.get_source_line(filename, id.line_number as usize); + let code = linecache + .get_source_line(filename, id.line_number.get_line_number() as usize); // Leading whitespace is dropped by SVG, so we'd like to // replace it with non-breaking space. However, inferno // trims whitespace @@ -215,7 +236,7 @@ impl Callstack { format!( "{filename}:{line} ({function});\u{2800}{code}", filename = filename, - line = id.line_number, + line = id.line_number.get_line_number(), function = function, code = &code.trim_end(), ) @@ -223,7 +244,7 @@ impl Callstack { format!( "{filename}:{line} ({function})", filename = filename, - line = id.line_number, + line = id.line_number.get_line_number(), function = function, ) } @@ -335,6 +356,10 @@ impl Allocation { } } +fn same_callstack(callstack: &Callstack) -> Cow { + Cow::Borrowed(callstack) +} + /// The main data structure tracking everything. pub struct AllocationTracker { // malloc()/calloc(): @@ -607,18 +632,22 @@ impl AllocationTracker { self.dump_to_flamegraph(path, true, "peak-memory", "Peak Tracked Memory Usage", true); } - pub fn to_lines( - &self, + pub fn to_lines<'a, F>( + &'a self, peak: bool, to_be_post_processed: bool, - ) -> impl ExactSizeIterator + '_ { + update_callstack: F, + ) -> impl ExactSizeIterator + '_ + where + F: Fn(&Callstack) -> Cow + 'a, + { let by_call = self.combine_callstacks(peak).into_iter(); let id_to_callstack = self.interner.get_reverse_map(); let mut linecache = LineCacher::default(); by_call.map(move |(callstack_id, size)| { format!( "{} {}", - id_to_callstack.get(&callstack_id).unwrap().as_string( + update_callstack(id_to_callstack.get(&callstack_id).unwrap()).as_string( to_be_post_processed, &self.functions, ";", @@ -675,7 +704,7 @@ impl AllocationTracker { subtitle, "bytes", to_be_post_processed, - |tbpp| self.to_lines(peak, tbpp), + |tbpp| self.to_lines(peak, tbpp, same_callstack), ) } @@ -746,8 +775,9 @@ impl AllocationTracker { #[cfg(test)] mod tests { - use crate::memorytracking::{ProcessUid, PARENT_PROCESS}; + use crate::memorytracking::{same_callstack, ProcessUid, PARENT_PROCESS}; + use super::LineNumberInfo::LineNumber; use super::{ Allocation, AllocationTracker, CallSiteId, Callstack, CallstackInterner, FunctionId, FunctionLocations, VecFunctionLocations, HIGH_32BIT, MIB, @@ -815,7 +845,7 @@ mod tests { let (process, allocation_size) = *allocated_sizes.get(i).unwrap(); let process = ProcessUid(process); let mut cs = Callstack::new(); - cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0)); + cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0))); let cs_id = tracker.get_callstack_id(&cs); tracker.add_allocation(process, i as usize, allocation_size, cs_id); expected_memory_usage.push_back(allocation_size); @@ -854,7 +884,7 @@ mod tests { let (process, allocation_size) = *allocated_sizes.get(i).unwrap(); let process = ProcessUid(process); let mut cs = Callstack::new(); - cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0)); + cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0))); let csid = tracker.get_callstack_id(&cs); tracker.add_anon_mmap(process, addresses[i] as usize, allocation_size, csid); expected_memory_usage.push_back(allocation_size); @@ -891,7 +921,7 @@ mod tests { let (process, allocation_size) = *allocated_sizes.get(i).unwrap(); let process = ProcessUid(process); let mut cs = Callstack::new(); - cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0)); + cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0))); let cs_id = tracker.get_callstack_id(&cs); tracker.add_allocation(process, i as usize, allocation_size, cs_id); expected_memory_usage += allocation_size; @@ -900,7 +930,7 @@ mod tests { let (process, allocation_size) = *allocated_mmaps.get(i).unwrap(); let process = ProcessUid(process); let mut cs = Callstack::new(); - cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0)); + cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0))); let csid = tracker.get_callstack_id(&cs); tracker.add_anon_mmap(process, mmap_addresses[i] as usize, allocation_size, csid); expected_memory_usage += allocation_size; @@ -933,9 +963,10 @@ mod tests { // Parent line number does nothing if it's first call: let mut cs1 = Callstack::new(); - let id1 = CallSiteId::new(fid1, 2); - let id2 = CallSiteId::new(fid3, 45); - let id3 = CallSiteId::new(fid5, 6); + // CallSiteId::new($a, $b) ==>> CallSiteId::new($a, LineNumber($b)) + let id1 = CallSiteId::new(fid1, LineNumber(2)); + let id2 = CallSiteId::new(fid3, LineNumber(45)); + let id3 = CallSiteId::new(fid5, LineNumber(6)); cs1.start_call(123, id1); assert_eq!(cs1.calls, vec![id1]); @@ -950,7 +981,11 @@ mod tests { cs2.start_call(12, id3); assert_eq!( cs2.calls, - vec![CallSiteId::new(fid1, 10), CallSiteId::new(fid3, 12), id3] + vec![ + CallSiteId::new(fid1, LineNumber(10)), + CallSiteId::new(fid3, LineNumber(12)), + id3 + ] ); } @@ -960,10 +995,10 @@ mod tests { let fid3 = FunctionId::new(3u64); let mut cs1 = Callstack::new(); - cs1.start_call(0, CallSiteId::new(fid1, 2)); + cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2))); let cs1b = cs1.clone(); let mut cs2 = Callstack::new(); - cs2.start_call(0, CallSiteId::new(fid3, 4)); + cs2.start_call(0, CallSiteId::new(fid3, LineNumber(4))); let cs3 = Callstack::new(); let cs3b = Callstack::new(); @@ -1014,7 +1049,7 @@ mod tests { let fid1 = FunctionId::new(1u64); - cs1.start_call(0, CallSiteId::new(fid1, 2)); + cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2))); let id1 = cs1.id_for_new_allocation(1, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ())); let id2 = @@ -1025,7 +1060,7 @@ mod tests { assert_ne!(id2, id0); assert_ne!(id2, id1); - cs1.start_call(3, CallSiteId::new(fid1, 2)); + cs1.start_call(3, CallSiteId::new(fid1, LineNumber(2))); let id3 = cs1.id_for_new_allocation(4, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ())); assert_ne!(id3, id0); @@ -1041,7 +1076,7 @@ mod tests { assert_eq!(id1, id1c); // Check for cache invalidation in start_call: - cs1.start_call(1, CallSiteId::new(fid1, 1)); + cs1.start_call(1, CallSiteId::new(fid1, LineNumber(1))); let id4 = cs1.id_for_new_allocation(1, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ())); assert_ne!(id4, id0); @@ -1063,9 +1098,9 @@ mod tests { let mut tracker = new_tracker(); let mut cs1 = Callstack::new(); - cs1.start_call(0, CallSiteId::new(fid1, 2)); + cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2))); let mut cs2 = Callstack::new(); - cs2.start_call(0, CallSiteId::new(fid3, 4)); + cs2.start_call(0, CallSiteId::new(fid3, LineNumber(4))); let cs1_id = tracker.get_callstack_id(&cs1); @@ -1156,12 +1191,12 @@ mod tests { .functions .add_function("c".to_string(), "cf".to_string()); - let id1 = CallSiteId::new(fid1, 1); + let id1 = CallSiteId::new(fid1, LineNumber(1)); // Same function, different line number—should be different item: - let id1_different = CallSiteId::new(fid1, 7); - let id2 = CallSiteId::new(fid2, 2); + let id1_different = CallSiteId::new(fid1, LineNumber(7)); + let id2 = CallSiteId::new(fid2, LineNumber(2)); - let id3 = CallSiteId::new(fid3, 3); + let id3 = CallSiteId::new(fid3, LineNumber(3)); let mut cs1 = Callstack::new(); cs1.start_call(0, id1); cs1.start_call(0, id2.clone()); @@ -1200,7 +1235,7 @@ mod tests { "c:3 (cf) 234", "a:7 (af);b:2 (bf) 6000", ]; - let mut result2: Vec = tracker.to_lines(true, false).collect(); + let mut result2: Vec = tracker.to_lines(true, false, same_callstack).collect(); result2.sort(); expected2.sort(); assert_eq!(expected2, result2);