Skip to content

Sciagraph support dec 2022 #463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 5, 2022
Merged

Sciagraph support dec 2022 #463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c7cef77
Allow store bytecode indexes in addition to line numbers.
pythonspeed Dec 2, 2022
9fde4ab
Allow post-processing of the callstack.
pythonspeed Dec 3, 2022
b882c00
Expose more functionality.
pythonspeed Dec 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions filpreload/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#![deny(unsafe_op_in_unsafe_fn)]
use parking_lot::Mutex;
use pymemprofile_api::memorytracking::LineNumberInfo::LineNumber;
use pymemprofile_api::memorytracking::{
AllocationTracker, CallSiteId, Callstack, FunctionId, VecFunctionLocations, PARENT_PROCESS,
};
Expand Down Expand Up @@ -57,8 +58,10 @@ fn add_function(filename: String, function_name: String) -> FunctionId {
/// Add to per-thread function stack:
fn start_call(call_site: FunctionId, parent_line_number: u16, line_number: u16) {
THREAD_CALLSTACK.with(|cs| {
cs.borrow_mut()
.start_call(parent_line_number, CallSiteId::new(call_site, line_number));
cs.borrow_mut().start_call(
parent_line_number as u32,
CallSiteId::new(call_site, LineNumber(line_number as u32)),
);
});
}

Expand Down Expand Up @@ -136,7 +139,7 @@ fn add_allocation(
// Will fail during thread shutdown, but not much we can do at that point.
let callstack_id = THREAD_CALLSTACK.try_with(|tcs| {
let mut callstack = tcs.borrow_mut();
callstack.id_for_new_allocation(line_number, |callstack| {
callstack.id_for_new_allocation(line_number as u32, |callstack| {
allocations.get_callstack_id(callstack)
})
})?;
Expand Down
111 changes: 73 additions & 38 deletions memapi/src/memorytracking.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,19 +82,35 @@ impl FunctionLocations for VecFunctionLocations {
}
}

pub type LineNumber = u16; // TODO u32, newtype
/// Either the line number, or the bytecode index needed to get it.
#[derive(Copy, Clone, Serialize, Deserialize, Hash, Eq, PartialEq, Debug)]
pub enum LineNumberInfo {
LineNumber(u32),
BytecodeIndex(i32),
}

impl LineNumberInfo {
fn get_line_number(&self) -> u32 {
if let LineNumberInfo::LineNumber(lineno) = self {
*lineno
} else {
debug_assert!(false, "This should never happen.");
0
}
}
}

/// A specific location: file + function + line number.
#[derive(Clone, Debug, PartialEq, Eq, Copy, Hash, Serialize, Deserialize)]
pub struct CallSiteId {
/// The function + filename. We use IDs for performance reasons (faster hashing).
function: FunctionId,
pub function: FunctionId,
/// Line number within the _file_, 1-indexed.
line_number: LineNumber,
pub line_number: LineNumberInfo,
}

impl CallSiteId {
pub fn new(function: FunctionId, line_number: u16) -> CallSiteId {
pub fn new(function: FunctionId, line_number: LineNumberInfo) -> CallSiteId {
CallSiteId {
function,
line_number,
Expand All @@ -108,7 +124,7 @@ impl CallSiteId {
pub struct Callstack {
calls: Vec<CallSiteId>,
#[derivative(Hash = "ignore", PartialEq = "ignore")]
cached_callstack_id: Option<(u16, CallstackId)>, // first bit is line number
cached_callstack_id: Option<(u32, CallstackId)>, // first bit is line number
}

impl Callstack {
Expand All @@ -126,10 +142,14 @@ impl Callstack {
}
}

pub fn start_call(&mut self, parent_line_number: u16, callsite_id: CallSiteId) {
pub fn to_vec(&self) -> Vec<CallSiteId> {
self.calls.clone()
}

pub fn start_call(&mut self, parent_line_number: u32, callsite_id: CallSiteId) {
if parent_line_number != 0 {
if let Some(mut call) = self.calls.last_mut() {
call.line_number = parent_line_number;
call.line_number = LineNumberInfo::LineNumber(parent_line_number);
}
}
self.calls.push(callsite_id);
Expand All @@ -141,7 +161,7 @@ impl Callstack {
self.cached_callstack_id = None;
}

pub fn id_for_new_allocation<F>(&mut self, line_number: u16, get_callstack_id: F) -> CallstackId
pub fn id_for_new_allocation<F>(&mut self, line_number: u32, get_callstack_id: F) -> CallstackId
where
F: FnOnce(&Callstack) -> CallstackId,
{
Expand All @@ -156,7 +176,7 @@ impl Callstack {
// Set the new line number:
if line_number != 0 {
if let Some(call) = self.calls.last_mut() {
call.line_number = line_number;
call.line_number = LineNumberInfo::LineNumber(line_number);
}
}

Expand Down Expand Up @@ -194,7 +214,8 @@ impl Callstack {
.map(|(id, (function, filename))| {
if to_be_post_processed {
// Get Python code.
let code = linecache.get_source_line(filename, id.line_number as usize);
let code = linecache
.get_source_line(filename, id.line_number.get_line_number() as usize);
// Leading whitespace is dropped by SVG, so we'd like to
// replace it with non-breaking space. However, inferno
// trims whitespace
Expand All @@ -215,15 +236,15 @@ impl Callstack {
format!(
"{filename}:{line} ({function});\u{2800}{code}",
filename = filename,
line = id.line_number,
line = id.line_number.get_line_number(),
function = function,
code = &code.trim_end(),
)
} else {
format!(
"{filename}:{line} ({function})",
filename = filename,
line = id.line_number,
line = id.line_number.get_line_number(),
function = function,
)
}
Expand Down Expand Up @@ -335,6 +356,10 @@ impl Allocation {
}
}

fn same_callstack(callstack: &Callstack) -> Cow<Callstack> {
Cow::Borrowed(callstack)
}

/// The main data structure tracking everything.
pub struct AllocationTracker<FL: FunctionLocations> {
// malloc()/calloc():
Expand Down Expand Up @@ -607,18 +632,22 @@ impl<FL: FunctionLocations> AllocationTracker<FL> {
self.dump_to_flamegraph(path, true, "peak-memory", "Peak Tracked Memory Usage", true);
}

pub fn to_lines(
&self,
pub fn to_lines<'a, F>(
&'a self,
peak: bool,
to_be_post_processed: bool,
) -> impl ExactSizeIterator<Item = String> + '_ {
update_callstack: F,
) -> impl ExactSizeIterator<Item = String> + '_
where
F: Fn(&Callstack) -> Cow<Callstack> + 'a,
{
let by_call = self.combine_callstacks(peak).into_iter();
let id_to_callstack = self.interner.get_reverse_map();
let mut linecache = LineCacher::default();
by_call.map(move |(callstack_id, size)| {
format!(
"{} {}",
id_to_callstack.get(&callstack_id).unwrap().as_string(
update_callstack(id_to_callstack.get(&callstack_id).unwrap()).as_string(
to_be_post_processed,
&self.functions,
";",
Expand Down Expand Up @@ -675,7 +704,7 @@ impl<FL: FunctionLocations> AllocationTracker<FL> {
subtitle,
"bytes",
to_be_post_processed,
|tbpp| self.to_lines(peak, tbpp),
|tbpp| self.to_lines(peak, tbpp, same_callstack),
)
}

Expand Down Expand Up @@ -746,8 +775,9 @@ impl<FL: FunctionLocations> AllocationTracker<FL> {

#[cfg(test)]
mod tests {
use crate::memorytracking::{ProcessUid, PARENT_PROCESS};
use crate::memorytracking::{same_callstack, ProcessUid, PARENT_PROCESS};

use super::LineNumberInfo::LineNumber;
use super::{
Allocation, AllocationTracker, CallSiteId, Callstack, CallstackInterner, FunctionId,
FunctionLocations, VecFunctionLocations, HIGH_32BIT, MIB,
Expand Down Expand Up @@ -815,7 +845,7 @@ mod tests {
let (process, allocation_size) = *allocated_sizes.get(i).unwrap();
let process = ProcessUid(process);
let mut cs = Callstack::new();
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0));
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0)));
let cs_id = tracker.get_callstack_id(&cs);
tracker.add_allocation(process, i as usize, allocation_size, cs_id);
expected_memory_usage.push_back(allocation_size);
Expand Down Expand Up @@ -854,7 +884,7 @@ mod tests {
let (process, allocation_size) = *allocated_sizes.get(i).unwrap();
let process = ProcessUid(process);
let mut cs = Callstack::new();
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0));
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0)));
let csid = tracker.get_callstack_id(&cs);
tracker.add_anon_mmap(process, addresses[i] as usize, allocation_size, csid);
expected_memory_usage.push_back(allocation_size);
Expand Down Expand Up @@ -891,7 +921,7 @@ mod tests {
let (process, allocation_size) = *allocated_sizes.get(i).unwrap();
let process = ProcessUid(process);
let mut cs = Callstack::new();
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0));
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0)));
let cs_id = tracker.get_callstack_id(&cs);
tracker.add_allocation(process, i as usize, allocation_size, cs_id);
expected_memory_usage += allocation_size;
Expand All @@ -900,7 +930,7 @@ mod tests {
let (process, allocation_size) = *allocated_mmaps.get(i).unwrap();
let process = ProcessUid(process);
let mut cs = Callstack::new();
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), 0));
cs.start_call(0, CallSiteId::new(FunctionId::new(i as u64), LineNumber(0)));
let csid = tracker.get_callstack_id(&cs);
tracker.add_anon_mmap(process, mmap_addresses[i] as usize, allocation_size, csid);
expected_memory_usage += allocation_size;
Expand Down Expand Up @@ -933,9 +963,10 @@ mod tests {

// Parent line number does nothing if it's first call:
let mut cs1 = Callstack::new();
let id1 = CallSiteId::new(fid1, 2);
let id2 = CallSiteId::new(fid3, 45);
let id3 = CallSiteId::new(fid5, 6);
// CallSiteId::new($a, $b) ==>> CallSiteId::new($a, LineNumber($b))
let id1 = CallSiteId::new(fid1, LineNumber(2));
let id2 = CallSiteId::new(fid3, LineNumber(45));
let id3 = CallSiteId::new(fid5, LineNumber(6));
cs1.start_call(123, id1);
assert_eq!(cs1.calls, vec![id1]);

Expand All @@ -950,7 +981,11 @@ mod tests {
cs2.start_call(12, id3);
assert_eq!(
cs2.calls,
vec![CallSiteId::new(fid1, 10), CallSiteId::new(fid3, 12), id3]
vec![
CallSiteId::new(fid1, LineNumber(10)),
CallSiteId::new(fid3, LineNumber(12)),
id3
]
);
}

Expand All @@ -960,10 +995,10 @@ mod tests {
let fid3 = FunctionId::new(3u64);

let mut cs1 = Callstack::new();
cs1.start_call(0, CallSiteId::new(fid1, 2));
cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2)));
let cs1b = cs1.clone();
let mut cs2 = Callstack::new();
cs2.start_call(0, CallSiteId::new(fid3, 4));
cs2.start_call(0, CallSiteId::new(fid3, LineNumber(4)));
let cs3 = Callstack::new();
let cs3b = Callstack::new();

Expand Down Expand Up @@ -1014,7 +1049,7 @@ mod tests {

let fid1 = FunctionId::new(1u64);

cs1.start_call(0, CallSiteId::new(fid1, 2));
cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2)));
let id1 =
cs1.id_for_new_allocation(1, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ()));
let id2 =
Expand All @@ -1025,7 +1060,7 @@ mod tests {
assert_ne!(id2, id0);
assert_ne!(id2, id1);

cs1.start_call(3, CallSiteId::new(fid1, 2));
cs1.start_call(3, CallSiteId::new(fid1, LineNumber(2)));
let id3 =
cs1.id_for_new_allocation(4, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ()));
assert_ne!(id3, id0);
Expand All @@ -1041,7 +1076,7 @@ mod tests {
assert_eq!(id1, id1c);

// Check for cache invalidation in start_call:
cs1.start_call(1, CallSiteId::new(fid1, 1));
cs1.start_call(1, CallSiteId::new(fid1, LineNumber(1)));
let id4 =
cs1.id_for_new_allocation(1, |cs| interner.get_or_insert_id(Cow::Borrowed(&cs), || ()));
assert_ne!(id4, id0);
Expand All @@ -1063,9 +1098,9 @@ mod tests {

let mut tracker = new_tracker();
let mut cs1 = Callstack::new();
cs1.start_call(0, CallSiteId::new(fid1, 2));
cs1.start_call(0, CallSiteId::new(fid1, LineNumber(2)));
let mut cs2 = Callstack::new();
cs2.start_call(0, CallSiteId::new(fid3, 4));
cs2.start_call(0, CallSiteId::new(fid3, LineNumber(4)));

let cs1_id = tracker.get_callstack_id(&cs1);

Expand Down Expand Up @@ -1156,12 +1191,12 @@ mod tests {
.functions
.add_function("c".to_string(), "cf".to_string());

let id1 = CallSiteId::new(fid1, 1);
let id1 = CallSiteId::new(fid1, LineNumber(1));
// Same function, different line number—should be different item:
let id1_different = CallSiteId::new(fid1, 7);
let id2 = CallSiteId::new(fid2, 2);
let id1_different = CallSiteId::new(fid1, LineNumber(7));
let id2 = CallSiteId::new(fid2, LineNumber(2));

let id3 = CallSiteId::new(fid3, 3);
let id3 = CallSiteId::new(fid3, LineNumber(3));
let mut cs1 = Callstack::new();
cs1.start_call(0, id1);
cs1.start_call(0, id2.clone());
Expand Down Expand Up @@ -1200,7 +1235,7 @@ mod tests {
"c:3 (cf) 234",
"a:7 (af);b:2 (bf) 6000",
];
let mut result2: Vec<String> = tracker.to_lines(true, false).collect();
let mut result2: Vec<String> = tracker.to_lines(true, false, same_callstack).collect();
result2.sort();
expected2.sort();
assert_eq!(expected2, result2);
Expand Down