Add a kind field to memory reports.

This is used for two memory reporting improvements.

- It's used to distinguish "explicit" memory reports from others. This
  mirrors the same categorization that is used in Firefox, and gives a single
  tree that's the best place to look. It replaces the "pages" tree which
  was always intended to be a temporary stand-in for "explicit".

- It's used to computed "heap-unclassified" values for both the jemalloc
  and system heaps, both of which are placed into the "explicit" tree.

Example output:
```
|  114.99 MiB -- explicit
|      52.34 MiB -- jemalloc-heap-unclassified
|      46.14 MiB -- system-heap-unclassified
|      14.95 MiB -- url(file:///home/njn/moz/servo2/../servo-static-suite/wikipe
dia/Guardians%20of%20the%20Galaxy%20(film)%20-%20Wikipedia,%20the%20free%20encyc
lopedia.html)
|          7.32 MiB -- js
|             3.07 MiB -- malloc-heap
|             3.00 MiB -- gc-heap
|                2.49 MiB -- used
|                0.34 MiB -- decommitted
|                0.09 MiB -- unused
|                0.09 MiB -- admin
|             1.25 MiB -- non-heap
|          1.36 MiB -- layout-worker-3-local-context
|          1.34 MiB -- layout-worker-0-local-context
|          1.24 MiB -- layout-worker-1-local-context
|          1.24 MiB -- layout-worker-4-local-context
|          1.16 MiB -- layout-worker-2-local-context
|          0.89 MiB -- layout-worker-5-local-context
|          0.38 MiB -- layout-task
|             0.31 MiB -- display-list
|             0.07 MiB -- local-context
|       1.56 MiB -- compositor-task
|          0.78 MiB -- surface-map
|          0.78 MiB -- layer-tree
```
The heap-unclassified values dominate the "explicit" tree because reporter
coverage is still quite poor.

components/compositing/compositor.rs

@@ -39,7 +39,7 @@ use msg::constellation_msg::{ConstellationChan, NavigationDirection};
 use msg::constellation_msg::{Key, KeyModifiers, KeyState, LoadData};
 use msg::constellation_msg::{PipelineId, WindowSizeData};
 use png;
-use profile_traits::mem::{self, Reporter, ReporterRequest};
+use profile_traits::mem::{self, Reporter, ReporterRequest, ReportKind};
 use profile_traits::time::{self, ProfilerCategory, profile};
 use script_traits::{ConstellationControlMsg, LayoutControlMsg, ScriptControlChan};
 use std::collections::HashMap;
@@ -501,11 +501,17 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             (Msg::CollectMemoryReports(reports_chan), ShutdownState::NotShuttingDown) => {
                 let mut reports = vec![];
                 let name = "compositor-task";
+                // These are both `ExplicitUnknownLocationSize` because the memory might be in the
+                // GPU or on the heap.
                 reports.push(mem::Report {
-                    path: path![name, "surface-map"], size: self.surface_map.mem(),
+                    path: path![name, "surface-map"],
+                    kind: ReportKind::ExplicitUnknownLocationSize,
+                    size: self.surface_map.mem(),
                 });
                 reports.push(mem::Report {
-                    path: path![name, "layer-tree"], size: self.scene.get_memory_usage(),
+                    path: path![name, "layer-tree"],
+                    kind: ReportKind::ExplicitUnknownLocationSize,
+                    size: self.scene.get_memory_usage(),
                 });
                 reports_chan.send(reports);
             }

components/layout/layout_task.rs

@@ -45,7 +45,7 @@ use log;
 use msg::compositor_msg::{Epoch, ScrollPolicy, LayerId};
 use msg::constellation_msg::Msg as ConstellationMsg;
 use msg::constellation_msg::{ConstellationChan, Failure, PipelineExitType, PipelineId};
-use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportsChan};
+use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportKind, ReportsChan};
 use profile_traits::time::{self, ProfilerMetadata, profile};
 use profile_traits::time::{TimerMetadataFrameType, TimerMetadataReflowType};
 use net_traits::{load_bytes_iter, PendingAsyncLoad};
@@ -600,13 +600,15 @@ impl LayoutTask {
         let rw_data = self.lock_rw_data(possibly_locked_rw_data);
         let stacking_context = rw_data.stacking_context.as_ref();
         reports.push(Report {
-            path: path!["pages", format!("url({})", self.url), "layout-task", "display-list"],
+            path: path![format!("url({})", self.url), "layout-task", "display-list"],
+            kind: ReportKind::ExplicitJemallocHeapSize,
             size: stacking_context.map_or(0, |sc| sc.heap_size_of_children()),
         });
 
         // The LayoutTask has a context in TLS...
         reports.push(Report {
-            path: path!["pages", format!("url({})", self.url), "layout-task", "local-context"],
+            path: path![format!("url({})", self.url), "layout-task", "local-context"],
+            kind: ReportKind::ExplicitJemallocHeapSize,
             size: heap_size_of_local_context(),
         });
 
@@ -615,9 +617,10 @@ impl LayoutTask {
             let sizes = traversal.heap_size_of_tls(heap_size_of_local_context);
             for (i, size) in sizes.iter().enumerate() {
                 reports.push(Report {
-                    path: path!["pages", format!("url({})", self.url),
+                    path: path![format!("url({})", self.url),
                                 format!("layout-worker-{}-local-context", i)],
-                    size: *size
+                    kind: ReportKind::ExplicitJemallocHeapSize,
+                    size: *size,
                 });
             }
         }

components/profile/mem.rs

@@ -6,7 +6,8 @@
 
 use ipc_channel::ipc::{self, IpcReceiver};
 use ipc_channel::router::ROUTER;
-use profile_traits::mem::{ProfilerChan, ProfilerMsg, Reporter, ReporterRequest, ReportsChan};
+use profile_traits::mem::{ProfilerChan, ProfilerMsg, Reporter, ReporterRequest, ReportKind};
+use profile_traits::mem::ReportsChan;
 use std::borrow::ToOwned;
 use std::cmp::Ordering;
 use std::collections::HashMap;
@@ -21,6 +22,9 @@ pub struct Profiler {
     reporters: HashMap<String, Reporter>,
 }
 
+const JEMALLOC_HEAP_ALLOCATED_STR: &'static str = "jemalloc-heap-allocated";
+const SYSTEM_HEAP_ALLOCATED_STR: &'static str = "system-heap-allocated";
+
 impl Profiler {
     pub fn create(period: Option<f64>) -> ProfilerChan {
         let (chan, port) = ipc::channel().unwrap();
@@ -122,16 +126,72 @@ impl Profiler {
         // each reporter once we have enough of them.
         //
         // If anything goes wrong with a reporter, we just skip it.
+        //
+        // We also track the total memory reported on the jemalloc heap and the system heap, and
+        // use that to compute the special "jemalloc-heap-unclassified" and
+        // "system-heap-unclassified" values.
+
         let mut forest = ReportsForest::new();
+
+        let mut jemalloc_heap_reported_size = 0;
+        let mut system_heap_reported_size = 0;
+
+        let mut jemalloc_heap_allocated_size: Option<usize> = None;
+        let mut system_heap_allocated_size: Option<usize> = None;
+
         for reporter in self.reporters.values() {
             let (chan, port) = ipc::channel().unwrap();
             reporter.collect_reports(ReportsChan(chan));
-            if let Ok(reports) = port.recv() {
-                for report in reports.iter() {
+            if let Ok(mut reports) = port.recv() {
+
+                for report in reports.iter_mut() {
+
+                    // Add "explicit" to the start of the path, when appropriate.
+                    match report.kind {
+                        ReportKind::ExplicitJemallocHeapSize |
+                        ReportKind::ExplicitSystemHeapSize |
+                        ReportKind::ExplicitNonHeapSize |
+                        ReportKind::ExplicitUnknownLocationSize =>
+                            report.path.insert(0, String::from("explicit")),
+                        ReportKind::NonExplicitSize => {},
+                    }
+
+                    // Update the reported fractions of the heaps, when appropriate.
+                    match report.kind {
+                        ReportKind::ExplicitJemallocHeapSize =>
+                            jemalloc_heap_reported_size += report.size,
+                        ReportKind::ExplicitSystemHeapSize =>
+                            system_heap_reported_size += report.size,
+                        _ => {},
+                    }
+
+                    // Record total size of the heaps, when we see them.
+                    if report.path.len() == 1 {
+                        if report.path[0] == JEMALLOC_HEAP_ALLOCATED_STR {
+                            assert!(jemalloc_heap_allocated_size.is_none());
+                            jemalloc_heap_allocated_size = Some(report.size);
+                        } else if report.path[0] == SYSTEM_HEAP_ALLOCATED_STR {
+                            assert!(system_heap_allocated_size.is_none());
+                            system_heap_allocated_size = Some(report.size);
+                        }
+                    }
+
+                    // Insert the report.
                     forest.insert(&report.path, report.size);
                 }
             }
         }
+
+        // Compute and insert the heap-unclassified values.
+        if let Some(jemalloc_heap_allocated_size) = jemalloc_heap_allocated_size {
+            forest.insert(&path!["explicit", "jemalloc-heap-unclassified"],
+                          jemalloc_heap_allocated_size - jemalloc_heap_reported_size);
+        }
+        if let Some(system_heap_allocated_size) = system_heap_allocated_size {
+            forest.insert(&path!["explicit", "system-heap-unclassified"],
+                          system_heap_allocated_size - system_heap_reported_size);
+        }
+
         forest.print();
 
         println!("|");
@@ -301,11 +361,12 @@ impl ReportsForest {
 
 mod system_reporter {
     use libc::{c_char, c_int, c_void, size_t};
-    use profile_traits::mem::{Report, ReporterRequest};
+    use profile_traits::mem::{Report, ReporterRequest, ReportKind};
     use std::borrow::ToOwned;
     use std::ffi::CString;
     use std::mem::size_of;
     use std::ptr::null_mut;
+    use super::{JEMALLOC_HEAP_ALLOCATED_STR, SYSTEM_HEAP_ALLOCATED_STR};
     #[cfg(target_os="macos")]
     use task_info::task_basic_info::{virtual_size, resident_size};
 
@@ -315,7 +376,11 @@ mod system_reporter {
         {
             let mut report = |path, size| {
                 if let Some(size) = size {
-                    reports.push(Report { path: path, size: size });
+                    reports.push(Report {
+                        path: path,
+                        kind: ReportKind::NonExplicitSize,
+                        size: size,
+                    });
                 }
             };
 
@@ -330,13 +395,13 @@ mod system_reporter {
 
             // Total number of bytes allocated by the application on the system
             // heap.
-            report(path!["system-heap-allocated"], get_system_heap_allocated());
+            report(path![SYSTEM_HEAP_ALLOCATED_STR], get_system_heap_allocated());
 
             // The descriptions of the following jemalloc measurements are taken
             // directly from the jemalloc documentation.
 
             // "Total number of bytes allocated by the application."
-            report(path!["jemalloc-heap-allocated"], get_jemalloc_stat("stats.allocated"));
+            report(path![JEMALLOC_HEAP_ALLOCATED_STR], get_jemalloc_stat("stats.allocated"));
 
             // "Total number of bytes in active pages allocated by the application.
             // This is a multiple of the page size, and greater than or equal to

components/profile_traits/mem.rs

@@ -23,12 +23,48 @@ impl ProfilerChan {
     }
 }
 
+/// The various kinds of memory measurement.
+///
+/// Here "explicit" means explicit memory allocations done by the application. It includes
+/// allocations made at the OS level (via functions such as VirtualAlloc, vm_allocate, and mmap),
+/// allocations made at the heap allocation level (via functions such as malloc, calloc, realloc,
+/// memalign, operator new, and operator new[]) and where possible, the overhead of the heap
+/// allocator itself. It excludes memory that is mapped implicitly such as code and data segments,
+/// and thread stacks. "explicit" is not guaranteed to cover every explicit allocation, but it does
+/// cover most (including the entire heap), and therefore it is the single best number to focus on
+/// when trying to reduce memory usage.
+#[derive(Deserialize, Serialize)]
+pub enum ReportKind {
+    /// A size measurement for an explicit allocation on the jemalloc heap. This should be used
+    /// for any measurements done via the `HeapSizeOf` trait.
+    ExplicitJemallocHeapSize,
+
+    /// A size measurement for an explicit allocation on the system heap. Only likely to be used
+    /// for external C or C++ libraries that don't use jemalloc.
+    ExplicitSystemHeapSize,
+
+    /// A size measurement for an explicit allocation not on the heap, e.g. via mmap().
+    ExplicitNonHeapSize,
+
+    /// A size measurement for an explicit allocation whose location is unknown or uncertain.
+    ExplicitUnknownLocationSize,
+
+    /// A size measurement for a non-explicit allocation. This kind is used for global
+    /// measurements such as "resident" and "vsize", and also for measurements that cross-cut the
+    /// measurements grouped under "explicit", e.g. by grouping those measurements in a way that's
+    /// different to how they are grouped under "explicit".
+    NonExplicitSize,
+}
+
 /// A single memory-related measurement.
 #[derive(Deserialize, Serialize)]
 pub struct Report {
     /// The identifying path for this report.
     pub path: Vec<String>,
 
+    /// The report kind.
+    pub kind: ReportKind,
+
     /// The size, in bytes.
     pub size: usize,
 }

components/script/script_task.rs

@@ -70,7 +70,7 @@ use net_traits::{ResourceTask, LoadConsumer, ControlMsg, Metadata};
 use net_traits::LoadData as NetLoadData;
 use net_traits::image_cache_task::{ImageCacheChan, ImageCacheTask, ImageCacheResult};
 use net_traits::storage_task::StorageTask;
-use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportsChan};
+use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportKind, ReportsChan};
 use string_cache::Atom;
 use util::str::DOMString;
 use util::task::spawn_named_with_send_on_failure;
@@ -1042,18 +1042,44 @@ impl ScriptTask {
             let rt = JS_GetRuntime(cx);
             let mut stats = ::std::mem::zeroed();
             if CollectServoSizes(rt, &mut stats) {
-                let mut report = |mut path_suffix, size| {
-                    let mut path = path!["pages", path_seg, "js"];
+                let mut report = |mut path_suffix, kind, size| {
+                    let mut path = path![path_seg, "js"];
                     path.append(&mut path_suffix);
-                    reports.push(Report { path: path, size: size as usize })
+                    reports.push(Report {
+                        path: path,
+                        kind: kind,
+                        size: size as usize,
+                    })
                 };
 
-                report(path!["gc-heap", "used"], stats.gcHeapUsed);
-                report(path!["gc-heap", "unused"], stats.gcHeapUnused);
-                report(path!["gc-heap", "admin"], stats.gcHeapAdmin);
-                report(path!["gc-heap", "decommitted"], stats.gcHeapDecommitted);
-                report(path!["malloc-heap"], stats.mallocHeap);
-                report(path!["non-heap"], stats.nonHeap);
+                // A note about possibly confusing terminology: the JS GC "heap" is allocated via
+                // mmap/VirtualAlloc, which means it's not on the malloc "heap", so we use
+                // `ExplicitNonHeapSize` as its kind.
+
+                report(path!["gc-heap", "used"],
+                       ReportKind::ExplicitNonHeapSize,
+                       stats.gcHeapUsed);
+
+                report(path!["gc-heap", "unused"],
+                       ReportKind::ExplicitNonHeapSize,
+                       stats.gcHeapUnused);
+
+                report(path!["gc-heap", "admin"],
+                       ReportKind::ExplicitNonHeapSize,
+                       stats.gcHeapAdmin);
+
+                report(path!["gc-heap", "decommitted"],
+                       ReportKind::ExplicitNonHeapSize,
+                       stats.gcHeapDecommitted);
+
+                // SpiderMonkey uses the system heap, not jemalloc.
+                report(path!["malloc-heap"],
+                       ReportKind::ExplicitSystemHeapSize,
+                       stats.mallocHeap);
+
+                report(path!["non-heap"],
+                       ReportKind::ExplicitNonHeapSize,
+                       stats.nonHeap);
             }
         }
         reports