From 7fb49dc062aa82dfe324d4bd8020a95e8dbf7d00 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Ricau Date: Fri, 19 Apr 2024 16:43:19 -0700 Subject: [PATCH] Report retained size --- .../java/shark/AndroidObjectSizeCalculator.kt | 16 ++ .../src/main/java/shark/DominatorTree.kt | 18 +- .../shark/HeapGraphObjectGrowthDetector.kt | 154 +++++++++++++----- .../src/main/java/shark/ObjectDominators.kt | 11 +- .../main/java/shark/RealLeakTracerFactory.kt | 12 +- shark/shark/src/main/java/shark/Reference.kt | 4 +- .../main/java/shark/ShortestPathObjectNode.kt | 38 +++++ 7 files changed, 186 insertions(+), 67 deletions(-) create mode 100644 shark/shark/src/main/java/shark/AndroidObjectSizeCalculator.kt diff --git a/shark/shark/src/main/java/shark/AndroidObjectSizeCalculator.kt b/shark/shark/src/main/java/shark/AndroidObjectSizeCalculator.kt new file mode 100644 index 0000000000..81f80aba55 --- /dev/null +++ b/shark/shark/src/main/java/shark/AndroidObjectSizeCalculator.kt @@ -0,0 +1,16 @@ +package shark + +import shark.DominatorTree.ObjectSizeCalculator +import shark.internal.ShallowSizeCalculator + +class AndroidObjectSizeCalculator(graph: HeapGraph) : ObjectSizeCalculator { + + private val nativeSizes = AndroidNativeSizeMapper(graph).mapNativeSizes() + private val shallowSizeCalculator = ShallowSizeCalculator(graph) + + override fun computeSize(objectId: Long): Int { + val nativeSize = nativeSizes[objectId] ?: 0 + val shallowSize = shallowSizeCalculator.computeShallowSize(objectId) + return nativeSize + shallowSize + } +} diff --git a/shark/shark/src/main/java/shark/DominatorTree.kt b/shark/shark/src/main/java/shark/DominatorTree.kt index e65351682c..c9182973e3 100644 --- a/shark/shark/src/main/java/shark/DominatorTree.kt +++ b/shark/shark/src/main/java/shark/DominatorTree.kt @@ -8,6 +8,10 @@ import shark.internal.hppc.LongScatterSet class DominatorTree(expectedElements: Int = 4) { + fun interface ObjectSizeCalculator { + fun computeSize(objectId: Long): Int + } + /** * Map of objects to their dominator. * @@ -16,6 +20,8 @@ class DominatorTree(expectedElements: Int = 4) { */ private val dominated = LongLongScatterMap(expectedElements) + operator fun contains(objectId: Long): Boolean = dominated.containsKey(objectId) + /** * Records that [objectId] is a root. */ @@ -98,7 +104,7 @@ class DominatorTree(expectedElements: Int = 4) { val dominated = mutableListOf() } - fun buildFullDominatorTree(computeSize: (Long) -> Int): Map { + fun buildFullDominatorTree(objectSizeCalculator: ObjectSizeCalculator): Map { val dominators = mutableMapOf() dominated.forEach(ForEachCallback {key, value -> // create entry for dominated @@ -115,7 +121,7 @@ class DominatorTree(expectedElements: Int = 4) { val allReachableObjectIds = dominators.keys.toSet() - ValueHolder.NULL_REFERENCE val retainedSizes = computeRetainedSizes(allReachableObjectIds) { objectId -> - val shallowSize = computeSize(objectId) + val shallowSize = objectSizeCalculator.computeSize(objectId) dominators.getValue(objectId).shallowSize = shallowSize shallowSize } @@ -147,12 +153,12 @@ class DominatorTree(expectedElements: Int = 4) { /** * Computes the size retained by [retainedObjectIds] using the dominator tree built using - * [updateDominatedAsRoot]. The shallow size of each object is provided by [computeSize]. + * [updateDominated]. The shallow size of each object is provided by [objectSizeCalculator]. * @return a map of object id to retained size. */ fun computeRetainedSizes( retainedObjectIds: Set, - computeSize: (Long) -> Int + objectSizeCalculator: ObjectSizeCalculator ): Map> { val nodeRetainedSizes = mutableMapOf>() retainedObjectIds.forEach { objectId -> @@ -169,7 +175,7 @@ class DominatorTree(expectedElements: Int = 4) { // If the entry is a node, add its size to nodeRetainedSizes nodeRetainedSizes[key]?.let { (currentRetainedSize, currentRetainedCount) -> - instanceSize = computeSize(key) + instanceSize = objectSizeCalculator.computeSize(key) nodeRetainedSizes[key] = currentRetainedSize + instanceSize to currentRetainedCount + 1 } @@ -186,7 +192,7 @@ class DominatorTree(expectedElements: Int = 4) { dominated[objectId] = dominator } if (instanceSize == -1) { - instanceSize = computeSize(key) + instanceSize = objectSizeCalculator.computeSize(key) } // Update retained size for that node val (currentRetainedSize, currentRetainedCount) = nodeRetainedSizes.getValue( diff --git a/shark/shark/src/main/java/shark/HeapGraphObjectGrowthDetector.kt b/shark/shark/src/main/java/shark/HeapGraphObjectGrowthDetector.kt index 13be35a5ae..9bf361734f 100644 --- a/shark/shark/src/main/java/shark/HeapGraphObjectGrowthDetector.kt +++ b/shark/shark/src/main/java/shark/HeapGraphObjectGrowthDetector.kt @@ -4,12 +4,14 @@ package shark import java.util.ArrayDeque import java.util.Deque +import shark.ByteSize.Companion.bytes import shark.HeapObject.HeapClass import shark.HeapObject.HeapInstance import shark.HeapObject.HeapObjectArray import shark.HeapObject.HeapPrimitiveArray import shark.ReferenceLocationType.ARRAY_ENTRY import shark.ReferenceReader.Factory +import shark.ShortestPathObjectNode.Retained import shark.internal.hppc.LongScatterSet class HeapGraphObjectGrowthDetector( @@ -20,15 +22,29 @@ class HeapGraphObjectGrowthDetector( fun findGrowingObjects( heapGraph: CloseableHeapGraph, scenarioLoops: Int, - previousTraversal: InputHeapTraversal = NoHeapTraversalYet, + previousTraversal: InputHeapTraversal = NoHeapTraversalYet ): HeapTraversal { - val state = TraversalState() + val computeRetainedHeapSize = previousTraversal !is NoHeapTraversalYet + // Estimate of how many objects we'll visit. This is a conservative estimate, we should always + // visit more than that but this limits the number of early array growths. + val estimatedVisitedObjects = (heapGraph.instanceCount / 2).coerceAtLeast(4) + val state = TraversalState( + estimatedVisitedObjects = estimatedVisitedObjects, + computeRetainedHeapSize = computeRetainedHeapSize + ) return heapGraph.use { - state.traverseHeapDiffingShortestPaths(heapGraph, scenarioLoops, previousTraversal) + state.traverseHeapDiffingShortestPaths( + heapGraph, + scenarioLoops, + previousTraversal + ) } } - private class TraversalState { + private class TraversalState( + estimatedVisitedObjects: Int, + computeRetainedHeapSize: Boolean + ) { var visitingLast = false /** Set of objects to visit */ @@ -39,7 +55,9 @@ class HeapGraphObjectGrowthDetector( */ val toVisitLastQueue: Deque = ArrayDeque() - val visitedSet = LongScatterSet() + val visitedSet = LongScatterSet(estimatedVisitedObjects) + val dominatorTree = + if (computeRetainedHeapSize) DominatorTree(estimatedVisitedObjects) else null val tree = ShortestPathObjectNode("root", null, newNode = false).apply { selfObjectCount = 1 @@ -52,7 +70,7 @@ class HeapGraphObjectGrowthDetector( private fun TraversalState.traverseHeapDiffingShortestPaths( graph: CloseableHeapGraph, detectedGrowth: Int, - previousTraversal: InputHeapTraversal, + previousTraversal: InputHeapTraversal ): HeapTraversal { // First iteration, all nodes are growing. @@ -94,6 +112,7 @@ class HeapGraphObjectGrowthDetector( val isLeafObject: Boolean ) + // Note: this is different from visitedSet.size(), which includes gc roots. var visitedObjectCount = 0 val edges = node.objectIds.flatMap { objectId -> @@ -106,33 +125,41 @@ class HeapGraphObjectGrowthDetector( if (node.isLeafObject) { emptySequence() } else { - val heapObject = graph.findObjectById(objectId) - val refs = objectReferenceReader.read(heapObject) - refs.mapNotNull { reference -> - if (reference.valueObjectId in visitedSet) { - null - } else { - val details = reference.lazyDetailsResolver.resolve() - val refType = details.locationType.name - val owningClassSimpleName = - graph.findObjectById(details.locationClassObjectId).asClass!!.simpleName - val refName = if (details.locationType == ARRAY_ENTRY) "[x]" else details.name - val referencedObjectName = - when (val referencedObject = graph.findObjectById(reference.valueObjectId)) { - is HeapClass -> "class ${referencedObject.name}" - is HeapInstance -> "instance of ${referencedObject.instanceClassName}" - is HeapObjectArray -> "array of ${referencedObject.arrayClassName}" - is HeapPrimitiveArray -> "array of ${referencedObject.primitiveType.name.lowercase()}" - } - val nodeAndEdgeName = - "$refType ${owningClassSimpleName}.${refName} -> $referencedObjectName" - ExpandedObject( - reference.valueObjectId, nodeAndEdgeName, reference.isLowPriority, - reference.isLeafObject + val heapObject = graph.findObjectById(objectId) + val refs = objectReferenceReader.read(heapObject) + refs.mapNotNull { reference -> + // dominatorTree is updated prior to enqueueing, because that's where we have the + // parent object id information. visitedSet is updated on dequeuing, because bumping + // node priority would be complex when as we'd need to move object ids between nodes + // rather than just move nodes. + dominatorTree?.updateDominated( + objectId = reference.valueObjectId, + parentObjectId = objectId ) + if (reference.valueObjectId in visitedSet) { + null + } else { + val details = reference.lazyDetailsResolver.resolve() + val refType = details.locationType.name + val owningClassSimpleName = + graph.findObjectById(details.locationClassObjectId).asClass!!.simpleName + val refName = if (details.locationType == ARRAY_ENTRY) "[x]" else details.name + val referencedObjectName = + when (val referencedObject = graph.findObjectById(reference.valueObjectId)) { + is HeapClass -> "class ${referencedObject.name}" + is HeapInstance -> "instance of ${referencedObject.instanceClassName}" + is HeapObjectArray -> "array of ${referencedObject.arrayClassName}" + is HeapPrimitiveArray -> "array of ${referencedObject.primitiveType.name.lowercase()}" + } + val nodeAndEdgeName = + "$refType ${owningClassSimpleName}.${refName} -> $referencedObjectName" + ExpandedObject( + reference.valueObjectId, nodeAndEdgeName, reference.isLowPriority, + reference.isLeafObject + ) + } } } - } } }.groupBy { it.nodeAndEdgeName + if (it.isLowPriority) "low-priority" else "" @@ -183,7 +210,8 @@ class HeapGraphObjectGrowthDetector( } val growingNodes = if (previousTree != null) { - nodesMaybeGrowing.mapNotNull { node -> + val growingNodePairs = mutableListOf>() + val growingNodes = nodesMaybeGrowing.mapNotNull { node -> val shortestPathNode = node.shortestPathNode val growing = if (node.previousPathNode != null) { // Existing node. Growing if was growing (already true) and edges increased at least detectedGrowth. @@ -216,14 +244,56 @@ class HeapGraphObjectGrowthDetector( child.selfObjectCountIncrease = child.selfObjectCount } } - // Mark as growing in the tree(useful for next iteration) + // Mark as growing in the tree (useful for next iteration) shortestPathNode.growing = true + + val previouslyGrowing = !shortestPathNode.newNode + val parentAlreadyReported = (shortestPathNode.parent?.growing) ?: false + + val repeatedlyGrowingNode = previouslyGrowing && !parentAlreadyReported // Return in list of growing nodes. - shortestPathNode + if (repeatedlyGrowingNode) { + if (dominatorTree != null) { + growingNodePairs += node to shortestPathNode + } + shortestPathNode + } else { + null + } } else { null } } + dominatorTree?.let { dominatorTree -> + val growingNodeObjectIds = growingNodePairs.flatMapTo(LinkedHashSet()) { (node, _) -> + node.objectIds + } + val objectSizeCalculator = AndroidObjectSizeCalculator(graph) + val retainedMap = + dominatorTree.computeRetainedSizes(growingNodeObjectIds, objectSizeCalculator) + growingNodePairs.forEach { (node, shortestPathNode) -> + var heapSize = ByteSize.ZERO + var objectCount = 0 + for (objectId in node.objectIds) { + val (additionalByteSize, additionalObjectCount) = retainedMap.getValue(objectId) + heapSize += additionalByteSize.bytes + objectCount += additionalObjectCount + } + shortestPathNode.retainedOrNull = Retained( + heapSize = heapSize, + objectCount = objectCount + ) + val previousRetained = node.previousPathNode?.retainedOrNull + shortestPathNode.retainedIncreaseOrNull = if (previousRetained == null) { + Retained(ByteSize.ZERO, 0) + } else { + Retained( + heapSize - previousRetained.heapSize, objectCount - previousRetained.objectCount + ) + } + } + } + growingNodes } else { null } @@ -233,13 +303,7 @@ class HeapGraphObjectGrowthDetector( InitialHeapTraversal(tree) } else { check(previousTraversal !is NoHeapTraversalYet) - val repeatedlyGrowingNodes = - growingNodes.filter { - val previouslyGrowing = !it.newNode - val parentAlreadyReported = (it.parent?.growing) ?: false - previouslyGrowing && !parentAlreadyReported - } - HeapTraversalWithDiff(tree, repeatedlyGrowingNodes) + HeapTraversalWithDiff(tree, growingNodes) } } @@ -276,10 +340,17 @@ class HeapGraphObjectGrowthDetector( } else { null } + val objectIds = gcRootReferences.map { it.second.gcRoot.id } + dominatorTree?.let { + objectIds.forEach { objectId -> + it.updateDominatedAsRoot(objectId) + } + } + enqueue( parentPathNode = tree, previousPathNode = previousPathNode, - objectIds = gcRootReferences.map { it.second.gcRoot.id }, + objectIds = objectIds, nodeAndEdgeName = nodeAndEdgeName, isLowPriority = firstOfGroup.second.isLowPriority, isLeafObject = false @@ -296,6 +367,7 @@ class HeapGraphObjectGrowthDetector( isLeafObject: Boolean ) { // TODO Maybe the filtering should happen at the callsite. + // TODO we already filter visited on the traversal side. maybe crash? val filteredObjectIds = objectIds.filter { objectId -> objectId != ValueHolder.NULL_REFERENCE && // note: we only update visitedSet once dequeued. This could lead diff --git a/shark/shark/src/main/java/shark/ObjectDominators.kt b/shark/shark/src/main/java/shark/ObjectDominators.kt index 3424f71785..4e492475b2 100644 --- a/shark/shark/src/main/java/shark/ObjectDominators.kt +++ b/shark/shark/src/main/java/shark/ObjectDominators.kt @@ -6,7 +6,6 @@ import shark.HeapObject.HeapClass import shark.HeapObject.HeapInstance import shark.HeapObject.HeapObjectArray import shark.HeapObject.HeapPrimitiveArray -import shark.internal.ShallowSizeCalculator /** * Exposes high level APIs to compute and render a dominator tree. This class @@ -169,15 +168,9 @@ class ObjectDominators { computeRetainedHeapSize = true, ).createFor(graph) - val nativeSizeMapper = AndroidNativeSizeMapper(graph) - val nativeSizes = nativeSizeMapper.mapNativeSizes() - val shallowSizeCalculator = ShallowSizeCalculator(graph) + val objectSizeCalculator = AndroidObjectSizeCalculator(graph) val result = pathFinder.findShortestPathsFromGcRoots(setOf()) - return result.dominatorTree!!.buildFullDominatorTree { objectId -> - val nativeSize = nativeSizes[objectId] ?: 0 - val shallowSize = shallowSizeCalculator.computeShallowSize(objectId) - nativeSize + shallowSize - } + return result.dominatorTree!!.buildFullDominatorTree(objectSizeCalculator) } } diff --git a/shark/shark/src/main/java/shark/RealLeakTracerFactory.kt b/shark/shark/src/main/java/shark/RealLeakTracerFactory.kt index 952cdd6dcf..42e47db840 100644 --- a/shark/shark/src/main/java/shark/RealLeakTracerFactory.kt +++ b/shark/shark/src/main/java/shark/RealLeakTracerFactory.kt @@ -57,6 +57,7 @@ class RealLeakTracerFactory constructor( sealed interface Event { object StartedBuildingLeakTraces : Event object StartedInspectingObjects : Event + @Deprecated("Event not sent anymore") object StartedComputingNativeRetainedSize: Event object StartedComputingJavaHeapRetainedSize: Event @@ -350,16 +351,9 @@ class RealLeakTracerFactory constructor( inspectedObjects.filter { it.leakingStatus == UNKNOWN || it.leakingStatus == LEAKING } .map { it.heapObject.objectId } }.toSet() - listener.onEvent(StartedComputingNativeRetainedSize) - val nativeSizeMapper = AndroidNativeSizeMapper(graph) - val nativeSizes = nativeSizeMapper.mapNativeSizes() listener.onEvent(StartedComputingJavaHeapRetainedSize) - val shallowSizeCalculator = ShallowSizeCalculator(graph) - return dominatorTree.computeRetainedSizes(nodeObjectIds) { objectId -> - val nativeSize = nativeSizes[objectId] ?: 0 - val shallowSize = shallowSizeCalculator.computeShallowSize(objectId) - nativeSize + shallowSize - } + val objectSizeCalculator = AndroidObjectSizeCalculator(graph) + return dominatorTree.computeRetainedSizes(nodeObjectIds, objectSizeCalculator) } private fun buildLeakTraceObjects( diff --git a/shark/shark/src/main/java/shark/Reference.kt b/shark/shark/src/main/java/shark/Reference.kt index 4c9eee86ed..b2d7163666 100644 --- a/shark/shark/src/main/java/shark/Reference.kt +++ b/shark/shark/src/main/java/shark/Reference.kt @@ -24,8 +24,8 @@ class Reference( // TODO Leverage this on the leakcanary side. /** - * Whether this object should be treated as a leaf object with no outgoing references (regardless - * of its actual content). + * Whether this object should be treated as a leaf / sink object with no outgoing references + * (regardless of its actual content). */ val isLeafObject: Boolean = false, diff --git a/shark/shark/src/main/java/shark/ShortestPathObjectNode.kt b/shark/shark/src/main/java/shark/ShortestPathObjectNode.kt index d951ef56aa..92d14119e8 100644 --- a/shark/shark/src/main/java/shark/ShortestPathObjectNode.kt +++ b/shark/shark/src/main/java/shark/ShortestPathObjectNode.kt @@ -17,6 +17,26 @@ class ShortestPathObjectNode( var selfObjectCountIncrease = 0 internal set + /** + * Set for growing nodes if the traversal requested the computation of retained sizes, otherwise + * null. + * This is on the last 2 traversals. + */ + var retainedOrNull: Retained? = null + internal set + + /** + * Set for growing nodes if [retainedOrNull] is not null. Non 0 if the previous traversal also + * computed retained size. + * This is on the last 2 traversals. + */ + var retainedIncreaseOrNull: Retained? = null + internal set + + val retained: Retained get() = retainedOrNull!! + + val retainedIncrease: Retained get() = retainedIncreaseOrNull!! + internal var growing = false val childrenObjectCount: Int @@ -50,6 +70,10 @@ class ShortestPathObjectNode( result.append(pathNode.selfObjectCount) result.append(" objects)") if (index == pathAfterRoot.lastIndex) { + result.appendLine() + result.append(" Retained size: ${retained.heapSize} (+ ${retainedIncrease.heapSize})") + result.appendLine() + result.append(" Retained objects: ${retained.objectCount} (+ ${retainedIncrease.objectCount})") result.appendLine() result.append(" Children:") result.appendLine() @@ -72,4 +96,18 @@ class ShortestPathObjectNode( } return result.toString() } + + class Retained( + /** + * The minimum number of bytes which would be freed if all references to this object were + * released. + */ + val heapSize: ByteSize, + + /** + * The minimum number of objects which would be unreachable if all references to this object were + * released. + */ + val objectCount: Int, + ) }