Skip to content

Commit

Permalink
On-deman backend heap and thread dump (#8320)
Browse files Browse the repository at this point in the history
close #8249

Changelog:
- add: `profiling/snapshot` request that takes a heap dump of the language server and puts it in the `ENSO_DATA_DIRECTORY/profiling` direcotry
  • Loading branch information
4e6 committed Nov 20, 2023
1 parent b8ebed6 commit b224f95
Show file tree
Hide file tree
Showing 15 changed files with 284 additions and 5 deletions.
4 changes: 3 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,9 @@ lazy val `profiling-utils` = project
exclude ("org.netbeans.api", "org-openide-util-ui")
exclude ("org.netbeans.api", "org-openide-awt")
exclude ("org.netbeans.api", "org-openide-modules")
exclude ("org.netbeans.api", "org-netbeans-api-annotations-common")
exclude ("org.netbeans.api", "org-netbeans-api-annotations-common"),
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test
)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,15 @@ object ProfilingApi {
}
}

case object ProfilingSnapshot extends Method("profiling/snapshot") {

implicit val hasParams: HasParams.Aux[this.type, Unused.type] =
new HasParams[this.type] {
type Params = Unused.type
}
implicit val hasResult: HasResult.Aux[this.type, Unused.type] =
new HasResult[this.type] {
type Result = Unused.type
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import com.typesafe.scalalogging.LazyLogging
import org.enso.distribution.DistributionManager
import org.enso.languageserver.runtime.RuntimeConnector
import org.enso.languageserver.runtime.events.RuntimeEventsMonitor
import org.enso.logger.masking.MaskedPath
import org.enso.profiling.events.NoopEventsMonitor
import org.enso.profiling.sampler.{MethodsSampler, OutputStreamSampler}
import org.enso.profiling.snapshot.{HeapDumpSnapshot, ProfilingSnapshot}

import java.io.{ByteArrayOutputStream, PrintStream}
import java.nio.charset.StandardCharsets
Expand All @@ -21,11 +23,13 @@ import scala.util.{Failure, Success, Try}
*
* @param runtimeConnector the connection to runtime
* @param distributionManager the distribution manager
* @param profilingSnapshot the profiling snapshot generator
* @param clock the system clock
*/
final class ProfilingManager(
runtimeConnector: ActorRef,
distributionManager: DistributionManager,
profilingSnapshot: ProfilingSnapshot,
clock: Clock
) extends Actor
with LazyLogging {
Expand Down Expand Up @@ -71,7 +75,10 @@ final class ProfilingManager(
case Failure(exception) =>
logger.error("Failed to save the sampler's result.", exception)
case Success(samplesPath) =>
logger.trace("Saved the sampler's result to {}", samplesPath)
logger.trace(
"Saved the sampler's result to [{}].",
MaskedPath(samplesPath)
)
}

runtimeConnector ! RuntimeConnector.RegisterEventsMonitor(
Expand All @@ -83,6 +90,21 @@ final class ProfilingManager(
case None =>
sender() ! ProfilingProtocol.ProfilingStopResponse
}

case ProfilingProtocol.ProfilingSnapshotRequest =>
val instant = clock.instant()

Try(saveHeapDump(instant)) match {
case Failure(exception) =>
logger.error("Failed to save the memory snapshot.", exception)
case Success(heapDumpPath) =>
logger.trace(
"Saved the memory snapshot to [{}].",
MaskedPath(heapDumpPath)
)
}

sender() ! ProfilingProtocol.ProfilingSnapshotResponse
}

private def saveSamplerResult(
Expand All @@ -98,6 +120,16 @@ final class ProfilingManager(
samplesPath
}

private def saveHeapDump(instant: Instant): Path = {
val heapDumpFileName = createHeapDumpFileName(instant)
val heapDumpPath =
distributionManager.paths.profiling.resolve(heapDumpFileName)

profilingSnapshot.generateSnapshot(heapDumpPath)

heapDumpPath
}

private def createEventsMonitor(instant: Instant): RuntimeEventsMonitor = {
val eventsLogFileName = createEventsFileName(instant)
val eventsLogPath =
Expand All @@ -112,6 +144,7 @@ object ProfilingManager {
private val PROFILING_FILE_PREFIX = "enso-language-server"
private val SAMPLES_FILE_EXT = ".npss"
private val EVENTS_FILE_EXT = ".log"
private val HEAP_DUMP_FILE_EXT = ".hprof"

private val PROFILING_FILE_DATE_PART_FORMATTER =
new DateTimeFormatterBuilder()
Expand Down Expand Up @@ -148,16 +181,30 @@ object ProfilingManager {
s"$baseName$EVENTS_FILE_EXT"
}

def createHeapDumpFileName(instant: Instant): String = {
val baseName = createProfilingFileName(instant)
s"$baseName$HEAP_DUMP_FILE_EXT"
}

/** Creates the configuration object used to create a [[ProfilingManager]].
*
* @param runtimeConnector the connection to runtime
* @param distributionManager the distribution manager
* @param profilingSnapshot the profiling snapshot generator
* @param clock the system clock
*/
def props(
runtimeConnector: ActorRef,
distributionManager: DistributionManager,
clock: Clock = Clock.systemUTC()
profilingSnapshot: ProfilingSnapshot = new HeapDumpSnapshot(),
clock: Clock = Clock.systemUTC()
): Props =
Props(new ProfilingManager(runtimeConnector, distributionManager, clock))
Props(
new ProfilingManager(
runtimeConnector,
distributionManager,
profilingSnapshot,
clock
)
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,10 @@ object ProfilingProtocol {
/** A response to request to stop the profiling. */
case object ProfilingStopResponse

/** A request to create a memory snapshot. */
case object ProfilingSnapshotRequest

/** A response to request to create a memory snapshot. */
case object ProfilingSnapshotResponse

}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import org.enso.languageserver.libraries.handler._
import org.enso.languageserver.monitoring.MonitoringApi.{InitialPing, Ping}
import org.enso.languageserver.monitoring.MonitoringProtocol
import org.enso.languageserver.profiling.ProfilingApi.{
ProfilingSnapshot,
ProfilingStart,
ProfilingStop
}
Expand All @@ -52,6 +53,7 @@ import org.enso.languageserver.requesthandler.monitoring.{
PingHandler
}
import org.enso.languageserver.requesthandler.profiling.{
ProfilingSnapshotHandler,
ProfilingStartHandler,
ProfilingStopHandler
}
Expand Down Expand Up @@ -645,6 +647,10 @@ class JsonConnectionController(
ProfilingStop -> ProfilingStopHandler.props(
requestTimeout,
profilingManager
),
ProfilingSnapshot -> ProfilingSnapshotHandler.props(
requestTimeout,
profilingManager
)
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import org.enso.languageserver.session.SessionApi.InitProtocolConnection
import org.enso.languageserver.text.TextApi._
import org.enso.languageserver.libraries.LibraryApi._
import org.enso.languageserver.profiling.ProfilingApi.{
ProfilingSnapshot,
ProfilingStart,
ProfilingStop
}
Expand Down Expand Up @@ -108,6 +109,7 @@ object JsonRpc {
.registerRequest(RuntimeGetComponentGroups)
.registerRequest(ProfilingStart)
.registerRequest(ProfilingStop)
.registerRequest(ProfilingSnapshot)
.registerNotification(TaskStarted)
.registerNotification(TaskProgressUpdate)
.registerNotification(TaskFinished)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package org.enso.languageserver.requesthandler.profiling

import akka.actor.{Actor, ActorRef, Cancellable, Props}
import com.typesafe.scalalogging.LazyLogging
import org.enso.jsonrpc._
import org.enso.languageserver.profiling.{ProfilingApi, ProfilingProtocol}
import org.enso.languageserver.requesthandler.RequestTimeout
import org.enso.languageserver.util.UnhandledLogging

import scala.concurrent.duration.FiniteDuration

/** A request handler for `profiling/snapshot` commands.
*
* @param timeout a request timeout
* @param profilingManager a reference to the profiling manager
*/
class ProfilingSnapshotHandler(
timeout: FiniteDuration,
profilingManager: ActorRef
) extends Actor
with LazyLogging
with UnhandledLogging {

import context.dispatcher

override def receive: Receive = requestStage

private def requestStage: Receive = {
case Request(ProfilingApi.ProfilingSnapshot, id, _) =>
profilingManager ! ProfilingProtocol.ProfilingSnapshotRequest
val cancellable =
context.system.scheduler.scheduleOnce(timeout, self, RequestTimeout)
context.become(
responseStage(
id,
sender(),
cancellable
)
)
}

private def responseStage(
id: Id,
replyTo: ActorRef,
cancellable: Cancellable
): Receive = {
case RequestTimeout =>
logger.error("Request [{}] timed out.", id)
replyTo ! ResponseError(Some(id), Errors.RequestTimeout)
context.stop(self)

case ProfilingProtocol.ProfilingSnapshotResponse =>
replyTo ! ResponseResult(ProfilingApi.ProfilingSnapshot, id, Unused)
cancellable.cancel()
context.stop(self)
}

}

object ProfilingSnapshotHandler {

/** Creates configuration object used to create a [[ProfilingSnapshotHandler]].
*
* @param timeout request timeout
* @param profilingManager reference to the profiling manager
*/
def props(timeout: FiniteDuration, profilingManager: ActorRef): Props =
Props(new ProfilingSnapshotHandler(timeout, profilingManager))

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.enso.languageserver.profiling

import org.enso.profiling.snapshot.ProfilingSnapshot

import java.nio.file.{Files, Path}

/** Generating a memory dump can take significant time. This test profiling snapshot
* just creates an empty file in the specified location.
*/
final class TestProfilingSnapshot extends ProfilingSnapshot {

/** @inheritdoc */
override def generateSnapshot(output: Path): Unit = {
Files.write(output, Array.emptyByteArray)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ import org.enso.languageserver.filemanager._
import org.enso.languageserver.io._
import org.enso.languageserver.libraries._
import org.enso.languageserver.monitoring.IdlenessMonitor
import org.enso.languageserver.profiling.ProfilingManager
import org.enso.languageserver.profiling.{
ProfilingManager,
TestProfilingSnapshot
}
import org.enso.languageserver.protocol.json.{
JsonConnectionControllerFactory,
JsonRpcProtocolFactory
Expand Down Expand Up @@ -87,6 +90,7 @@ class BaseServerTest
val runtimeConnectorProbe = TestProbe()
val versionCalculator = Sha3_224VersionCalculator
val clock = TestClock()
val profilingSnapshot = new TestProfilingSnapshot

val typeGraph: TypeGraph = {
val graph = TypeGraph("Any")
Expand Down Expand Up @@ -351,6 +355,7 @@ class BaseServerTest
ProfilingManager.props(
runtimeConnectorProbe.ref,
distributionManager,
profilingSnapshot,
clock
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ object ProfilingJsonMessages {
"id": $reqId,
"result": null
}"""

def profilingStart(reqId: Int) =
json"""
{ "jsonrpc": "2.0",
Expand All @@ -26,4 +27,12 @@ object ProfilingJsonMessages {
"params": null
}"""

def profilingSnapshot(reqId: Int) =
json"""
{ "jsonrpc": "2.0",
"method": "profiling/snapshot",
"id": $reqId,
"params": null
}"""

}
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,21 @@ class ProfilingManagerTest extends BaseServerTest {
Files.exists(samplesFile) shouldEqual true
Files.exists(eventsFile) shouldEqual true
}

"save memory snapshot" in {
val client = getInitialisedWsClient()

client.send(json.profilingSnapshot(1))
client.expectJson(json.ok(1))

val distributionManager = getDistributionManager
val instant = clock.instant
val snapshotFile = distributionManager.paths.profiling.resolve(
ProfilingManager.createHeapDumpFileName(instant)
)

Files.exists(snapshotFile) shouldEqual true
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.enso.profiling.snapshot;

import com.sun.management.HotSpotDiagnosticMXBean;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.Path;
import javax.management.MBeanServer;

final class HeapDumpGenerator {

private static final String HotSpotBeanName = "com.sun.management:type=HotSpotDiagnostic";
private static volatile HotSpotDiagnosticMXBean hotSpotDiagnosticMXBean;

private HeapDumpGenerator() {}

/**
* Store the heap dump in the output file in the same format as the hprof heap dump.
*
* @param output the output file.
* @param dumpOnlyLiveObjects if {@code true} dump only <i>live</i> objects i.e. objects that are
* reachable from others.
*/
public static void generateHeapDump(Path output, boolean dumpOnlyLiveObjects) throws IOException {
if (hotSpotDiagnosticMXBean == null) {
synchronized (HeapDumpGenerator.class) {
hotSpotDiagnosticMXBean = getHotSpotDiagnosticMXBean();
}
}

hotSpotDiagnosticMXBean.dumpHeap(output.toString(), dumpOnlyLiveObjects);
}

private static HotSpotDiagnosticMXBean getHotSpotDiagnosticMXBean() throws IOException {
MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
return ManagementFactory.newPlatformMXBeanProxy(
mBeanServer, HotSpotBeanName, HotSpotDiagnosticMXBean.class);
}
}
Loading

0 comments on commit b224f95

Please sign in to comment.