Skip to content

Commit

Permalink
Reduce stack trace capture overhead
Browse files Browse the repository at this point in the history
Capture stack traces for multiple threads at once to reduce number of safe
points needed.
  • Loading branch information
trask committed Nov 20, 2014
1 parent fac8e40 commit 36d8d38
Show file tree
Hide file tree
Showing 35 changed files with 431 additions and 514 deletions.
23 changes: 0 additions & 23 deletions core/app/scripts/controllers/config/profiling.js
Expand Up @@ -30,33 +30,10 @@ glowroot.controller('ConfigProfilingCtrl', [
};
$scope.$on('$locationChangeStart', confirmIfHasChanges($scope));

$scope.$watchCollection('[page.traceStoreThresholdOverride, page.traceStoreThresholdOverrideMillis]',
function (newValues) {
if (newValues[0] === undefined) {
// initial
return;
}
if (newValues[0]) {
$scope.config.traceStoreThresholdOverrideMillis = newValues[1];
} else {
$scope.config.traceStoreThresholdOverrideMillis = -1;
// update disabled input text to show the overridden value from general config
$scope.page.traceStoreThresholdOverrideMillis = $scope.defaultTraceStoreThresholdMillis;
}
});

function onNewData(data) {
$scope.loaded = true;
$scope.config = data.config;
$scope.originalConfig = angular.copy(data.config);

$scope.defaultTraceStoreThresholdMillis = data.defaultTraceStoreThresholdMillis;
if (data.config.traceStoreThresholdOverrideMillis === -1) {
$scope.page.traceStoreThresholdOverride = false;
} else {
$scope.page.traceStoreThresholdOverride = true;
$scope.page.traceStoreThresholdOverrideMillis = $scope.config.traceStoreThresholdOverrideMillis;
}
}

$scope.save = function (deferred) {
Expand Down
24 changes: 19 additions & 5 deletions core/app/views/config/advanced.html
Expand Up @@ -65,20 +65,34 @@ <h2>Advanced</h2>
</div>
</div>
<div gt-form-group
gt-label="Max entries per trace"
gt-model="config.maxEntriesPerTrace"
gt-label="Max trace entries per transaction"
gt-model="config.maxTraceEntriesPerTransaction"
gt-number="true"
gt-pattern="pattern.integer"
gt-required="loaded"
gt-width="7em">
<div class="help-block">
Maximum number of entries collected for a given trace. This is used to
Maximum number of trace entries collected for a given transaction. This is used to
limit the memory requirement of very long transactions that would capture
potentially hundreds of thousands of trace entries (e.g. large batch or background
operations).
Also, the UI is not really optimized for viewing super large numbers of trace entries.
</div>
</div>
<div gt-form-group
gt-label="Max stack trace samples per transaction"
gt-model="config.maxStackTraceSamplesPerTransaction"
gt-number="true"
gt-pattern="pattern.integer"
gt-required="loaded"
gt-width="7em">
<div class="help-block">
Maximum number of stack trace samples collected for a given transaction. This is used to
limit the memory requirement of very long transactions that would capture
potentially hundreds of thousands of stack traces (e.g. large batch or background
operations).
</div>
</div>
<div gt-form-group
gt-label="Thread info"
gt-checkbox-label="Capture thread info per transaction"
Expand All @@ -91,11 +105,11 @@ <h2>Advanced</h2>
</div>
<div gt-form-group
gt-label="GC info"
gt-checkbox-label="Capture garbage collection info per trace"
gt-checkbox-label="Capture garbage collection info per transaction"
gt-model="config.captureGcInfo"
gt-type="checkbox">
<div class="help-block">
Capture global garbage collection info per trace.
Capture global garbage collection info per transaction.
</div>
</div>
<div gt-form-group
Expand Down
45 changes: 0 additions & 45 deletions core/app/views/config/profiling.html
Expand Up @@ -31,18 +31,6 @@ <h2>Profiling</h2>
gt-type="switch"
class="gt-form-group-without-help-block">
</div>
<div gt-form-group
gt-label="Transaction percentage"
gt-model="config.transactionPercentage"
gt-number="true"
gt-pattern="pattern.percentage"
gt-required="loaded"
gt-width="7em"
gt-addon="%">
<div class="help-block">
The percentage of transactions to profile.
</div>
</div>
<div gt-form-group
gt-label="Interval"
gt-model="config.intervalMillis"
Expand All @@ -55,39 +43,6 @@ <h2>Profiling</h2>
The interval at which the profiler captures stack traces.
</div>
</div>
<div class="form-group"
ng-class="{'has-error': formCtrl.traceStoreThresholdOverrideMillis.$invalid}">
<label class="col-lg-3 control-label" for="traceStoreThresholdOverrideMillis">
Trace store threshold override
</label>
<div class="col-lg-9">
<div class="input-group">
<div class="input-group-addon"
style="padding-top: 8px;">
<input type="checkbox"
ng-model="page.traceStoreThresholdOverride">
</div>
<input type="text"
class="form-control"
ng-model="page.traceStoreThresholdOverrideMillis"
ng-pattern="pattern.integer"
id="traceStoreThresholdOverrideMillis"
name="traceStoreThresholdOverrideMillis"
style="width: 7em;"
ng-readonly="!page.traceStoreThresholdOverride"
ng-required="page.traceStoreThresholdOverride">
<span class="input-group-addon">
milliseconds
</span>
</div>
<div class="help-block">
Override the general store threshold for profiled traces.
After going through the trouble of collecting profiling info,
it may be worth while to store the given trace even if it doesn't hit the
general store threshold.
</div>
</div>
</div>
<div class="form-group form-buttons">
<div class="col-lg-offset-3 col-lg-9">
<div gt-button
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/glowroot/GlowrootModule.java
Expand Up @@ -179,7 +179,8 @@ public long read() {
}
collectorModule = new CollectorModule(clock, ticker, jvmModule, configModule,
storageModule.getTraceRepository(), storageModule.getAggregateRepository(),
storageModule.getGaugePointDao(), scheduledExecutor, viewerModeEnabled);
storageModule.getGaugePointDao(), transactionModule.getTransactionRegistry(),
scheduledExecutor, viewerModeEnabled);
// now inject the real TransactionCollector into the proxy
transactionCollectorProxy.setInstance(collectorModule.getTransactionCollector());
// now init plugins to give them a chance to do something in their static initializer
Expand Down
12 changes: 11 additions & 1 deletion core/src/main/java/org/glowroot/collector/CollectorModule.java
Expand Up @@ -26,6 +26,7 @@
import org.glowroot.jvm.JvmModule;
import org.glowroot.markers.OnlyUsedByTests;
import org.glowroot.markers.ThreadSafe;
import org.glowroot.transaction.TransactionRegistry;

import static java.util.concurrent.TimeUnit.SECONDS;

Expand All @@ -51,15 +52,19 @@ public class CollectorModule {
private final AggregateCollector aggregateCollector;
@Nullable
private final GaugeCollector gaugeCollector;
@Nullable
private final StackTraceCollector stackTraceCollector;

public CollectorModule(Clock clock, Ticker ticker, JvmModule jvmModule,
ConfigModule configModule, TraceRepository traceRepository,
AggregateRepository aggregateRepository, GaugePointRepository gaugePointRepository,
ScheduledExecutorService scheduledExecutor, boolean viewerModeEnabled) {
TransactionRegistry transactionRegistry, ScheduledExecutorService scheduledExecutor,
boolean viewerModeEnabled) {
ConfigService configService = configModule.getConfigService();
if (viewerModeEnabled) {
aggregateCollector = null;
gaugeCollector = null;
stackTraceCollector = null;
} else {
aggregateCollector = new AggregateCollector(scheduledExecutor, aggregateRepository,
clock, fixedAggregateIntervalSeconds);
Expand All @@ -70,6 +75,8 @@ public CollectorModule(Clock clock, Ticker ticker, JvmModule jvmModule,
- (clock.currentTimeMillis() % fixedGaugeIntervalSeconds);
gaugeCollector.scheduleAtFixedRate(scheduledExecutor, initialDelay,
fixedGaugeIntervalSeconds, SECONDS);
stackTraceCollector = StackTraceCollector.create(transactionRegistry, configService,
scheduledExecutor);
}
// TODO should be no need for transaction collector in viewer mode
transactionCollector = new TransactionCollectorImpl(scheduledExecutor, configService,
Expand All @@ -93,5 +100,8 @@ public void close() {
if (aggregateCollector != null) {
aggregateCollector.close();
}
if (stackTraceCollector != null) {
stackTraceCollector.close();
}
}
}
131 changes: 131 additions & 0 deletions core/src/main/java/org/glowroot/collector/StackTraceCollector.java
@@ -0,0 +1,131 @@
/*
* Copyright 2011-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.glowroot.collector;

import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;

import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.glowroot.api.PluginServices.ConfigListener;
import org.glowroot.config.ConfigService;
import org.glowroot.config.ProfilingConfig;
import org.glowroot.markers.OnlyUsedByTests;
import org.glowroot.markers.Singleton;
import org.glowroot.transaction.TransactionRegistry;
import org.glowroot.transaction.model.Transaction;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

/**
* @author Trask Stalnaker
* @since 0.5
*/
@Singleton
class StackTraceCollector implements Runnable {

private static final Logger logger = LoggerFactory.getLogger(StackTraceCollector.class);

private final TransactionRegistry transactionRegistry;
private final ConfigService configService;
private final ScheduledExecutorService scheduledExecutor;

private volatile boolean currentEnabled;
private volatile int currentIntervalMillis;
@Nullable
private volatile Future<?> currentFuture;

public static StackTraceCollector create(TransactionRegistry transactionRegistry,
ConfigService configService, ScheduledExecutorService scheduledExecutor) {
final StackTraceCollector stackTraceCollector =
new StackTraceCollector(transactionRegistry, configService, scheduledExecutor);
configService.addConfigListener(new ConfigListener() {
@Override
public void onChange() {
stackTraceCollector.updateScheduleIfNeeded();
}
});
stackTraceCollector.updateScheduleIfNeeded();
return stackTraceCollector;
}

private StackTraceCollector(TransactionRegistry transactionRegistry,
final ConfigService configService, ScheduledExecutorService scheduledExecutor) {
this.transactionRegistry = transactionRegistry;
this.configService = configService;
this.scheduledExecutor = scheduledExecutor;
}

@Override
public void run() {
try {
runInternal();
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}

void updateScheduleIfNeeded() {
boolean newEnabled = configService.getProfilingConfig().isEnabled();
int newIntervalMillis = configService.getProfilingConfig().getIntervalMillis();
if (newEnabled != currentEnabled || newIntervalMillis != currentIntervalMillis) {
if (currentFuture != null) {
currentFuture.cancel(false);
}
if (newEnabled) {
currentFuture = scheduledExecutor.scheduleAtFixedRate(this, newIntervalMillis,
newIntervalMillis, MILLISECONDS);
}
currentEnabled = newEnabled;
currentIntervalMillis = newIntervalMillis;
}
}

private void runInternal() {
ProfilingConfig config = configService.getProfilingConfig();
if (!config.isEnabled()) {
return;
}
List<Transaction> transactions =
ImmutableList.copyOf(transactionRegistry.getTransactions());
if (transactions.isEmpty()) {
return;
}
long[] threadIds = new long[transactions.size()];
for (int i = 0; i < transactions.size(); i++) {
threadIds[i] = transactions.get(i).getThreadId();
}
ThreadInfo[] threadInfos =
ManagementFactory.getThreadMXBean().getThreadInfo(threadIds, Integer.MAX_VALUE);
for (int i = 0; i < transactions.size(); i++) {
transactions.get(i).captureStackTrace(threadInfos[i], false,
configService.getAdvancedConfig().getMaxStackTraceSamplesPerTransaction());
}
}

@OnlyUsedByTests
void close() {
if (currentFuture != null) {
currentFuture.cancel(false);
}
}
}
Expand Up @@ -30,7 +30,6 @@
import org.glowroot.common.Clock;
import org.glowroot.common.Ticker;
import org.glowroot.config.ConfigService;
import org.glowroot.config.ProfilingConfig;
import org.glowroot.markers.GuardedBy;
import org.glowroot.markers.OnlyUsedByTests;
import org.glowroot.markers.Singleton;
Expand Down Expand Up @@ -92,16 +91,6 @@ public boolean shouldStore(Transaction transaction) {
return true;
}
}
// check if should store for profiling
if (transaction.isProfiled()) {
int traceStoreThresholdOverrideMillis =
configService.getProfilingConfig().getTraceStoreThresholdOverrideMillis();
if (traceStoreThresholdOverrideMillis != ProfilingConfig.USE_GENERAL_STORE_THRESHOLD
&& transaction.getDuration() >= MILLISECONDS.toNanos(
traceStoreThresholdOverrideMillis)) {
return true;
}
}
// check if trace-specific store threshold was set
long traceStoreThresholdMillis = transaction.getStoreThresholdMillisOverride();
if (traceStoreThresholdMillis != Transaction.USE_GENERAL_STORE_THRESHOLD
Expand Down

0 comments on commit 36d8d38

Please sign in to comment.