Skip to content

Commit

Permalink
Script: compile/cache eviction history metric placeholders (#78257) (#…
Browse files Browse the repository at this point in the history 
…78333)

Adds 5m/15m/24h metrics to _nodes/stats when those metrics
are non-zero.  Those metrics are not yet populated.

BWC: history metrics are only sent between v8.0.0+ nodes,
v7.16.0 nodes will not send those metrics until they are populated.

Refs: #62899
Backport: db75c4b
  • Loading branch information
@stu-elastic
stu-elastic committed Sep 27, 2021
1 parent e4718fe commit 650b493
Show file tree
Hide file tree
Showing 4 changed files with 326 additions and 7 deletions.
96 changes: 95 additions & 1 deletion server/src/main/java/org/elasticsearch/script/ScriptContextStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,28 @@
public class ScriptContextStats implements Writeable, ToXContentFragment, Comparable<ScriptContextStats> {
private final String context;
private final long compilations;
private final TimeSeries compilationsHistory;
private final long cacheEvictions;
private final TimeSeries cacheEvictionsHistory;
private final long compilationLimitTriggered;

public ScriptContextStats(String context, long compilations, long cacheEvictions, long compilationLimitTriggered) {
public ScriptContextStats(String context, long compilations, long cacheEvictions, long compilationLimitTriggered,
TimeSeries compilationsHistory, TimeSeries cacheEvictionsHistory) {
this.context = Objects.requireNonNull(context);
this.compilations = compilations;
this.cacheEvictions = cacheEvictions;
this.compilationLimitTriggered = compilationLimitTriggered;
this.compilationsHistory = compilationsHistory;
this.cacheEvictionsHistory = cacheEvictionsHistory;
}

public ScriptContextStats(StreamInput in) throws IOException {
context = in.readString();
compilations = in.readVLong();
cacheEvictions = in.readVLong();
compilationLimitTriggered = in.readVLong();
compilationsHistory = null;
cacheEvictionsHistory = null;
}

@Override
Expand All @@ -45,6 +52,65 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(compilationLimitTriggered);
}

public static class TimeSeries implements Writeable, ToXContentFragment {
public final long fiveMinutes;
public final long fifteenMinutes;
public final long twentyFourHours;

public TimeSeries() {
this.fiveMinutes = 0;
this.fifteenMinutes = 0;
this.twentyFourHours = 0;
}

public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) {
assert fiveMinutes >= 0;
this.fiveMinutes = fiveMinutes;
assert fifteenMinutes >= fiveMinutes;
this.fifteenMinutes = fifteenMinutes;
assert twentyFourHours >= fifteenMinutes;
this.twentyFourHours = twentyFourHours;
}

public TimeSeries(StreamInput in) throws IOException {
fiveMinutes = in.readVLong();
fifteenMinutes = in.readVLong();
twentyFourHours = in.readVLong();
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(Fields.FIVE_MINUTES, fiveMinutes);
builder.field(Fields.FIFTEEN_MINUTES, fifteenMinutes);
builder.field(Fields.TWENTY_FOUR_HOURS, twentyFourHours);
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(fiveMinutes);
out.writeVLong(fifteenMinutes);
out.writeVLong(twentyFourHours);
}

public boolean isEmpty() {
return twentyFourHours == 0;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TimeSeries that = (TimeSeries) o;
return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours;
}

@Override
public int hashCode() {
return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours);
}
}

public String getContext() {
return context;
}
Expand All @@ -53,10 +119,18 @@ public long getCompilations() {
return compilations;
}

public TimeSeries getCompilationsHistory() {
return compilationsHistory;
}

public long getCacheEvictions() {
return cacheEvictions;
}

public TimeSeries getCacheEvictionsHistory() {
return cacheEvictionsHistory;
}

public long getCompilationLimitTriggered() {
return compilationLimitTriggered;
}
Expand All @@ -66,7 +140,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject();
builder.field(Fields.CONTEXT, getContext());
builder.field(Fields.COMPILATIONS, getCompilations());

TimeSeries series = getCompilationsHistory();
if (series != null && series.isEmpty() == false) {
builder.startObject(Fields.COMPILATIONS_HISTORY);
series.toXContent(builder, params);
builder.endObject();
}

builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions());
series = getCacheEvictionsHistory();
if (series != null && series.isEmpty() == false) {
builder.startObject(Fields.CACHE_EVICTIONS_HISTORY);
series.toXContent(builder, params);
builder.endObject();
}

builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered());
builder.endObject();
return builder;
Expand All @@ -80,7 +169,12 @@ public int compareTo(ScriptContextStats o) {
static final class Fields {
static final String CONTEXT = "context";
static final String COMPILATIONS = "compilations";
static final String COMPILATIONS_HISTORY = "compilations_history";
static final String CACHE_EVICTIONS = "cache_evictions";
static final String CACHE_EVICTIONS_HISTORY = "cache_evictions_history";
static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
static final String FIVE_MINUTES = "5m";
static final String FIFTEEN_MINUTES = "15m";
static final String TWENTY_FOUR_HOURS = "24h";
}
}
4 changes: 3 additions & 1 deletion server/src/main/java/org/elasticsearch/script/ScriptMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ public ScriptContextStats stats(String context) {
context,
compilationsMetric.count(),
cacheEvictionsMetric.count(),
compilationLimitTriggered.count()
compilationLimitTriggered.count(),
new ScriptContextStats.TimeSeries(),
new ScriptContextStats.TimeSeries()
);
}
}
70 changes: 65 additions & 5 deletions server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.node.AdaptiveSelectionStats;
import org.elasticsearch.node.ResponseCollectorService;
import org.elasticsearch.script.ScriptCacheStats;
import org.elasticsearch.script.ScriptContextStats;
import org.elasticsearch.script.ScriptStats;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
Expand All @@ -36,9 +37,11 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
Expand Down Expand Up @@ -233,11 +236,38 @@ public void testSerialization() throws IOException {
}
}
ScriptStats scriptStats = nodeStats.getScriptStats();
ScriptStats deserializedScriptStats = deserializedNodeStats.getScriptStats();
if (scriptStats == null) {
assertNull(deserializedNodeStats.getScriptStats());
assertNull(deserializedScriptStats);
} else {
assertEquals(scriptStats.getCacheEvictions(), deserializedNodeStats.getScriptStats().getCacheEvictions());
assertEquals(scriptStats.getCompilations(), deserializedNodeStats.getScriptStats().getCompilations());
List<ScriptContextStats> deserialized = deserializedScriptStats.getContextStats();
long evictions = 0;
long limited = 0;
long compilations = 0;
List<ScriptContextStats> stats = scriptStats.getContextStats();
for (ScriptContextStats generatedStats: stats) {
List<ScriptContextStats> maybeDeserStats = deserialized.stream().filter(
s -> s.getContext().equals(generatedStats.getContext())
).collect(Collectors.toList());

assertEquals(1, maybeDeserStats.size());
ScriptContextStats deserStats = maybeDeserStats.get(0);

evictions += generatedStats.getCacheEvictions();
assertEquals(generatedStats.getCacheEvictions(), deserStats.getCacheEvictions());

limited += generatedStats.getCompilationLimitTriggered();
assertEquals(generatedStats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered());

compilations += generatedStats.getCompilations();
assertEquals(generatedStats.getCompilations(), deserStats.getCompilations());

assertNull(deserStats.getCacheEvictionsHistory());
assertNull(deserStats.getCompilationsHistory());
}
assertEquals(evictions, scriptStats.getCacheEvictions());
assertEquals(limited, scriptStats.getCompilationLimitTriggered());
assertEquals(compilations, scriptStats.getCompilations());
}
DiscoveryStats discoveryStats = nodeStats.getDiscoveryStats();
DiscoveryStats deserializedDiscoveryStats = deserializedNodeStats.getDiscoveryStats();
Expand Down Expand Up @@ -551,8 +581,27 @@ public static NodeStats createNodeStats() {
}
allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray);
}
ScriptStats scriptStats = frequently() ?
new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
ScriptStats scriptStats = null;
if (frequently()) {
int numContents = randomIntBetween(0, 20);
List<ScriptContextStats> stats = new ArrayList<>(numContents);
HashSet<String> contexts = new HashSet<>();
for (int i = 0; i < numContents; i++) {
long compile = randomLongBetween(0, 1024);
long eviction = randomLongBetween(0, 1024);
String context = randomValueOtherThanMany(contexts::contains, () -> randomAlphaOfLength(12));
contexts.add(context);
stats.add(new ScriptContextStats(
context,
compile,
eviction,
randomLongBetween(0, 1024),
randomTimeSeries(),
randomTimeSeries())
);
}
scriptStats = new ScriptStats(stats);
}
DiscoveryStats discoveryStats = frequently()
? new DiscoveryStats(
randomBoolean()
Expand Down Expand Up @@ -653,6 +702,17 @@ public static NodeStats createNodeStats() {
ingestStats, adaptiveSelectionStats, scriptCacheStats, null);
}

private static ScriptContextStats.TimeSeries randomTimeSeries() {
if (randomBoolean()) {
long day = randomLongBetween(0, 1024);
long fifteen = day >= 1 ? randomLongBetween(0, day) : 0;
long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0;
return new ScriptContextStats.TimeSeries(five, fifteen, day);
} else {
return new ScriptContextStats.TimeSeries();
}
}

private IngestStats.Stats getPipelineStats(List<IngestStats.PipelineStat> pipelineStats, String id) {
return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null);
}
Expand Down

0 comments on commit 650b493

Please sign in to comment.