Skip to content

Commit

Permalink
[Profiling] Add new optional CO2 request params (#102576)
Browse files Browse the repository at this point in the history 
* [Profiling] Add new optional CO2 request params

We add new request params for the profiling API

co2_per_kwh
datacenter_pue
per_core_watt
  • Loading branch information
@rockdaboot @timgrein
rockdaboot authored and timgrein committed Nov 30, 2023
1 parent 616305a commit 1e02244
Show file tree
Hide file tree
Showing 10 changed files with 205 additions and 94 deletions.
2 changes: 1 addition & 1 deletion ...src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

public class GetFlameGraphActionIT extends ProfilingTestCase {
public void testGetStackTracesUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null);
GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null, null, null, null);
GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get();
// only spot-check top level properties - detailed tests are done in unit tests
assertEquals(297, response.getSize());
Expand Down
12 changes: 9 additions & 3 deletions ...rc/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

public class GetStackTracesActionIT extends ProfilingTestCase {
public void testGetStackTracesUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null);
GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null, null, null, null);
request.setAdjustSampleCount(true);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
assertEquals(40, response.getTotalSamples());
Expand Down Expand Up @@ -50,7 +50,10 @@ public void testGetStackTracesFromAPMWithMatch() throws Exception {
1.0d,
query,
"apm-test-*",
"transaction.profiler_stack_trace_ids"
"transaction.profiler_stack_trace_ids",
null,
null,
null
);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
assertEquals(43, response.getTotalFrames());
Expand Down Expand Up @@ -84,7 +87,10 @@ public void testGetStackTracesFromAPMNoMatch() throws Exception {
1.0d,
query,
"apm-test-*",
"transaction.profiler_stack_trace_ids"
"transaction.profiler_stack_trace_ids",
null,
null,
null
);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
assertEquals(0, response.getTotalFrames());
Expand Down
30 changes: 21 additions & 9 deletions x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/CO2Calculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,57 @@ final class CO2Calculator {
private static final double DEFAULT_KILOWATTS_PER_CORE_AARCH64 = 2.8d / 1000.0d; // unit: watt / core
private static final double DEFAULT_KILOWATTS_PER_CORE = DEFAULT_KILOWATTS_PER_CORE_X86_64; // unit: watt / core
private static final double DEFAULT_DATACENTER_PUE = 1.7d;
private static final double CUSTOM_CO2_FACTOR = 1.0d;
private static final Provider DEFAULT_PROVIDER = new Provider(DEFAULT_DATACENTER_PUE, Collections.emptyMap());
private final InstanceTypeService instanceTypeService;
private final Map<String, HostMetadata> hostMetadata;
private final double samplingDurationInSeconds;
private final double customCO2PerKWH;
private final double customDatacenterPUE;
private final double customKilowattsPerCore;

CO2Calculator(InstanceTypeService instanceTypeService, Map<String, HostMetadata> hostMetadata, double samplingDurationInSeconds) {
CO2Calculator(
InstanceTypeService instanceTypeService,
Map<String, HostMetadata> hostMetadata,
double samplingDurationInSeconds,
Double customCO2PerKWH,
Double customDatacenterPUE,
Double customPerCoreWatt
) {
this.instanceTypeService = instanceTypeService;
this.hostMetadata = hostMetadata;
this.samplingDurationInSeconds = samplingDurationInSeconds;
this.samplingDurationInSeconds = samplingDurationInSeconds > 0 ? samplingDurationInSeconds : 1.0d; // avoid division by zero
this.customCO2PerKWH = customCO2PerKWH == null ? DEFAULT_CO2_TONS_PER_KWH : customCO2PerKWH;
this.customDatacenterPUE = customDatacenterPUE == null ? DEFAULT_DATACENTER_PUE : customDatacenterPUE;
this.customKilowattsPerCore = customPerCoreWatt == null ? DEFAULT_KILOWATTS_PER_CORE : customPerCoreWatt / 1000.0d;
}

public double getAnnualCO2Tons(String hostID, long samples) {
double annualCoreHours = CostCalculator.annualCoreHours(samplingDurationInSeconds, samples, DEFAULT_SAMPLING_FREQUENCY);

HostMetadata host = hostMetadata.get(hostID);
if (host == null) {
return DEFAULT_KILOWATTS_PER_CORE * DEFAULT_CO2_TONS_PER_KWH * annualCoreHours * DEFAULT_DATACENTER_PUE;
return customKilowattsPerCore * customCO2PerKWH * annualCoreHours * customDatacenterPUE;
}

CostEntry costs = instanceTypeService.getCosts(host.instanceType);
if (costs == null) {
return getKiloWattsPerCore(host) * getCO2TonsPerKWH(host) * annualCoreHours * getDatacenterPUE(host);
}

return annualCoreHours * costs.co2Factor * CUSTOM_CO2_FACTOR; // unit: metric tons
return annualCoreHours * costs.co2Factor; // unit: metric tons
}

private static double getKiloWattsPerCore(HostMetadata host) {
private double getKiloWattsPerCore(HostMetadata host) {
if ("aarch64".equals(host.profilingHostMachine)) {
// Assume that AARCH64 (aka ARM64) machines are more energy efficient than x86_64 machines.
return DEFAULT_KILOWATTS_PER_CORE_AARCH64;
}
return DEFAULT_KILOWATTS_PER_CORE;
return customKilowattsPerCore;
}

private static double getCO2TonsPerKWH(HostMetadata host) {
private double getCO2TonsPerKWH(HostMetadata host) {
Provider provider = PROVIDERS.getOrDefault(host.instanceType.provider, DEFAULT_PROVIDER);
return provider.co2TonsPerKWH.getOrDefault(host.instanceType.region, DEFAULT_CO2_TONS_PER_KWH);
return provider.co2TonsPerKWH.getOrDefault(host.instanceType.region, customCO2PerKWH);
}

private static double getDatacenterPUE(HostMetadata host) {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/CostCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ final class CostCalculator {
) {
this.instanceTypeService = instanceTypeService;
this.hostMetadata = hostMetadata;
this.samplingDurationInSeconds = samplingDurationInSeconds;
this.samplingDurationInSeconds = samplingDurationInSeconds > 0 ? samplingDurationInSeconds : 1.0d; // avoid division by zero
this.customCostFactor = customCostFactor == null ? DEFAULT_CUSTOM_COST_FACTOR : customCostFactor;
}

Expand Down
134 changes: 74 additions & 60 deletions ...ugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.index.query.QueryBuilder;
Expand Down Expand Up @@ -40,21 +39,28 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids");
public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration");
public static final ParseField CUSTOM_COST_FACTOR_FIELD = new ParseField("custom_cost_factor");
public static final ParseField CUSTOM_CO2_PER_KWH = new ParseField("co2_per_kwh");
public static final ParseField CUSTOM_DATACENTER_PUE = new ParseField("datacenter_pue");
public static final ParseField CUSTOM_PER_CORE_WATT = new ParseField("per_core_watt");
private static final int DEFAULT_SAMPLE_SIZE = 20_000;

private QueryBuilder query;
private Integer sampleSize;
private String indices;
private String stackTraceIds;
private Double requestedDuration;
private Double customCostFactor;
private Double customCO2PerKWH;
private Double customDatacenterPUE;
private Double customPerCoreWatt;

// We intentionally don't expose this field via the REST API, but we can control behavior within Elasticsearch.
// Once we have migrated all client-side code to dedicated APIs (such as the flamegraph API), we can adjust
// sample counts by default and remove this flag.
private Boolean adjustSampleCount;

public GetStackTracesRequest() {
this(null, null, null, null, null, null);
this(null, null, null, null, null, null, null, null, null);
}

public GetStackTracesRequest(
Expand All @@ -63,14 +69,20 @@ public GetStackTracesRequest(
Double customCostFactor,
QueryBuilder query,
String indices,
String stackTraceIds
String stackTraceIds,
Double customCO2PerKWH,
Double customDatacenterPUE,
Double customPerCoreWatt
) {
this.sampleSize = sampleSize;
this.requestedDuration = requestedDuration;
this.customCostFactor = customCostFactor;
this.query = query;
this.indices = indices;
this.stackTraceIds = stackTraceIds;
this.customCO2PerKWH = customCO2PerKWH;
this.customDatacenterPUE = customDatacenterPUE;
this.customPerCoreWatt = customPerCoreWatt;
}

public GetStackTracesRequest(StreamInput in) throws IOException {
Expand All @@ -81,6 +93,9 @@ public GetStackTracesRequest(StreamInput in) throws IOException {
this.adjustSampleCount = in.readOptionalBoolean();
this.indices = in.readOptionalString();
this.stackTraceIds = in.readOptionalString();
this.customCO2PerKWH = in.readOptionalDouble();
this.customDatacenterPUE = in.readOptionalDouble();
this.customPerCoreWatt = in.readOptionalDouble();
}

@Override
Expand All @@ -92,10 +107,13 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalBoolean(adjustSampleCount);
out.writeOptionalString(indices);
out.writeOptionalString(stackTraceIds);
out.writeOptionalDouble(customCO2PerKWH);
out.writeOptionalDouble(customDatacenterPUE);
out.writeOptionalDouble(customPerCoreWatt);
}

public Integer getSampleSize() {
return sampleSize;
return sampleSize != null ? sampleSize : DEFAULT_SAMPLE_SIZE;
}

public Double getRequestedDuration() {
Expand All @@ -106,6 +124,18 @@ public Double getCustomCostFactor() {
return customCostFactor;
}

public Double getCustomCO2PerKWH() {
return customCO2PerKWH;
}

public Double getCustomDatacenterPUE() {
return customDatacenterPUE;
}

public Double getCustomPerCoreWatt() {
return customPerCoreWatt;
}

public QueryBuilder getQuery() {
return query;
}
Expand Down Expand Up @@ -151,6 +181,12 @@ public void parseXContent(XContentParser parser) throws IOException {
this.requestedDuration = parser.doubleValue();
} else if (CUSTOM_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.customCostFactor = parser.doubleValue();
} else if (CUSTOM_CO2_PER_KWH.match(currentFieldName, parser.getDeprecationHandler())) {
this.customCO2PerKWH = parser.doubleValue();
} else if (CUSTOM_DATACENTER_PUE.match(currentFieldName, parser.getDeprecationHandler())) {
this.customDatacenterPUE = parser.doubleValue();
} else if (CUSTOM_PER_CORE_WATT.match(currentFieldName, parser.getDeprecationHandler())) {
this.customPerCoreWatt = parser.doubleValue();
} else {
throw new ParsingException(
parser.getTokenLocation(),
Expand Down Expand Up @@ -201,35 +237,23 @@ public ActionRequestValidationException validate() {
validationException
);
}
if (sampleSize == null) {
validationException = addValidationError(
"[" + SAMPLE_SIZE_FIELD.getPreferredName() + "] is mandatory",
validationException
);
} else if (sampleSize <= 0) {
validationException = addValidationError(
"[" + SAMPLE_SIZE_FIELD.getPreferredName() + "] must be greater or equals than 1, got: " + sampleSize,
validationException
);
}
}
if (requestedDuration != null) {
if (requestedDuration <= 0.0d) {
validationException = addValidationError(
"[" + REQUESTED_DURATION_FIELD.getPreferredName() + "] must be greater than 0, got: " + requestedDuration,
validationException
);
}
validationException = requirePositive(SAMPLE_SIZE_FIELD, sampleSize, validationException);
}
if (customCostFactor != null) {
if (customCostFactor <= 0.0d) {
validationException = addValidationError(
"[" + CUSTOM_COST_FACTOR_FIELD.getPreferredName() + "] must be greater than 0, got: " + customCostFactor,
validationException
);
validationException = requirePositive(REQUESTED_DURATION_FIELD, requestedDuration, validationException);
validationException = requirePositive(CUSTOM_COST_FACTOR_FIELD, customCostFactor, validationException);
validationException = requirePositive(CUSTOM_CO2_PER_KWH, customCO2PerKWH, validationException);
validationException = requirePositive(CUSTOM_DATACENTER_PUE, customDatacenterPUE, validationException);
validationException = requirePositive(CUSTOM_PER_CORE_WATT, customPerCoreWatt, validationException);
return validationException;
}

private static ActionRequestValidationException requirePositive(ParseField field, Number value, ActionRequestValidationException e) {
if (value != null) {
if (value.doubleValue() <= 0.0d) {
return addValidationError("[" + field.getPreferredName() + "] must be greater than 0, got: " + value, e);
}
}
return validationException;
return e;
}

@Override
Expand All @@ -239,41 +263,31 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
public String getDescription() {
// generating description lazily since the query could be large
StringBuilder sb = new StringBuilder();
if (indices == null) {
sb.append("indices[]");
} else {
sb.append("indices[").append(indices).append("]");
}
if (stackTraceIds == null) {
sb.append("stackTraceIds[]");
} else {
sb.append("stackTraceIds[").append(stackTraceIds).append("]");
}
if (sampleSize == null) {
sb.append("sample_size[]");
} else {
sb.append("sample_size[").append(sampleSize).append("]");
}
if (requestedDuration == null) {
sb.append(", requested_duration[]");
} else {
sb.append(", requested_duration[").append(requestedDuration).append("]");
}
if (customCostFactor == null) {
sb.append(", custom_cost_factor[]");
} else {
sb.append(", custom_cost_factor[").append(customCostFactor).append("]");
}
if (query == null) {
sb.append(", query[]");
} else {
sb.append(", query[").append(Strings.toString(query)).append("]");
}
appendField(sb, "indices", indices);
appendField(sb, "stacktrace_ids", stackTraceIds);
appendField(sb, "sample_size", sampleSize);
appendField(sb, "requested_duration", requestedDuration);
appendField(sb, "custom_cost_factor", customCostFactor);
appendField(sb, "co2_per_kwh", customCO2PerKWH);
appendField(sb, "datacenter_pue", customDatacenterPUE);
appendField(sb, "per_core_watt", customPerCoreWatt);
appendField(sb, "query", query);
return sb.toString();
}
};
}

private static void appendField(StringBuilder sb, String name, Object value) {
if (sb.isEmpty() == false) {
sb.append(", ");
}
if (value == null) {
sb.append(name).append("[]");
} else {
sb.append(name).append("[").append(value).append("]");
}
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down

0 comments on commit 1e02244

Please sign in to comment.