forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Search Pipelines] Add stats for search pipelines (opensearch-project…
…#8053) * [Search Pipelines] Add stats for search pipelines This adds statistics on executions and time spent on search pipeline operations, similar to the stats that are available for ingest pipelines. Signed-off-by: Michael Froh <froh@amazon.com> * Compare parsed JSON structure, not exact JSON string As @lukas-vlcek pointed out, asserting equality with an exact JSON string is sensitive to formatting, which makes the test brittle. Instead, we can parse the expected JSON and compare as Maps. Signed-off-by: Michael Froh <froh@amazon.com> * Refactor to common stats/metrics classes Search pipelines and ingest pipelines had identical functionality for tracking metrics around operations and converting those to immutable "stats" objects. That approach isn't even really specific to pipelines, but can be used to track metrics on any repeated operation, so I moved that common logic to the common.metrics package. Signed-off-by: Michael Froh <froh@amazon.com> * Split pipeline metrics tracking into its own class Thanks @saratvemulapalli for the suggestion! This lets the Pipeline class focus on transforming requests / responses, while the subclass focuses on tracking and managing metrics. Signed-off-by: Michael Froh <froh@amazon.com> --------- Signed-off-by: Michael Froh <froh@amazon.com> Signed-off-by: Shivansh Arora <hishiv@amazon.com>
- Loading branch information
Showing
33 changed files
with
1,398 additions
and
458 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.metrics; | ||
|
|
||
| import java.util.concurrent.atomic.AtomicLong; | ||
|
|
||
| /** | ||
| * Mutable tracker of a repeated operation. | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class OperationMetrics { | ||
| /** | ||
| * The mean time it takes to complete the measured item. | ||
| */ | ||
| private final MeanMetric time = new MeanMetric(); | ||
| /** | ||
| * The current count of things being measured. | ||
| * Useful when aggregating multiple metrics to see how many things are in flight. | ||
| */ | ||
| private final AtomicLong current = new AtomicLong(); | ||
| /** | ||
| * The non-decreasing count of failures | ||
| */ | ||
| private final CounterMetric failed = new CounterMetric(); | ||
|
|
||
| /** | ||
| * Invoked before the given operation begins. | ||
| */ | ||
| public void before() { | ||
| current.incrementAndGet(); | ||
| } | ||
|
|
||
| /** | ||
| * Invoked upon completion (success or failure) of the given operation | ||
| * @param currentTime elapsed time of the operation | ||
| */ | ||
| public void after(long currentTime) { | ||
| current.decrementAndGet(); | ||
| time.inc(currentTime); | ||
| } | ||
|
|
||
| /** | ||
| * Invoked upon failure of the operation. | ||
| */ | ||
| public void failed() { | ||
| failed.inc(); | ||
| } | ||
|
|
||
| public void add(OperationMetrics other) { | ||
| // Don't try copying over current, since in-flight requests will be linked to the existing metrics instance. | ||
| failed.inc(other.failed.count()); | ||
| time.add(other.time); | ||
| } | ||
|
|
||
| /** | ||
| * @return an immutable snapshot of the current metric values. | ||
| */ | ||
| public OperationStats createStats() { | ||
| return new OperationStats(time.count(), time.sum(), current.get(), failed.count()); | ||
| } | ||
| } |
107 changes: 107 additions & 0 deletions
107
server/src/main/java/org/opensearch/common/metrics/OperationStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.metrics; | ||
|
|
||
| import org.opensearch.common.io.stream.StreamInput; | ||
| import org.opensearch.common.io.stream.StreamOutput; | ||
| import org.opensearch.common.io.stream.Writeable; | ||
| import org.opensearch.common.unit.TimeValue; | ||
| import org.opensearch.core.xcontent.ToXContentFragment; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| /** | ||
| * An immutable representation of a {@link OperationMetrics} | ||
| */ | ||
| public class OperationStats implements Writeable, ToXContentFragment { | ||
| private final long count; | ||
| private final long totalTimeInMillis; | ||
| private final long current; | ||
| private final long failedCount; | ||
|
|
||
| public OperationStats(long count, long totalTimeInMillis, long current, long failedCount) { | ||
| this.count = count; | ||
| this.totalTimeInMillis = totalTimeInMillis; | ||
| this.current = current; | ||
| this.failedCount = failedCount; | ||
| } | ||
|
|
||
| /** | ||
| * Read from a stream. | ||
| */ | ||
| public OperationStats(StreamInput in) throws IOException { | ||
| count = in.readVLong(); | ||
| totalTimeInMillis = in.readVLong(); | ||
| current = in.readVLong(); | ||
| failedCount = in.readVLong(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeVLong(count); | ||
| out.writeVLong(totalTimeInMillis); | ||
| out.writeVLong(current); | ||
| out.writeVLong(failedCount); | ||
| } | ||
|
|
||
| /** | ||
| * @return The total number of executed operations. | ||
| */ | ||
| public long getCount() { | ||
| return count; | ||
| } | ||
|
|
||
| /** | ||
| * @return The total time spent of in millis. | ||
| */ | ||
| public long getTotalTimeInMillis() { | ||
| return totalTimeInMillis; | ||
| } | ||
|
|
||
| /** | ||
| * @return The total number of operations currently executing. | ||
| */ | ||
| public long getCurrent() { | ||
| return current; | ||
| } | ||
|
|
||
| /** | ||
| * @return The total number of operations that have failed. | ||
| */ | ||
| public long getFailedCount() { | ||
| return failedCount; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| return builder.field("count", count) | ||
| .humanReadableField("time_in_millis", "time", new TimeValue(totalTimeInMillis, TimeUnit.MILLISECONDS)) | ||
| .field("current", current) | ||
| .field("failed", failedCount); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) return true; | ||
| if (o == null || getClass() != o.getClass()) return false; | ||
| OperationStats that = (OperationStats) o; | ||
| return Objects.equals(count, that.count) | ||
| && Objects.equals(totalTimeInMillis, that.totalTimeInMillis) | ||
| && Objects.equals(failedCount, that.failedCount) | ||
| && Objects.equals(current, that.current); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(count, totalTimeInMillis, failedCount, current); | ||
| } | ||
| } |
Oops, something went wrong.