-
Notifications
You must be signed in to change notification settings - Fork 74
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implementation of statistics computation based on RxJava
This is a combination of 8 commits. The first commit's message is: Introduce BucketedOutputOperator and children This is the 2nd commit message: Update MetricsServiceImpl to use BucketedOutputOperator This is the 3rd commit message: Gauges buckets computation without custom operator This is to show what the process could look like, code for avail has not been updated, and code for counters not implemented. In this attempt, no custom operator is used. Instead, N(=number of buckets) requests for data are sent in parallel. Then the resulting observables are flat mapped to form an observable of bucket points. The bucket point observable emits just one value. The value is created by processing data points as they arrive, instead of buffering all the points for the bucket in an array. This has a huge implication on stats computation. Some stats algos are "storeless", which means you can get the exact result without buffering the data. Examples are min, max, avg. But some are not, like percentile. As a consequence, for median and 95th percentile, we need to use an approximative algorithm, provided by org.apache.commons.math3.stat.descriptive.rank.PSquarePercentile. See http://www.cse.wustl.edu/~jain/papers/psqr.htm for a description of the algorithm. On the upside, this algorithm allows to compute stats when the number of data points in the bucket is VERY large, with low memory consumption. On the downside, the result might look a bit inaccurate when there VERY few points in the buckets. It's also sensible to order of value submission. Note that the observable of bucket points will often be incorrectly ordered, as there's no guarantee in the order the responses will be processed. Side work: - log throwable at TRACE level for 500 responses - overloaded assertDoubleEquals so that caller can pass a failure message This is the 4th commit message: Refactor to use #collect for readability This is the 5th commit message: Refactor to use as single request to C* Sending N requests to C* might be bad wrt performance, especially if N(=number of buckets) is high This implementation uses #groupBy. Not sure how to make that work with #window This is the 6th commit message: Remove unused code This is the 7th commit message: Apply logic to avail metrics This is the 8th commit message: Final step, refactoring for better readability
- Loading branch information
1 parent
38ff28c
commit fba9fff
Showing
20 changed files
with
634 additions
and
732 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
core/metrics-core-api/src/main/java/org/hawkular/metrics/core/api/BucketPoint.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* | ||
| * Copyright 2014-2015 Red Hat, Inc. and/or its affiliates | ||
| * and other contributors as indicated by the @author tags. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.hawkular.metrics.core.api; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.function.BiFunction; | ||
|
|
||
| /** | ||
| * Statistics for metric data in a time range (bucket). | ||
| * | ||
| * @author Thomas Segismont | ||
| */ | ||
| public abstract class BucketPoint { | ||
| private final long start; | ||
| private final long end; | ||
|
|
||
| protected BucketPoint(long start, long end) { | ||
| this.start = start; | ||
| this.end = end; | ||
| } | ||
|
|
||
| /** | ||
| * @return start of the time range | ||
| */ | ||
| public long getStart() { | ||
| return start; | ||
| } | ||
|
|
||
| /** | ||
| * @return end of the time range | ||
| */ | ||
| public long getEnd() { | ||
| return end; | ||
| } | ||
|
|
||
| /** | ||
| * @return returns true if there was no data to build this bucket | ||
| */ | ||
| public abstract boolean isEmpty(); | ||
|
|
||
| /** | ||
| * Converts bucket points indexed by start time into a list, ordered by start time. Blanks will be filled with | ||
| * empty bucket points. | ||
| */ | ||
| public static <T extends BucketPoint> List<T> toList(Map<Long, T> pointMap, Buckets buckets, BiFunction<Long, | ||
| Long, T> emptyBucketFactory) { | ||
| List<T> result = new ArrayList<>(buckets.getCount()); | ||
| for (int index = 0; index < buckets.getCount(); index++) { | ||
| long from = buckets.getBucketStart(index); | ||
| T bucketPoint = pointMap.get(from); | ||
| if (bucketPoint == null) { | ||
| long to = from + buckets.getStep(); | ||
| bucketPoint = emptyBucketFactory.apply(from, to); | ||
| } | ||
| result.add(bucketPoint); | ||
| } | ||
| return result; | ||
| } | ||
| } |
Oops, something went wrong.