TINKERPOP-2487 add stdev and percentile steps

docs/src/reference/the-traversal.asciidoc

@@ -1922,6 +1922,42 @@ link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gre
 link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#min-org.apache.tinkerpop.gremlin.process.traversal.Scope-++[`min(Scope)`],
 link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/Scope.html++[`Scope`]
 
+[[stdev-step]]
+=== Stdev Step
+
+The `stdev()`-step (*map*) operates on a stream of numbers and calculates the standard deviation of those numbers.
+
+[gremlin-groovy,modern]
+----
+g.V().values('ages')
+g.V().values('ages').stdev()
+----
+
+*Additional References*
+
+link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#stdev--++[`stdev()`],
+link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#stdev-org.apache.tinkerpop.gremlin.process.traversal.Scope-++[`stdev(Scope)`],
+link:++https://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/Scope.html++[`Scope`]
+
+[[percentile-step]]
+=== Percentile Step
+
+The `percentile()`-step (*map*) operates on a stream of numbers and determine the n-th percentile values.
+It accepts one or more integer values within range [0..100]. If there is only one percentile input, the step result would be a single number value.
+If multiple percentile inputs are provided, the step result would be a `Map<Integer, Number>`.
+
+[gremlin-groovy,modern]
+----
+g.V().values('ages')
+g.V().values('ages').percentile(50) <1>
+g.V().values('ages').percentile(50, 90) <2>
+----
+
+<1> Calculate the 50-th percentile / median value of all persons' ages. The result is a single value.
+
+<2> Calculate the 50-th and 90-th percentile value of persons' ages. The result is a map.
+
+
 [[none-step]]
 === None Step
 

docs/src/upgrade/release-3.5.x.asciidoc

@@ -366,6 +366,46 @@ return these types of results and not have to work around them:
 
 See: link:https://issues.apache.org/jira/browse/TINKERPOP-2395[TINKERPOP-2395]
 
+==== Statistical Steps
+
+Gremlin is commonly used in statistical analysis. Examples include product, standard deviation and percentile calculation.
+While there are workarounds to achieve statistical calculations, it is better to add dedicated steps for performance or user experience reasons.
+In 3.5.x version of Gremlin, below statistical steps are introduced.
+
+===== stdev() Step
+
+The `stdev()`-step (*map*) operates on a stream of numbers and calculates the standard deviation of those numbers.
+
+[source,text]
+----
+gremlin> g.V().values('ages')
+==>1
+==>2
+==>3
+gremlin> g.V().values('ages').stdev()
+==>0.816
+gremlin> g.V().values('ages').fold().stdev(Scope.local)
+==>0.816
+----
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-2487[TINKERPOP-2487]
+
+===== percentile() Step
+
+The `percentile()`-step (*map*) operates on a stream of numbers and determine the n-th percentile values.
+It accepts one or more integer values within range [0..100]. If there is only one percentile input, the step result would be a single number value.
+If multiple percentile inputs are provided, the step result would be a `Map<Integer, Number>`.
+
+[source,text]
+----
+gremlin> g.V().values('ages').percentile(50)
+==>2
+gremlin> g.V().values('ages').percentile(0, 100)
+==>[0: 1, 100: 3]
+----
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-2487[TINKERPOP-2487]
+
 ==== Gremlin Server Audit Logging
 
 The `authentication.enableAuditlog` configuration property is deprecated, but replaced by the `enableAuditLog` property

gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java

@@ -105,6 +105,8 @@
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderLocalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.PathStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.PercentileGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.PercentileLocalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyKeyStep;
@@ -115,6 +117,8 @@
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.SampleLocalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectOneStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.StdevGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.StdevLocalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumGlobalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumLocalStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.TailLocalStep;
@@ -987,6 +991,52 @@ public default <E2 extends Number> GraphTraversal<S, E2> mean(final Scope scope)
         return this.asAdmin().addStep(scope.equals(Scope.global) ? new MeanGlobalStep(this.asAdmin()) : new MeanLocalStep(this.asAdmin()));
     }
 
+    /**
+     * Calculate the standard deviation value in the stream.
+     *
+     * @return the traversal with an appended {@link StdevGlobalStep}.
+     * @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#stdev-step" target="_blank">Reference Documentation - Stdev Step</a>
+     */
+    public default <E2 extends Number> GraphTraversal<S, E2> stdev() {
+        this.asAdmin().getBytecode().addStep(Symbols.stdev);
+        return this.asAdmin().addStep(new StdevGlobalStep<>(this.asAdmin()));
+    }
+
+    /**
+     * Calculate the standard deviation value in the stream given the {@link Scope}.
+     *
+     * @return the traversal with an appended {@link StdevGlobalStep} or {@link StdevLocalStep} depending on the {@link Scope}.
+     * @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#stdev-step" target="_blank">Reference Documentation - Stdev Step</a>
+     */
+    public default <E2 extends Number> GraphTraversal<S, E2> stdev(final Scope scope) {
+        this.asAdmin().getBytecode().addStep(Symbols.stdev, scope);
+        return this.asAdmin().addStep(scope.equals(Scope.global) ? new StdevGlobalStep(this.asAdmin()) : new StdevLocalStep(this.asAdmin()));
+    }
+
+    /**
+     * Calculate required percentile value.
+     *
+     * @param n the quantile value must be an integer between 0 and 100, or an IllegalArgumentException will be thrown.
+     * @return the traversal with an appended {@link PercentileGlobalStep}.
+     * @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#percentile-step" target="_blank">Reference Documentation - Percentile Step</a>
+     */
+    public default GraphTraversal<S, Object> percentile(final int... n) {
+        this.asAdmin().getBytecode().addStep(Symbols.percentile);
+        return this.asAdmin().addStep(new PercentileGlobalStep(this.asAdmin(), n));
+    }
+
+    /**
+     * Calculate required percentile value.
+     *
+     * @param n the quantile value must be an integer between 0 and 100, or an IllegalArgumentException will be thrown.
+     * @return the traversal with an appended {@link PercentileGlobalStep} or {@link PercentileLocalStep} depending on the {@link Scope}.
+     * @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#percentile-step" target="_blank">Reference Documentation - Percentile Step</a>
+     */
+    public default GraphTraversal<S, Object> percentile(final Scope scope, final int... n) {
+        this.asAdmin().getBytecode().addStep(Symbols.percentile);
+        return this.asAdmin().addStep(scope.equals(Scope.global) ? new PercentileGlobalStep(this.asAdmin(), n) : new PercentileLocalStep(this.asAdmin(), n));
+    }
+
     /**
      * Organize objects in the stream into a {@code Map}. Calls to {@code group()} are typically accompanied with
      * {@link #by()} modulators which help specify how the grouping should occur.
@@ -3006,6 +3056,8 @@ private Symbols() {
         public static final String max = "max";
         public static final String min = "min";
         public static final String mean = "mean";
+        public static final String stdev = "stdev";
+        public static final String percentile = "percentile";
         public static final String group = "group";
         public static final String groupCount = "groupCount";
         public static final String tree = "tree";

gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/__.java

@@ -473,6 +473,34 @@ public static <A> GraphTraversal<A, Double> mean(final Scope scope) {
         return __.<A>start().mean(scope);
     }
 
+    /**
+     * @see GraphTraversal#stdev()
+     */
+    public static <A> GraphTraversal<A, Double> stdev() {
+        return __.<A>start().stdev();
+    }
+
+    /**
+     * @see GraphTraversal#stdev(Scope)
+     */
+    public static <A> GraphTraversal<A, Double> stdev(final Scope scope) {
+        return __.<A>start().stdev(scope);
+    }
+
+    /**
+     * @see GraphTraversal#percentile(int...)
+     */
+    public static <A> GraphTraversal<A, Object> percentile(final int... n) {
+        return __.<A>start().percentile(n);
+    }
+
+    /**
+     * @see GraphTraversal#percentile(Scope, int...)
+     */
+    public static <A> GraphTraversal<A, Object> percentile(final Scope scope, final int... n) {
+        return __.<A>start().percentile(scope, n);
+    }
+
     /**
      * @see GraphTraversal#group()
      */

gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PercentileGlobalStep.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
+import org.apache.tinkerpop.gremlin.util.NumberHelper;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+
+/**
+ * @author Junshi Guo
+ */
+public class PercentileGlobalStep<S extends Number> extends ReducingBarrierStep<S, Object> {
+
+    private static final Set<TraverserRequirement> REQUIREMENTS = EnumSet.of(TraverserRequirement.OBJECT, TraverserRequirement.BULK);
+    List<S> buffer;
+    private final int[] percentiles;
+
+    public PercentileGlobalStep(final Traversal.Admin traversal, final int... percentiles) {
+        super(traversal);
+        this.buffer = new ArrayList<>();
+        this.percentiles = percentiles;
+        this.setSeedSupplier(() -> Collections.emptyMap());
+        this.setReducingBiOperator((x, y) -> Collections.emptyMap());
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        this.buffer.clear();
+    }
+
+    @Override
+    public void done() {
+        super.done();
+        this.buffer.clear();
+    }
+
+    @Override
+    public void processAllStarts() {
+        if (this.starts.hasNext()) {
+            super.processAllStarts();
+        }
+    }
+
+    @Override
+    public Object projectTraverser(final Traverser.Admin<S> traverser) {
+        Stream.iterate(0, n -> n + 1).limit(traverser.bulk()).forEach(n -> this.buffer.add(traverser.get()));
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public Set<TraverserRequirement> getRequirements() {
+        return REQUIREMENTS;
+    }
+
+    @Override
+    public Object generateFinalResult(final Object percentileNumber) {
+        if (this.percentiles.length > 1) {
+            return NumberHelper.percentiles(this.buffer, percentiles);
+        } else if (this.percentiles.length == 1) {
+            return NumberHelper.percentile(this.buffer, percentiles[0]);
+        }
+        return null;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        PercentileGlobalStep<?> that = (PercentileGlobalStep<?>) o;
+        return Arrays.equals(percentiles, that.percentiles);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + Arrays.hashCode(percentiles);
+        return result;
+    }
+}

gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PercentileLocalStep.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
+import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.tinkerpop.gremlin.util.NumberHelper.percentile;
+import static org.apache.tinkerpop.gremlin.util.NumberHelper.percentiles;
+
+/**
+ * @author Junshi Guo
+ */
+public class PercentileLocalStep<E extends Number, S extends Iterable<E>> extends ScalarMapStep<S, Object> {
+
+    private final int[] percentiles;
+
+    public PercentileLocalStep(final Traversal.Admin traversal, final int... percentiles) {
+        super(traversal);
+        this.percentiles = percentiles;
+        if (percentiles == null || percentiles.length == 0) {
+            throw new IllegalArgumentException("Percentile step should be provided at least one argument.");
+        }
+    }
+
+    @Override
+    protected Object map(final Traverser.Admin<S> traverser) {
+        final Iterator<E> iterator = traverser.get().iterator();
+        if (!iterator.hasNext()) {
+            throw FastNoSuchElementException.instance();
+        }
+        List<E> buffer = new ArrayList<>();
+        iterator.forEachRemaining(result -> buffer.add(result));
+        if (percentiles.length == 1) {
+            return percentile(buffer, percentiles[0]);
+        } else {
+            return percentiles(buffer, percentiles);
+        }
+    }
+
+    @Override
+    public Set<TraverserRequirement> getRequirements() {
+        return Collections.singleton(TraverserRequirement.OBJECT);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        PercentileLocalStep<?, ?> that = (PercentileLocalStep<?, ?>) o;
+        return Arrays.equals(percentiles, that.percentiles);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + Arrays.hashCode(percentiles);
+        return result;
+    }
+}