Merge branch 'master' into repository-encrypted-client-side

.ci/bwcVersions

@@ -17,7 +17,8 @@ BWC_VERSION:
   - "7.6.0"
   - "7.6.1"
   - "7.6.2"
-  - "7.6.3"
   - "7.7.0"
+  - "7.7.1"
   - "7.8.0"
+  - "7.9.0"
   - "8.0.0"

.github/ISSUE_TEMPLATE/bug.md

@@ -0,0 +1,44 @@
+---
+name: Bug
+about: "Report a confirmed bug. For unconfirmed bugs please
+ visit https://discuss.elastic.co/c/elasticsearch"
+labels: ">bug,needs:triage"
+
+---
+<!--
+GitHub is reserved for bug reports and feature requests; it is not the place
+for general questions. If you have a question or an unconfirmed bug , please
+visit the [forums](https://discuss.elastic.co/c/elasticsearch).  Please also
+check your OS is [supported](https://www.elastic.co/support/matrix#show_os).
+If it is not, the issue is likely to be closed.
+
+For security vulnerabilities please only send reports to security@elastic.co.
+See https://www.elastic.co/community/security for more information.
+
+Please fill in the following details to help us reproduce the bug:
+-->
+
+**Elasticsearch version** (`bin/elasticsearch --version`):
+
+**Plugins installed**: []
+
+**JVM version** (`java -version`):
+
+**OS version** (`uname -a` if on a Unix-like system):
+
+**Description of the problem including expected versus actual behavior**:
+
+**Steps to reproduce**:
+
+Please include a *minimal* but *complete* recreation of the problem,
+including (e.g.) index creation, mappings, settings, query etc.  The easier
+you make for us to reproduce it, the more likely that somebody will take the
+time to look at it.
+
+ 1.
+ 2.
+ 3.
+
+**Provide logs (if relevant)**:
+
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question
+    url: https://discuss.elastic.co/c/elasticsearch
+    about: Ask (and answer) questions here.
+  - name: Security Vulnerability
+    url: https://www.elastic.co/community/security
+    about: Send security vulnerability reports to security@elastic.co.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,21 @@
+---
+name: Feature Request
+about: Request a new feature we haven't thought of
+labels: ">enhancement,needs:triage"
+
+---
+<!--
+Please first search existing issues for the feature you are requesting;
+it may already exist, even as a closed issue.
+-->
+
+<!--
+Describe the feature.
+
+Please give us as much context as possible about the feature. For example,
+you could include a story about a time when you wanted to use the feature,
+and also tell us what you had to do instead. The last part is helpful
+because it gives us an idea of how much harder your life is without the
+feature.
+
+-->

.github/ISSUE_TEMPLATE/test-failure.md

@@ -0,0 +1,28 @@
+---
+name: Test Failure
+about: A test failure in CI
+labels: ">test-failure"
+
+---
+
+<!--
+Please fill out the following information, and ensure you have attempted
+to reproduce locally
+-->
+
+**Build scan**:
+
+**Repro line**:
+
+**Reproduces locally?**:
+
+**Applicable branches**:
+
+**Failure history**:
+<!--
+Link to build stats and possible indication of when this started failing and how often it fails
+<https://build-stats.elastic.co/app/kibana>
+-->
+**Failure excerpt**:
+
+

CONTRIBUTING.md

@@ -115,10 +115,20 @@ onwards and Eclipse 2020-3 and onwards.
 
     ./gradlew :run
 
-You can access Elasticsearch with:
+That'll spend a while building Elasticsearch and then it'll start Elasticsearch,
+writing its log above Gradle's status message. We log a lot of stuff on startup,
+specifically these lines tell you that Elasticsearch is ready:
+
+    [2020-05-29T14:50:35,167][INFO ][o.e.h.AbstractHttpServerTransport] [runTask-0] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
+    [2020-05-29T14:50:35,169][INFO ][o.e.n.Node               ] [runTask-0] started
+
+But to be honest its typically easier to wait until the console stopps scrolling
+and then run `curl` in another window like this:
 
     curl -u elastic:password localhost:9200
 
+
+
 ### Importing the project into IntelliJ IDEA
 
 Elasticsearch builds using Java 14. When importing into IntelliJ you will need

TESTING.asciidoc

@@ -253,10 +253,10 @@ To run all verification tasks, including static checks, unit tests, and integrat
 ---------------------------------------------------------------------------
 
 Note that this will also run the unit tests and precommit tasks first. If you want to just
-run the integration tests (because you are debugging them):
+run the in memory cluster integration tests (because you are debugging them):
 
 ---------------------------------------------------------------------------
-./gradlew integTest
+./gradlew internalClusterTest
 ---------------------------------------------------------------------------
 
 If you want to just run the precommit checks:

Vagrantfile

@@ -491,22 +491,12 @@ JAVA
       rm -rf /tmp/bats
     }
 
-    cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
-export ZIP=/elasticsearch/distribution/zip/build/distributions
-export TAR=/elasticsearch/distribution/tar/build/distributions
-export RPM=/elasticsearch/distribution/rpm/build/distributions
-export DEB=/elasticsearch/distribution/deb/build/distributions
-export PACKAGING_TESTS=/project/build/packaging/tests
-VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
-Defaults   env_keep += "ZIP"
-Defaults   env_keep += "TAR"
-Defaults   env_keep += "RPM"
-Defaults   env_keep += "DEB"
-Defaults   env_keep += "PACKAGING_ARCHIVES"
-Defaults   env_keep += "PACKAGING_TESTS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
+Defaults   env_keep += "BATS_PLUGINS"
+Defaults   env_keep += "BATS_UPGRADE"
+Defaults   env_keep += "PACKAGE_NAME"
 Defaults   env_keep += "JAVA_HOME"
 Defaults   env_keep += "SYSTEM_JAVA_HOME"
 SUDOERS_VARS

benchmarks/README.md

@@ -53,9 +53,9 @@ To get realistic results, you should exercise care when running benchmarks. Here
   `performance` CPU governor.
 * Vary the problem input size with `@Param`.
 * Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
-    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews
+    * Add `-prof gc` to the options to check whether the garbage collector runs during a microbenchmarks and skews
    your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats.
-    * Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
+    * Add `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
 * Have your benchmarks peer-reviewed.
 
 ### Don't

benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.time.ZoneId;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+@Fork(2)
+@Warmup(iterations = 10)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+public class RoundingBenchmark {
+    private static final DateFormatter FORMATTER = DateFormatter.forPattern("date_optional_time");
+
+    @Param({
+        "2000-01-01 to 2020-01-01", // A super long range
+        "2000-10-01 to 2000-11-01", // A whole month which is pretty believable
+        "2000-10-29 to 2000-10-30", // A date right around daylight savings time.
+        "2000-06-01 to 2000-06-02"  // A date fully in one time zone. Should be much faster than above.
+    })
+    public String range;
+
+    @Param({ "java time", "es" })
+    public String rounder;
+
+    @Param({ "UTC", "America/New_York" })
+    public String zone;
+
+    @Param({ "calendar year", "calendar hour", "10d", "5d", "1h" })
+    public String interval;
+
+    @Param({ "1", "10000", "1000000", "100000000" })
+    public int count;
+
+    private long min;
+    private long max;
+    private long[] dates;
+    private Supplier<Rounding.Prepared> rounderBuilder;
+
+    @Setup
+    public void buildDates() {
+        String[] r = range.split(" to ");
+        min = FORMATTER.parseMillis(r[0]);
+        max = FORMATTER.parseMillis(r[1]);
+        dates = new long[count];
+        long date = min;
+        long diff = (max - min) / dates.length;
+        for (int i = 0; i < dates.length; i++) {
+            if (date >= max) {
+                throw new IllegalStateException("made a bad date [" + date + "]");
+            }
+            dates[i] = date;
+            date += diff;
+        }
+        Rounding.Builder roundingBuilder;
+        if (interval.startsWith("calendar ")) {
+            roundingBuilder = Rounding.builder(
+                DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(interval.substring("calendar ".length()))
+            );
+        } else {
+            roundingBuilder = Rounding.builder(TimeValue.parseTimeValue(interval, "interval"));
+        }
+        Rounding rounding = roundingBuilder.timeZone(ZoneId.of(zone)).build();
+        switch (rounder) {
+            case "java time":
+                rounderBuilder = rounding::prepareJavaTime;
+                break;
+            case "es":
+                rounderBuilder = () -> rounding.prepare(min, max);
+                break;
+            default:
+                throw new IllegalArgumentException("Expectd rounder to be [java time] or [es]");
+        }
+    }
+
+    @Benchmark
+    public void round(Blackhole bh) {
+        Rounding.Prepared rounder = rounderBuilder.get();
+        for (int i = 0; i < dates.length; i++) {
+            bh.consume(rounder.round(dates[i]));
+        }
+    }
+
+    @Benchmark
+    public void nextRoundingValue(Blackhole bh) {
+        Rounding.Prepared rounder = rounderBuilder.get();
+        for (int i = 0; i < dates.length; i++) {
+            bh.consume(rounder.nextRoundingValue(dates[i]));
+        }
+    }
+}