Merge branch 'main' into fail_replica2

Signed-off-by: Ankit Kala <ankikala@amazon.com>
opensearch-project · Aug 31, 2023 · 75ad56c · 75ad56c
2 parents 5aace0a + ff65403
commit 75ad56c
Show file tree

Hide file tree

Showing 99 changed files with 2,024 additions and 262 deletions.
diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml
@@ -15,6 +15,15 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Increase swapfile
+        run: |
+          sudo swapoff -a
+          sudo fallocate -l 10G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          sudo swapon --show
+
       - name: Run compatibility task
         run: ./gradlew checkCompatibility -i | tee $HOME/gradlew-check.out
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -78,17 +78,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583)
-- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221)
-- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195)
+- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221](https://github.com/opensearch-project/OpenSearch/pull/8221))
+- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195](https://github.com/opensearch-project/OpenSearch/pull/8195))
 - Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606))
-- Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
+- Disallow compression level to be set for default and best_compression index codecs ([#8737](https://github.com/opensearch-project/OpenSearch/pull/8737))
 - Prioritize replica shard movement during shard relocation ([#8875](https://github.com/opensearch-project/OpenSearch/pull/8875))
-- Introducing Default and Best Compression codecs as their algorithm name ([#9123]()https://github.com/opensearch-project/OpenSearch/pull/9123)
-- Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122]()https://github.com/opensearch-project/OpenSearch/pull/9122)
+- Introducing Default and Best Compression codecs as their algorithm name ([#9123](https://github.com/opensearch-project/OpenSearch/pull/9123))
+- Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122](https://github.com/opensearch-project/OpenSearch/pull/9122))
 - [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223))
 - [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212))
 - [Feature] Expose term frequency in Painless script score context ([#9081](https://github.com/opensearch-project/OpenSearch/pull/9081))
 - Add support for reading partial files to HDFS repository ([#9513](https://github.com/opensearch-project/OpenSearch/issues/9513))
+- Add support for extensions to search responses using SearchExtBuilder ([#9379](https://github.com/opensearch-project/OpenSearch/pull/9379))
+- [BWC and API enforcement] Decorate the existing APIs with proper annotations (part 1) ([#9520](https://github.com/opensearch-project/OpenSearch/pull/9520))
+- Add concurrent segment search related metrics to node and index stats ([#9622](https://github.com/opensearch-project/OpenSearch/issues/9622))
 - Decouple replication lag from logic to fail stale replicas ([#9507](https://github.com/opensearch-project/OpenSearch/pull/9507))
 
 ### Dependencies
@@ -181,4 +184,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
diff --git a/.../main/java/org/opensearch/benchmark/index/mapper/CustomBinaryDocValuesFieldBenchmark.java b/.../main/java/org/opensearch/benchmark/index/mapper/CustomBinaryDocValuesFieldBenchmark.java
@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.benchmark.index.mapper;
+
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.index.mapper.BinaryFieldMapper;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 1)
+@Measurement(iterations = 1)
+@Fork(1)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@SuppressWarnings("unused") // invoked by benchmarking framework
+public class CustomBinaryDocValuesFieldBenchmark {
+
+    static final String FIELD_NAME = "dummy";
+    static final String SEED_VALUE = "seed";
+
+    @Benchmark
+    public void add(CustomBinaryDocValuesFieldBenchmark.BenchmarkParameters parameters, Blackhole blackhole) {
+        // Don't use the parameter binary doc values object.
+        // Start with a fresh object every call and add maximum number of entries
+        BinaryFieldMapper.CustomBinaryDocValuesField customBinaryDocValuesField = new BinaryFieldMapper.CustomBinaryDocValuesField(
+            FIELD_NAME,
+            new BytesRef(SEED_VALUE).bytes
+        );
+        for (int i = 0; i < parameters.maximumNumberOfEntries; ++i) {
+            ThreadLocalRandom.current().nextBytes(parameters.bytes);
+            customBinaryDocValuesField.add(parameters.bytes);
+        }
+    }
+
+    @Benchmark
+    public void binaryValue(CustomBinaryDocValuesFieldBenchmark.BenchmarkParameters parameters, Blackhole blackhole) {
+        blackhole.consume(parameters.customBinaryDocValuesField.binaryValue());
+    }
+
+    @State(Scope.Benchmark)
+    public static class BenchmarkParameters {
+        @Param({ "8", "32", "128", "512" })
+        int maximumNumberOfEntries;
+
+        @Param({ "8", "32", "128", "512" })
+        int entrySize;
+
+        BinaryFieldMapper.CustomBinaryDocValuesField customBinaryDocValuesField;
+        byte[] bytes;
+
+        @Setup
+        public void setup() {
+            customBinaryDocValuesField = new BinaryFieldMapper.CustomBinaryDocValuesField(FIELD_NAME, new BytesRef(SEED_VALUE).bytes);
+            bytes = new byte[entrySize];
+            for (int i = 0; i < maximumNumberOfEntries; ++i) {
+                ThreadLocalRandom.current().nextBytes(bytes);
+                customBinaryDocValuesField.add(bytes);
+            }
+        }
+    }
+}
diff --git a/build.gradle b/build.gradle
@@ -501,6 +501,7 @@ subprojects {
         includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests")
         includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
         includeClasses.add("org.opensearch.index.shard.IndexShardIT")
+        includeClasses.add("org.opensearch.index.shard.RemoteIndexShardTests")
         includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests")
         includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests")
         includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT")

diff --git a/libs/common/src/main/java/org/opensearch/common/CheckedConsumer.java b/libs/common/src/main/java/org/opensearch/common/CheckedConsumer.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.Consumer;
 
 /**
  * A {@link Consumer}-like interface which allows throwing checked exceptions.
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface CheckedConsumer<T, E extends Exception> {
     void accept(T t) throws E;

diff --git a/libs/common/src/main/java/org/opensearch/common/action/ActionFuture.java b/libs/common/src/main/java/org/opensearch/common/action/ActionFuture.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.common.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 import java.util.concurrent.Future;
@@ -42,6 +43,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ActionFuture<T> extends Future<T> {
 
     /**

diff --git a/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common.lifecycle;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Lifecycle state. Allows the following transitions:
  * <ul>
@@ -73,15 +75,17 @@
  * }
  * </pre>
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Lifecycle {
 
     /**
      * State in the lifecycle
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum State {
         INITIALIZED,
         STOPPED,

diff --git a/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java
@@ -32,13 +32,15 @@
 
 package org.opensearch.common.lifecycle;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 
 /**
  * Base interface for a lifecycle component.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface LifecycleComponent extends Releasable {
 
     Lifecycle.State lifecycleState();

diff --git a/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java b/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common.unit;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
@@ -41,6 +43,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TimeValue implements Comparable<TimeValue> {
 
     /** How many nano-seconds in one milli-second */

diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -33,6 +33,7 @@
 package org.opensearch;
 
 import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 
@@ -50,6 +51,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Version implements Comparable<Version>, ToXContentFragment {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA

diff --git a/libs/core/src/main/java/org/opensearch/core/ParseField.java b/libs/core/src/main/java/org/opensearch/core/ParseField.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.core;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.XContentLocation;
 
@@ -43,7 +44,11 @@
 /**
  * Holds a field that can be found in a request while parsing and its different
  * variants, which may be deprecated.
+ *
+ * @opensearch.api
+ *
  */
+@PublicApi(since = "1.0.0")
 public class ParseField {
     private final String name;
     private final String[] deprecatedNames;

diff --git a/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java b/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java
@@ -37,6 +37,7 @@
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -48,6 +49,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ActionListener<Response> {
     /**
      * Handle action response. This response may constitute a failure or a

diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.BytesStream;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.util.ByteArray;
@@ -50,8 +51,9 @@
 /**
  * A reference to bytes.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface BytesReference extends Comparable<BytesReference>, ToXContentFragment {
 
     /**

diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.core.common.io.stream;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -45,15 +47,17 @@
  * The registration is keyed by the combination of the category class of {@link NamedWriteable}, and a name unique
  * to that category.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NamedWriteableRegistry {
 
     /**
      * An entry in the registry, made up of a category class and name, and a reader for that category class.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Entry {
 
         /** The superclass of a {@link NamedWriteable} which will be read by {@link #reader}. */

diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
@@ -46,6 +46,7 @@
 import org.opensearch.Version;
 import org.opensearch.common.CharArrays;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -104,8 +105,9 @@
  * lists, either by storing {@code List}s internally or just converting to and from a {@code List} when calling. This comment is repeated
  * on {@link StreamInput}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class StreamInput extends InputStream {
 
     private Version version = Version.CURRENT;

diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
@@ -45,6 +45,7 @@
 import org.opensearch.Version;
 import org.opensearch.common.CharArrays;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -96,8 +97,9 @@
  * lists, either by storing {@code List}s internally or just converting to and from a {@code List} when calling. This comment is repeated
  * on {@link StreamInput}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class StreamOutput extends OutputStream {
 
     private static final int MAX_NESTED_EXCEPTION_LEVEL = 100;

diff --git a/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java b/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java
@@ -32,15 +32,18 @@
 
 package org.opensearch.core.common.settings;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.Closeable;
 import java.util.Arrays;
 import java.util.Objects;
 
 /**
  * A String implementations which allows clearing the underlying char array.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class SecureString implements CharSequence, Closeable {
 
     private char[] chars;