Skip to content

Commit

Permalink
Merge branch 'main' into search-idle
Browse files Browse the repository at this point in the history
Signed-off-by: Ruirui Zhang <mariazrr@amazon.com>
  • Loading branch information
ruai0511 committed Mar 21, 2024
2 parents ef646c6 + 7bd3715 commit 2d9bb3d
Show file tree
Hide file tree
Showing 126 changed files with 5,939 additions and 333 deletions.
1 change: 1 addition & 0 deletions .ci/bwcVersions
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,4 @@ BWC_VERSION:
- "2.12.0"
- "2.12.1"
- "2.13.0"
- "2.14.0"
6 changes: 3 additions & 3 deletions .github/workflows/version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:

- name: Create PR for BASE
id: base_pr
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
base: ${{ env.BASE }}
branch: 'create-pull-request/patch-${{ env.BASE }}'
Expand All @@ -88,7 +88,7 @@ jobs:

- name: Create PR for BASE_X
id: base_x_pr
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
base: ${{ env.BASE_X }}
branch: 'create-pull-request/patch-${{ env.BASE_X }}'
Expand All @@ -114,7 +114,7 @@ jobs:

- name: Create PR for main
id: main_pr
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
base: main
branch: 'create-pull-request/patch-main'
Expand Down
9 changes: 8 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Tiered caching] Add serializer integration to allow ehcache disk cache to use non-primitive values ([#12709](https://github.com/opensearch-project/OpenSearch/pull/12709))
- [Admission Control] Integrated IO Based AdmissionController to AdmissionControl Framework ([#12583](https://github.com/opensearch-project/OpenSearch/pull/12583))
- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
- Introduce a new setting `index.check_pending_flush.enabled` to expose the ability to disable the check for pending flushes by write threads ([#12710](https://github.com/opensearch-project/OpenSearch/pull/12710))
- Built-in secure transports support ([#12435](https://github.com/opensearch-project/OpenSearch/pull/12435))
- Lightweight Transport action to verify local term before fetching cluster-state from remote ([#12252](https://github.com/opensearch-project/OpenSearch/pull/12252/))
- Integrate with admission controller for cluster-manager Read API. ([#12496](https://github.com/opensearch-project/OpenSearch/pull/12496))

### Dependencies
- Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288))
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289))
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.37.1 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289), [#12365](https://github.com/opensearch-project/OpenSearch/pull/12365))
- Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))
- Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367))
- Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372))
Expand All @@ -142,6 +146,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `aws-sdk-java` from 2.20.55 to 2.20.86 ([#12251](https://github.com/opensearch-project/OpenSearch/pull/12251))
- Bump `reactor-netty` from 1.1.15 to 1.1.17 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
- Bump `reactor` from 3.5.14 to 3.5.15 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
- Bump `peter-evans/create-pull-request` from 5 to 6 ([#12724](https://github.com/opensearch-project/OpenSearch/pull/12724))
- Bump `org.apache.commons:commons-configuration2` from 2.9.0 to 2.10.0 ([#12721](https://github.com/opensearch-project/OpenSearch/pull/12721))
- Bump `com.azure:azure-json` from 1.0.1 to 1.1.0 ([#12723](https://github.com/opensearch-project/OpenSearch/pull/12723))

### Changed
- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
Expand Down
1 change: 1 addition & 0 deletions libs/core/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2);
public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2);
public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
public static final Version CURRENT = V_3_0_0;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
* Copyright 2015-2017 floragunn GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

package org.opensearch.http.netty4.ssl;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.BigArrays;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.http.HttpChannel;
import org.opensearch.http.HttpHandlingSettings;
import org.opensearch.http.netty4.Netty4HttpChannel;
import org.opensearch.http.netty4.Netty4HttpServerTransport;
import org.opensearch.plugins.SecureTransportSettingsProvider;
import org.opensearch.telemetry.tracing.Tracer;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.SharedGroupFactory;
import org.opensearch.transport.netty4.ssl.SslUtils;

import javax.net.ssl.SSLEngine;

import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.DecoderException;
import io.netty.handler.ssl.ApplicationProtocolNames;
import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler;
import io.netty.handler.ssl.SslHandler;

/**
* @see <a href="https://github.com/opensearch-project/security/blob/d526c9f6c2a438c14db8b413148204510b9fe2e2/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java">SecuritySSLNettyHttpServerTransport</a>
*/
public class SecureNetty4HttpServerTransport extends Netty4HttpServerTransport {
private static final Logger logger = LogManager.getLogger(SecureNetty4HttpServerTransport.class);
private final SecureTransportSettingsProvider secureTransportSettingsProvider;
private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;

public SecureNetty4HttpServerTransport(
final Settings settings,
final NetworkService networkService,
final BigArrays bigArrays,
final ThreadPool threadPool,
final NamedXContentRegistry namedXContentRegistry,
final Dispatcher dispatcher,
final ClusterSettings clusterSettings,
final SharedGroupFactory sharedGroupFactory,
final SecureTransportSettingsProvider secureTransportSettingsProvider,
final Tracer tracer
) {
super(
settings,
networkService,
bigArrays,
threadPool,
namedXContentRegistry,
dispatcher,
clusterSettings,
sharedGroupFactory,
tracer
);
this.secureTransportSettingsProvider = secureTransportSettingsProvider;
this.exceptionHandler = secureTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
.orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
}

@Override
public ChannelHandler configureServerChannelHandler() {
return new SslHttpChannelHandler(this, handlingSettings);
}

@Override
public void onException(HttpChannel channel, Exception cause0) {
Throwable cause = cause0;

if (cause0 instanceof DecoderException && cause0 != null) {
cause = cause0.getCause();
}

exceptionHandler.onError(cause);
logger.error("Exception during establishing a SSL connection: " + cause, cause);
super.onException(channel, cause0);
}

protected class SslHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler {
/**
* Application negotiation handler to select either HTTP 1.1 or HTTP 2 protocol, based
* on client/server ALPN negotiations.
*/
private class Http2OrHttpHandler extends ApplicationProtocolNegotiationHandler {
protected Http2OrHttpHandler() {
super(ApplicationProtocolNames.HTTP_1_1);
}

@Override
protected void configurePipeline(ChannelHandlerContext ctx, String protocol) throws Exception {
if (ApplicationProtocolNames.HTTP_2.equals(protocol)) {
configureDefaultHttp2Pipeline(ctx.pipeline());
} else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) {
configureDefaultHttpPipeline(ctx.pipeline());
} else {
throw new IllegalStateException("Unknown application protocol: " + protocol);
}
}

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
super.exceptionCaught(ctx, cause);
Netty4HttpChannel channel = ctx.channel().attr(HTTP_CHANNEL_KEY).get();
if (channel != null) {
if (cause instanceof Error) {
onException(channel, new Exception(cause));
} else {
onException(channel, (Exception) cause);
}
}
}
}

protected SslHttpChannelHandler(final Netty4HttpServerTransport transport, final HttpHandlingSettings handlingSettings) {
super(transport, handlingSettings);
}

@Override
protected void initChannel(Channel ch) throws Exception {
super.initChannel(ch);

final SSLEngine sslEngine = secureTransportSettingsProvider.buildSecureHttpServerEngine(
settings,
SecureNetty4HttpServerTransport.this
).orElseGet(SslUtils::createDefaultServerSSLEngine);

final SslHandler sslHandler = new SslHandler(sslEngine);
ch.pipeline().addFirst("ssl_http", sslHandler);
}

@Override
protected void configurePipeline(Channel ch) {
ch.pipeline().addLast(new Http2OrHttpHandler());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,14 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.netty4.Netty4HttpServerTransport;
import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport;
import org.opensearch.plugins.NetworkPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.SecureTransportSettingsProvider;
import org.opensearch.telemetry.tracing.Tracer;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.netty4.Netty4Transport;
import org.opensearch.transport.netty4.ssl.SecureNetty4Transport;

import java.util.Arrays;
import java.util.Collections;
Expand All @@ -61,7 +64,9 @@
public class Netty4ModulePlugin extends Plugin implements NetworkPlugin {

public static final String NETTY_TRANSPORT_NAME = "netty4";
public static final String NETTY_SECURE_TRANSPORT_NAME = "netty4-secure";
public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
public static final String NETTY_SECURE_HTTP_TRANSPORT_NAME = "netty4-secure";

private final SetOnce<SharedGroupFactory> groupFactory = new SetOnce<>();

Expand Down Expand Up @@ -144,6 +149,65 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
);
}

@Override
public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
Settings settings,
ThreadPool threadPool,
BigArrays bigArrays,
PageCacheRecycler pageCacheRecycler,
CircuitBreakerService circuitBreakerService,
NamedXContentRegistry xContentRegistry,
NetworkService networkService,
HttpServerTransport.Dispatcher dispatcher,
ClusterSettings clusterSettings,
SecureTransportSettingsProvider secureTransportSettingsProvider,
Tracer tracer
) {
return Collections.singletonMap(
NETTY_SECURE_HTTP_TRANSPORT_NAME,
() -> new SecureNetty4HttpServerTransport(
settings,
networkService,
bigArrays,
threadPool,
xContentRegistry,
dispatcher,
clusterSettings,
getSharedGroupFactory(settings),
secureTransportSettingsProvider,
tracer
)
);
}

@Override
public Map<String, Supplier<Transport>> getSecureTransports(
Settings settings,
ThreadPool threadPool,
PageCacheRecycler pageCacheRecycler,
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry,
NetworkService networkService,
SecureTransportSettingsProvider secureTransportSettingsProvider,
Tracer tracer
) {
return Collections.singletonMap(
NETTY_SECURE_TRANSPORT_NAME,
() -> new SecureNetty4Transport(
settings,
Version.CURRENT,
threadPool,
networkService,
pageCacheRecycler,
namedWriteableRegistry,
circuitBreakerService,
getSharedGroupFactory(settings),
secureTransportSettingsProvider,
tracer
)
);
}

SharedGroupFactory getSharedGroupFactory(Settings settings) {
SharedGroupFactory groupFactory = this.groupFactory.get();
if (groupFactory != null) {
Expand Down
Loading

0 comments on commit 2d9bb3d

Please sign in to comment.