From e1fbfaf55e79585d7edf2e3af6145e650eea7067 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 May 2018 14:32:28 -0400 Subject: [PATCH 01/23] Introduce client feature tracking This commit introduces the ability for a client to communicate to the server features that it can support and for these features to be used in influencing the decisions that the server makes when communicating with the client. To this end we carry the features from the client to the underlying stream as we carry the version of the client today. This enables us to enhance the logic where we make protocol decisions on the basis of the version on the stream to also make protocol decisions on the basis of the features on the stream. With such functionality, the client can communicate to the server if it is a transport client, or if it has, for example, X-Pack installed. This enables us to support rolling upgrades from the OSS distribution to the default distribution without breaking client connectivity as we can now elect to serialize customs in the cluster state depending on whether or not the client reports to us using the feature capabilities that it can under these customs. This means that we would avoid sending a client pieces of the cluster state that it can not understand. However, we want to take care and always send the full cluster state during node-to-node communication as otherwise we would end up with different understanding of what is in the cluster state across nodes depending on which features they reported to have. This is why when deciding whether or not to write out a custom we always send the custom if the client is not a transport client and otherwise do not send the custom if the client is transport client that does not report to have the feature required by the custom. Co-authored-by: Yannick Welsch --- .../transport/netty4/ESLoggingHandler.java | 2 + .../client/transport/TransportClient.java | 10 +- .../elasticsearch/cluster/ClusterState.java | 66 +++- .../cluster/metadata/MetaData.java | 13 +- .../common/io/stream/StreamOutput.java | 11 + .../common/settings/ClusterSettings.java | 1 + .../PersistentTasksCustomMetaData.java | 3 +- .../org/elasticsearch/plugins/Plugin.java | 12 + .../elasticsearch/plugins/PluginsService.java | 25 +- .../elasticsearch/transport/TcpTransport.java | 74 +++- .../transport/TcpTransportChannel.java | 14 +- .../transport/TransportClientTests.java | 23 +- .../elasticsearch/cluster/ClusterStateIT.java | 351 ++++++++++++++++++ .../cluster/FeatureAwareTests.java | 172 +++++++++ .../transport/TcpTransportTests.java | 1 + .../elasticsearch/test/ESIntegTestCase.java | 98 ++++- .../test/client/RandomizingClient.java | 4 + .../AbstractSimpleTransportTestCase.java | 3 +- .../license/LicensesMetaData.java | 3 +- .../xpack/core/XPackClientPlugin.java | 27 +- .../elasticsearch/xpack/core/XPackPlugin.java | 25 +- .../xpack/core/ml/MlMetadata.java | 3 +- .../core/security/authc/TokenMetaData.java | 3 +- .../xpack/core/watcher/WatcherMetaData.java | 3 +- 24 files changed, 879 insertions(+), 68 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java index 47a31f268a6a8..18ab5cc0bd169 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java @@ -104,6 +104,8 @@ else if (readableBytes >= TcpHeader.HEADER_SIZE) { try (ThreadContext context = new ThreadContext(Settings.EMPTY)) { context.readHeaders(in); } + // now we decode the features + in.readStringArray(); // now we can decode the action name sb.append(", action: ").append(in.readString()); } diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java index f6d3a87f10da9..40904e9a8248b 100644 --- a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -98,6 +98,8 @@ public abstract class TransportClient extends AbstractClient { public static final Setting CLIENT_TRANSPORT_SNIFF = Setting.boolSetting("client.transport.sniff", false, Setting.Property.NodeScope); + public static final String TRANSPORT_CLIENT_FEATURE = "transport_client"; + private static PluginsService newPluginService(final Settings settings, Collection> plugins) { final Settings.Builder settingsBuilder = Settings.builder() .put(TcpTransport.PING_SCHEDULE.getKey(), "5s") // enable by default the transport schedule ping interval @@ -130,8 +132,12 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build(); } final PluginsService pluginsService = newPluginService(providedSettings, plugins); - final Settings settings = Settings.builder().put(defaultSettings).put(pluginsService.updatedSettings()).put(ThreadContext.PREFIX - + "." + "transport_client", true).build(); + final Settings settings = + Settings.builder() + .put(defaultSettings) + .put(pluginsService.updatedSettings()) + .put(TcpTransport.FEATURE_PREFIX + "." + TRANSPORT_CLIENT_FEATURE, true) + .build(); final List resourcesToClose = new ArrayList<>(); final ThreadPool threadPool = new ThreadPool(settings); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 2b991d1dc611a..6bc555eae0bd9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -61,6 +62,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -90,7 +92,51 @@ public class ClusterState implements ToXContentFragment, Diffable public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build(); - public interface Custom extends NamedDiffable, ToXContentFragment { + /** + * An interface that implementors use when a class requires a client to maybe have a feature. + */ + public interface FeatureAware { + + /** + * An optional feature that is required for the client to have. + * + * @return an empty optional if no feature is required otherwise a string representing the required feature + */ + default Optional getRequiredFeature() { + return Optional.empty(); + } + + /** + * Tests whether or not the custom should be serialized. The criteria are: + *
    + *
  • the output stream must be at least the minimum supported version of the custom
    • + *
  • the output stream must have the feature required by the custom (if any) or not be a transport client
    • + *
+ *

+ * That is, we only serialize customs to clients than can understand the custom based on the version of the client and the features + * that the client has. For transport clients we can be lenient in requiring a feature in which case we do not send the custom but + * for connected nodes we always require that the node has the required feature. + * + * @param out the output stream + * @param custom the custom to serialize + * @param the type of the custom + * @return true if the custom should be serialized and false otherwise + */ + static boolean shouldSerializeCustom(final StreamOutput out, final T custom) { + if (out.getVersion().before(custom.getMinimalSupportedVersion())) { + return false; + } + if (custom.getRequiredFeature().isPresent()) { + final String requiredFeature = custom.getRequiredFeature().get(); + // if it is a transport client we are lenient yet for a connected node it must have the required feature + return out.hasFeature(requiredFeature) || out.hasFeature(TransportClient.TRANSPORT_CLIENT_FEATURE) == false; + } + return true; + } + + } + + public interface Custom extends NamedDiffable, ToXContentFragment, FeatureAware { /** * Returns true iff this {@link Custom} is private to the cluster and should never be send to a client. @@ -99,6 +145,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment { default boolean isPrivate() { return false; } + } private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); @@ -244,6 +291,15 @@ public String toString() { sb.append("isa_ids ").append(indexMetaData.inSyncAllocationIds(shard)).append("\n"); } } + if (metaData.customs().isEmpty() == false) { + sb.append("metadata customs:\n"); + for (final ObjectObjectCursor cursor : metaData.customs()) { + final String type = cursor.key; + final MetaData.Custom custom = cursor.value; + sb.append(TAB).append(type).append(": ").append(custom); + } + sb.append("\n"); + } sb.append(blocks()); sb.append(nodes()); sb.append(routingTable()); @@ -691,14 +747,14 @@ public void writeTo(StreamOutput out) throws IOException { blocks.writeTo(out); // filter out custom states not supported by the other node int numberOfCustoms = 0; - for (ObjectCursor cursor : customs.values()) { - if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) { + for (final ObjectCursor cursor : customs.values()) { + if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { numberOfCustoms++; } } out.writeVInt(numberOfCustoms); - for (ObjectCursor cursor : customs.values()) { - if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) { + for (final ObjectCursor cursor : customs.values()) { + if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { out.writeNamedWriteable(cursor.value); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index b18c82712b37e..8c73222519502 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -24,6 +24,8 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterState.FeatureAware; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; @@ -117,9 +119,10 @@ public enum XContentContext { */ public static EnumSet ALL_CONTEXTS = EnumSet.allOf(XContentContext.class); - public interface Custom extends NamedDiffable, ToXContentFragment { + public interface Custom extends NamedDiffable, ToXContentFragment, ClusterState.FeatureAware { EnumSet context(); + } public static final Setting SETTING_READ_ONLY_SETTING = @@ -782,14 +785,14 @@ public void writeTo(StreamOutput out) throws IOException { } // filter out custom states not supported by the other node int numberOfCustoms = 0; - for (ObjectCursor cursor : customs.values()) { - if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) { + for (final ObjectCursor cursor : customs.values()) { + if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { numberOfCustoms++; } } out.writeVInt(numberOfCustoms); - for (ObjectCursor cursor : customs.values()) { - if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) { + for (final ObjectCursor cursor : customs.values()) { + if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { out.writeNamedWriteable(cursor.value); } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index ab0f47bf14d6d..f9617be4b56ed 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -58,10 +58,12 @@ import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; @@ -98,6 +100,7 @@ public abstract class StreamOutput extends OutputStream { } private Version version = Version.CURRENT; + private Set features = Collections.emptySet(); /** * The version of the node on the other side of this stream. @@ -113,6 +116,14 @@ public void setVersion(Version version) { this.version = version; } + public boolean hasFeature(final String feature) { + return this.features.contains(feature); + } + + public void setFeatures(final Set features) { + this.features = Collections.unmodifiableSet(new HashSet<>(features)); + } + public long position() throws IOException { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index d9cf0f630c0f2..e616613a425a9 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -379,6 +379,7 @@ public void apply(Settings value, Settings current, Settings previous) { ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING, EsExecutors.PROCESSORS_SETTING, ThreadContext.DEFAULT_HEADERS_SETTING, + TcpTransport.DEFAULT_FEATURES_SETTING, Loggers.LOG_DEFAULT_LEVEL_SETTING, Loggers.LOG_LEVEL_SETTING, NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING, diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 6d2c21a764ad5..b9c5b32306397 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -189,7 +190,7 @@ public long getNumberOfTasksOnNode(String nodeId, String taskName) { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_4_0; + return Version.V_6_3_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 82c8bf1bbcb18..6294f942d08cb 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -56,6 +56,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.UnaryOperator; /** @@ -79,6 +80,17 @@ */ public abstract class Plugin implements Closeable { + /** + * A feature exposed by the plugin. This should be used if a plugin exposes {@link org.elasticsearch.cluster.ClusterState.Custom} or + * {@link MetaData.Custom}; see also {@link org.elasticsearch.cluster.ClusterState.FeatureAware}. + * + * @return a feature set represented by this plugin, or the empty optional if the plugin does not expose cluster state or metadata + * customs + */ + protected Optional getFeature() { + return Optional.empty(); + } + /** * Node level guice modules. */ diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 68a19bb9bca9b..5b64b5be6390d 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -41,8 +41,10 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexModule; import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.transport.TcpTransport; import java.io.IOException; import java.lang.reflect.Constructor; @@ -57,16 +59,17 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; @@ -196,6 +199,7 @@ private static void logPluginInfo(final List pluginInfos, final Stri public Settings updatedSettings() { Map foundSettings = new HashMap<>(); + final Map features = new TreeMap<>(); final Settings.Builder builder = Settings.builder(); for (Tuple plugin : plugins) { Settings settings = plugin.v2().additionalSettings(); @@ -207,6 +211,23 @@ public Settings updatedSettings() { } } builder.put(settings); + final Optional maybeFeature = plugin.v2().getFeature(); + if (maybeFeature.isPresent()) { + final String feature = maybeFeature.get(); + if (features.containsKey(feature)) { + final String message = String.format( + Locale.ROOT, + "duplicate feature [%s] in plugin [%s], already added in [%s]", + feature, + plugin.v1().getName(), + features.get(feature)); + throw new IllegalArgumentException(message); + } + features.put(feature, plugin.v1().getName()); + } + } + for (final String feature : features.keySet()) { + builder.put(TcpTransport.FEATURE_PREFIX + "." + feature, true); } return builder.put(this.settings).build(); } diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 0b3d4e1b0a1ef..82b7aa8aed428 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.IntSet; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.Booleans; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -93,6 +94,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @@ -189,6 +191,10 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9); private static final BytesReference EMPTY_BYTES_REFERENCE = new BytesArray(new byte[0]); + public static final String FEATURE_PREFIX = "client.features"; + public static final Setting DEFAULT_FEATURES_SETTING = Setting.groupSetting(FEATURE_PREFIX + ".", Setting.Property.NodeScope); + private final String[] features; + private final CircuitBreakerService circuitBreakerService; // package visibility for tests protected final ScheduledPing scheduledPing; @@ -240,6 +246,17 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo this.networkService = networkService; this.transportName = transportName; defaultConnectionProfile = buildDefaultConnectionProfile(settings); + final Settings defaultFeatures = DEFAULT_FEATURES_SETTING.get(settings); + if (defaultFeatures == null) { + this.features = new String[0]; + } else { + defaultFeatures.names().forEach(key -> { + if (Booleans.parseBoolean(defaultFeatures.get(key)) == false) { + throw new IllegalArgumentException("feature settings must have default [true] value"); + } + }); + this.features = new TreeSet<>(defaultFeatures.names()).toArray(new String[defaultFeatures.names().size()]); + } } static ConnectionProfile buildDefaultConnectionProfile(Settings settings) { @@ -1103,6 +1120,9 @@ private void sendRequestToChannel(final DiscoveryNode node, final TcpChannel cha stream.setVersion(version); threadPool.getThreadContext().writeTo(stream); + if (version.onOrAfter(Version.V_7_0_0_alpha1)) { + stream.writeStringArray(features); + } stream.writeString(action); BytesReference message = buildMessage(requestId, status, node.getVersion(), request, stream); final TransportRequestOptions finalOptions = options; @@ -1135,15 +1155,22 @@ private void internalSendMessage(TcpChannel channel, BytesReference message, Sen * Sends back an error response to the caller via the given channel * * @param nodeVersion the caller node version + * @param features the caller features * @param channel the channel to send the response to * @param error the error to return * @param requestId the request ID this response replies to * @param action the action this response replies to */ - public void sendErrorResponse(Version nodeVersion, TcpChannel channel, final Exception error, final long requestId, - final String action) throws IOException { + public void sendErrorResponse( + final Version nodeVersion, + final Set features, + final TcpChannel channel, + final Exception error, + final long requestId, + final String action) throws IOException { try (BytesStreamOutput stream = new BytesStreamOutput()) { stream.setVersion(nodeVersion); + stream.setFeatures(features); RemoteTransportException tx = new RemoteTransportException( nodeName(), new TransportAddress(channel.getLocalAddress()), action, error); threadPool.getThreadContext().writeTo(stream); @@ -1163,15 +1190,28 @@ public void sendErrorResponse(Version nodeVersion, TcpChannel channel, final Exc /** * Sends the response to the given channel. This method should be used to send {@link TransportResponse} objects back to the caller. * - * @see #sendErrorResponse(Version, TcpChannel, Exception, long, String) for sending back errors to the caller + * @see #sendErrorResponse(Version, Set, TcpChannel, Exception, long, String) for sending back errors to the caller */ - public void sendResponse(Version nodeVersion, TcpChannel channel, final TransportResponse response, final long requestId, - final String action, TransportResponseOptions options) throws IOException { - sendResponse(nodeVersion, channel, response, requestId, action, options, (byte) 0); - } - - private void sendResponse(Version nodeVersion, TcpChannel channel, final TransportResponse response, final long requestId, - final String action, TransportResponseOptions options, byte status) throws IOException { + public void sendResponse( + final Version nodeVersion, + final Set features, + final TcpChannel channel, + final TransportResponse response, + final long requestId, + final String action, + final TransportResponseOptions options) throws IOException { + sendResponse(nodeVersion, features, channel, response, requestId, action, options, (byte) 0); + } + + private void sendResponse( + final Version nodeVersion, + final Set features, + final TcpChannel channel, + final TransportResponse response, + final long requestId, + final String action, + TransportResponseOptions options, + byte status) throws IOException { if (compress) { options = TransportResponseOptions.builder(options).withCompress(true).build(); } @@ -1185,6 +1225,7 @@ private void sendResponse(Version nodeVersion, TcpChannel channel, final Transpo } threadPool.getThreadContext().writeTo(stream); stream.setVersion(nodeVersion); + stream.setFeatures(features); BytesReference message = buildMessage(requestId, status, nodeVersion, response, stream); final TransportResponseOptions finalOptions = options; @@ -1546,13 +1587,19 @@ private void handleException(final TransportResponseHandler handler, Throwable e protected String handleRequest(TcpChannel channel, String profileName, final StreamInput stream, long requestId, int messageLengthBytes, Version version, InetSocketAddress remoteAddress, byte status) throws IOException { + final Set features; + if (version.onOrAfter(Version.V_7_0_0_alpha1)) { + features = Collections.unmodifiableSet(new TreeSet<>(Arrays.asList(stream.readStringArray()))); + } else { + features = Collections.emptySet(); + } final String action = stream.readString(); transportService.onRequestReceived(requestId, action); TransportChannel transportChannel = null; try { if (TransportStatus.isHandshake(status)) { final VersionHandshakeResponse response = new VersionHandshakeResponse(getCurrentVersion()); - sendResponse(version, channel, response, requestId, HANDSHAKE_ACTION_NAME, TransportResponseOptions.EMPTY, + sendResponse(version, features, channel, response, requestId, HANDSHAKE_ACTION_NAME, TransportResponseOptions.EMPTY, TransportStatus.setHandshake((byte) 0)); } else { final RequestHandlerRegistry reg = transportService.getRequestHandler(action); @@ -1564,7 +1611,7 @@ protected String handleRequest(TcpChannel channel, String profileName, final Str } else { getInFlightRequestBreaker().addWithoutBreaking(messageLengthBytes); } - transportChannel = new TcpTransportChannel(this, channel, transportName, action, requestId, version, profileName, + transportChannel = new TcpTransportChannel(this, channel, transportName, action, requestId, version, features, profileName, messageLengthBytes); final TransportRequest request = reg.newRequest(stream); request.remoteAddress(new TransportAddress(remoteAddress)); @@ -1575,7 +1622,8 @@ protected String handleRequest(TcpChannel channel, String profileName, final Str } catch (Exception e) { // the circuit breaker tripped if (transportChannel == null) { - transportChannel = new TcpTransportChannel(this, channel, transportName, action, requestId, version, profileName, 0); + transportChannel = + new TcpTransportChannel(this, channel, transportName, action, requestId, version, features, profileName, 0); } try { transportChannel.sendResponse(e); diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java index eb4c244c7a920..1bf1d027329b5 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java @@ -16,16 +16,20 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.transport; import org.elasticsearch.Version; import java.io.IOException; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public final class TcpTransportChannel implements TransportChannel { private final TcpTransport transport; private final Version version; + private final Set features; private final String action; private final long requestId; private final String profileName; @@ -34,9 +38,10 @@ public final class TcpTransportChannel implements TransportChannel { private final String channelType; private final TcpChannel channel; - TcpTransportChannel(TcpTransport transport, TcpChannel channel, String channelType, String action, - long requestId, Version version, String profileName, long reservedBytes) { + TcpTransportChannel(TcpTransport transport, TcpChannel channel, String channelType, String action, long requestId, Version version, + Set features, String profileName, long reservedBytes) { this.version = version; + this.features = features; this.channel = channel; this.transport = transport; this.action = action; @@ -59,7 +64,7 @@ public void sendResponse(TransportResponse response) throws IOException { @Override public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { try { - transport.sendResponse(version, channel, response, requestId, action, options); + transport.sendResponse(version, features, channel, response, requestId, action, options); } finally { release(false); } @@ -68,7 +73,7 @@ public void sendResponse(TransportResponse response, TransportResponseOptions op @Override public void sendResponse(Exception exception) throws IOException { try { - transport.sendErrorResponse(version, channel, exception, requestId, action); + transport.sendErrorResponse(version, features, channel, exception, requestId, action); } finally { release(true); } @@ -100,5 +105,6 @@ public Version getVersion() { public TcpChannel getChannel() { return channel; } + } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java index 1830698d90c6f..583dca95f65bd 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.transport.TcpTransport; import java.io.IOException; import java.util.Arrays; @@ -38,6 +39,8 @@ import java.util.concurrent.ExecutionException; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.object.HasToString.hasToString; public class TransportClientTests extends ESTestCase { @@ -64,13 +67,25 @@ public void testPluginNamedWriteablesRegistered() { } } - public void testDefaultHeaderContainsPlugins() { - Settings baseSettings = Settings.builder() + public void testSettingsContainsTransportClient() { + final Settings baseSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); try (TransportClient client = new MockTransportClient(baseSettings, Arrays.asList(MockPlugin.class))) { - ThreadContext threadContext = client.threadPool().getThreadContext(); - assertEquals("true", threadContext.getHeader("transport_client")); + final Settings settings = TcpTransport.DEFAULT_FEATURES_SETTING.get(client.settings()); + assertThat(settings.keySet(), hasItem("transport_client")); + assertThat(settings.get("transport_client"), equalTo("true")); + final ThreadContext threadContext = client.threadPool().getThreadContext(); + assertEquals("true", threadContext.getHeader("test")); + } + } + + public void testDefaultHeader() { + final Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .build(); + try (TransportClient client = new MockTransportClient(baseSettings, Arrays.asList(MockPlugin.class))) { + final ThreadContext threadContext = client.threadPool().getThreadContext(); assertEquals("true", threadContext.getHeader("test")); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java new file mode 100644 index 0000000000000..a66136c8f8234 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java @@ -0,0 +1,351 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexGraveyard; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; +import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.instanceOf; + +/** + * This test suite sets up a situation where the cluster has two plugins installed (node, and node-and-transport-client), and a transport + * client only has node-and-transport-client plugin installed. Each of these plugins inject customs into the cluster state and we want to + * check that the client can de-serialize a cluster state response based on the fact that the response should not contain customs that the + * transport client does not understand based on the fact that it only presents the node-and-transport-client-feature. + */ +@ESIntegTestCase.ClusterScope(scope = TEST) +public class ClusterStateIT extends ESIntegTestCase { + + public abstract static class Custom implements MetaData.Custom { + + private static final ParseField VALUE = new ParseField("value"); + + private final int value; + + int value() { + return value; + } + + Custom(final int value) { + this.value = value; + } + + Custom(final StreamInput in) throws IOException { + value = in.readInt(); + } + + @Override + public EnumSet context() { + return MetaData.ALL_CONTEXTS; + } + + @Override + public Diff diff(final MetaData.Custom previousState) { + return null; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeInt(value); + } + + public static int fromXContent(final XContentParser parser) throws IOException { + XContentParser.Token token; + int value = 0; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + switch (token) { + case FIELD_NAME: + currentFieldName = parser.currentName(); + break; + case VALUE_BOOLEAN: + if (VALUE.match(currentFieldName, parser.getDeprecationHandler())) { + value = parser.intValue(); + } + break; + } + } + return value; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(VALUE.getPreferredName(), value); + return builder; + } + + } + + public static class NodeCustom extends Custom { + + public static final String TYPE = "node"; + + NodeCustom(final int value) { + super(value); + } + + NodeCustom(final StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Optional getRequiredFeature() { + return Optional.of("node"); + } + + } + + public static class NodeAndTransportClientCustom extends Custom { + + public static final String TYPE = "node-and-transport-client"; + + NodeAndTransportClientCustom(final int value) { + super(value); + } + + public NodeAndTransportClientCustom(final StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Optional getRequiredFeature() { + return Optional.of("node-and-transport-client"); + } + + } + + public abstract static class CustomPlugin extends Plugin { + + private final List namedWritables = new ArrayList<>(); + private final List namedXContents = new ArrayList<>(); + + public CustomPlugin() { + registerBuiltinWritables(); + } + + protected void registerMetaDataCustom( + final String name, final Writeable.Reader reader, final CheckedFunction parser) { + namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader)); + namedXContents.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(name), parser)); + } + + protected abstract void registerBuiltinWritables(); + + protected abstract String getType(); + + protected abstract Custom getInstance(); + + @Override + public List getNamedWriteables() { + return namedWritables; + } + + @Override + public List getNamedXContent() { + return namedXContents; + } + + private final AtomicBoolean installed = new AtomicBoolean(); + + @Override + public Collection createComponents( + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final ResourceWatcherService resourceWatcherService, + final ScriptService scriptService, + final NamedXContentRegistry xContentRegistry, + final Environment environment, + final NodeEnvironment nodeEnvironment, + final NamedWriteableRegistry namedWriteableRegistry) { + clusterService.addListener(event -> { + final ClusterState state = event.state(); + if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { + return; + } + + final MetaData metaData = state.metaData(); + if (state.nodes().isLocalNodeElectedMaster()) { + if (metaData.custom(getType()) == null) { + if (installed.compareAndSet(false, true)) { + clusterService.submitStateUpdateTask("install-metadata-custom", new ClusterStateUpdateTask(Priority.URGENT) { + + @Override + public ClusterState execute(ClusterState currentState) { + if (currentState.custom(getType()) == null) { + final MetaData.Builder builder = MetaData.builder(currentState.metaData()); + builder.putCustom(getType(), getInstance()); + return ClusterState.builder(currentState).metaData(builder).build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + throw new AssertionError(e); + } + + }); + } + } + } + + }); + return Collections.emptyList(); + } + } + + public static class NodePlugin extends CustomPlugin { + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + static final Optional NODE_PLUGIN_FEATURE = Optional.of("node"); + + public Optional getFeature() { + return NODE_PLUGIN_FEATURE; + } + + static final int VALUE = randomInt(); + + @Override + protected void registerBuiltinWritables() { + registerMetaDataCustom(NodeCustom.TYPE, NodeCustom::new, parser -> new NodeCustom(NodeCustom.fromXContent(parser))); + } + + @Override + protected String getType() { + return NodeCustom.TYPE; + } + + @Override + protected Custom getInstance() { + return new NodeCustom(VALUE); + } + + } + + public static class NodeAndTransportClientPlugin extends CustomPlugin { + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + static final Optional FEATURE = Optional.of("node-and-transport-client"); + + @Override + protected Optional getFeature() { + return FEATURE; + } + + static final int VALUE = randomInt(); + + @Override + protected void registerBuiltinWritables() { + registerMetaDataCustom( + NodeAndTransportClientCustom.TYPE, + NodeAndTransportClientCustom::new, + parser -> new NodeAndTransportClientCustom(NodeAndTransportClientCustom.fromXContent(parser))); + } + + @Override + protected String getType() { + return NodeAndTransportClientCustom.TYPE; + } + + @Override + protected Custom getInstance() { + return new NodeAndTransportClientCustom(VALUE); + } + + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(NodePlugin.class, NodeAndTransportClientPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(NodeAndTransportClientPlugin.class); + } + + public void testOptionalCustoms() throws Exception { + // ensure that the customs are injected into the cluster state + assertBusy(() -> assertTrue(clusterService().state().metaData().customs().containsKey(NodeCustom.TYPE))); + assertBusy(() -> assertTrue(clusterService().state().metaData().customs().containsKey(NodeAndTransportClientCustom.TYPE))); + final ClusterStateResponse state = internalCluster().transportClient().admin().cluster().prepareState().get(); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, hasItem(IndexGraveyard.TYPE)); + assertThat(keys, not(hasItem(NodeCustom.TYPE))); + assertThat(keys, hasItem(NodeAndTransportClientCustom.TYPE)); + final MetaData.Custom actual = customs.get(NodeAndTransportClientCustom.TYPE); + assertThat(actual, instanceOf(NodeAndTransportClientCustom.class)); + assertThat(((NodeAndTransportClientCustom)actual).value(), equalTo(NodeAndTransportClientPlugin.VALUE)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java new file mode 100644 index 0000000000000..c4ecca1516e82 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java @@ -0,0 +1,172 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.Version; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.ClusterState.FeatureAware; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Optional; + +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; + +public class FeatureAwareTests extends ESTestCase { + + abstract static class Custom implements MetaData.Custom { + + private final Version version; + + Custom(final Version version) { + this.version = version; + } + + @Override + public EnumSet context() { + return MetaData.ALL_CONTEXTS; + } + + @Override + public Diff diff(final MetaData.Custom previousState) { + return null; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + return builder; + } + + @Override + public Version getMinimalSupportedVersion() { + return version; + } + + } + + static class NoRequiredFeatureCustom extends Custom { + + NoRequiredFeatureCustom(final Version version) { + super(version); + } + + @Override + public String getWriteableName() { + return "no-required-feature"; + } + + } + + static class RequiredFeatureCustom extends Custom { + + RequiredFeatureCustom(final Version version) { + super(version); + } + + @Override + public String getWriteableName() { + return null; + } + + @Override + public Optional getRequiredFeature() { + return Optional.of("required-feature"); + } + + } + + public void testVersion() { + final Version version = VersionUtils.randomVersion(random()); + for (final Custom custom : Arrays.asList(new NoRequiredFeatureCustom(version), new RequiredFeatureCustom(version))) { + { + final BytesStreamOutput out = new BytesStreamOutput(); + final Version afterVersion = randomVersionBetween(random(), version, Version.CURRENT); + out.setVersion(afterVersion); + if (custom.getRequiredFeature().isPresent()) { + out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); + } + assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + } + { + final BytesStreamOutput out = new BytesStreamOutput(); + final Version beforeVersion = + randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(version)); + out.setVersion(beforeVersion); + assertFalse(FeatureAware.shouldSerializeCustom(out, custom)); + } + } + } + + public void testFeature() { + final Version version = VersionUtils.randomVersion(random()); + final Version afterVersion = randomVersionBetween(random(), version, Version.CURRENT); + final Custom custom = new RequiredFeatureCustom(version); + { + // the feature is present and the client is not a transport client + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(afterVersion); + assertTrue(custom.getRequiredFeature().isPresent()); + out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); + assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + } + { + // the feature is present and the client is a transport client + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(afterVersion); + assertTrue(custom.getRequiredFeature().isPresent()); + out.setFeatures(new HashSet<>(Arrays.asList(custom.getRequiredFeature().get(), TransportClient.TRANSPORT_CLIENT_FEATURE))); + assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + } + } + + public void testMissingFeature() { + final Version version = VersionUtils.randomVersion(random()); + final Version afterVersion = randomVersionBetween(random(), version, Version.CURRENT); + final Custom custom = new RequiredFeatureCustom(version); + { + // the feature is missing but we should serialize it anyway because the client is not a transport client + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(afterVersion); + assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + } + { + // the feature is missing and we should not serialize it because the client is a transport client + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(afterVersion); + out.setFeatures(Collections.singleton(TransportClient.TRANSPORT_CLIENT_FEATURE)); + assertFalse(FeatureAware.shouldSerializeCustom(out, custom)); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java index 2cedb5419e08e..7e83e1cdc0bdf 100644 --- a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java @@ -227,6 +227,7 @@ public NodeChannels getConnection(DiscoveryNode node) { .streamInput(streamIn); } threadPool.getThreadContext().readHeaders(streamIn); + assertThat(streamIn.readStringArray(), equalTo(new String[0])); // features assertEquals("foobar", streamIn.readString()); Req readReq = new Req(""); readReq.readFrom(streamIn); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 505a5937d290b..8b58cea4d0a54 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -26,7 +26,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.http.HttpHost; import org.apache.lucene.search.Sort; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; @@ -68,12 +67,18 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotDeletionsInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.RepositoriesMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -105,6 +110,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; @@ -130,9 +136,11 @@ import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.ScriptMetaData; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.MockSearchService; import org.elasticsearch.search.SearchHit; @@ -1108,7 +1116,8 @@ protected void ensureClusterSizeConsistency() { protected void ensureClusterStateConsistency() throws IOException { if (cluster() != null && cluster().size() > 0) { final NamedWriteableRegistry namedWriteableRegistry = cluster().getNamedWriteableRegistry(); - ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState(); + final Client masterClient = client(); + ClusterState masterClusterState = masterClient.admin().cluster().prepareState().all().get().getState(); byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); // remove local node reference masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); @@ -1124,16 +1133,37 @@ protected void ensureClusterStateConsistency() throws IOException { final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; // Check that the non-master node has the same version of the cluster state as the master and // that the master node matches the master (otherwise there is no requirement for the cluster state to match) - if (masterClusterState.version() == localClusterState.version() && masterId.equals(localClusterState.nodes().getMasterNodeId())) { + if (masterClusterState.version() == localClusterState.version() + && masterId.equals(localClusterState.nodes().getMasterNodeId())) { try { - assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); - // We cannot compare serialization bytes since serialization order of maps is not guaranteed - // but we can compare serialization sizes - they should be the same - assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize); - // Compare JSON serialization - assertNull("clusterstate JSON serialization does not match", differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)); - } catch (AssertionError error) { - logger.error("Cluster state from master:\n{}\nLocal cluster state:\n{}", masterClusterState.toString(), localClusterState.toString()); + assertEquals("cluster state UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); + /* + * The cluster state received by the transport client can miss customs that the client does not understand. This + * means that we only expect equality in the cluster state including customs if the master client and the local + * client are of the same type (both or neither are transport clients). Otherwise, we can only assert equality + * modulo non-core customs. + */ + if (isTransportClient(masterClient) == isTransportClient(client)) { + // We cannot compare serialization bytes since serialization order of maps is not guaranteed + // but we can compare serialization sizes - they should be the same + assertEquals("cluster state size does not match", masterClusterStateSize, localClusterStateSize); + // Compare JSON serialization + assertNull( + "cluster state JSON serialization does not match", + differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)); + } else { + // remove non-core customs and compare the cluster states + assertNull( + "cluster state JSON serialization does not match (after removing some customs)", + differenceBetweenMapsIgnoringArrayOrder( + convertToMap(removePluginCustoms(masterClusterState)), + convertToMap(removePluginCustoms(localClusterState)))); + } + } catch (final AssertionError error) { + logger.error( + "Cluster state from master:\n{}\nLocal cluster state:\n{}", + masterClusterState.toString(), + localClusterState.toString()); throw error; } } @@ -1142,6 +1172,52 @@ protected void ensureClusterStateConsistency() throws IOException { } + /** + * Tests if the client is a transport client or wraps a transport client. + * + * @param client the client to test + * @return true if the client is a transport client or a wrapped transport client + */ + private boolean isTransportClient(final Client client) { + if (TransportClient.class.isAssignableFrom(client.getClass())) { + return true; + } else if (client instanceof RandomizingClient) { + return isTransportClient(((RandomizingClient) client).in()); + } + return false; + } + + private static final Set SAFE_METADATA_CUSTOMS = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(IndexGraveyard.TYPE, IngestMetadata.TYPE, RepositoriesMetaData.TYPE, ScriptMetaData.TYPE))); + + private static final Set SAFE_CUSTOMS = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(RestoreInProgress.TYPE, SnapshotDeletionsInProgress.TYPE, SnapshotsInProgress.TYPE))); + + /** + * Remove any customs except for customs that we know all clients understand. + * + * @param clusterState the cluster state to remove possibly-unknown customs from + * @return the cluster state with possibly-unknown customs removed + */ + private ClusterState removePluginCustoms(final ClusterState clusterState) { + final ClusterState.Builder builder = ClusterState.builder(clusterState); + clusterState.customs().keysIt().forEachRemaining(key -> { + if (SAFE_CUSTOMS.contains(key) == false) { + builder.removeCustom(key); + } + }); + final MetaData.Builder mdBuilder = MetaData.builder(clusterState.metaData()); + clusterState.metaData().customs().keysIt().forEachRemaining(key -> { + if (SAFE_METADATA_CUSTOMS.contains(key) == false) { + mdBuilder.removeCustom(key); + } + }); + builder.metaData(mdBuilder); + return builder.build(); + } + /** * Ensures the cluster is in a searchable state for the given indices. This means a searchable copy of each * shard is available on the cluster. diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java index e1a6ba030fde8..4c826101780a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java @@ -93,4 +93,8 @@ public String toString() { return "randomized(" + super.toString() + ")"; } + public Client in() { + return super.in(); + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 9a44f99c7c6b4..6c53ca6edb387 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -23,7 +23,6 @@ import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.Constants; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; @@ -32,7 +31,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.network.NetworkService; @@ -45,6 +43,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockServerSocket; import org.elasticsearch.node.Node; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java index 56475de123f3c..d9f7068b2181e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.license.License.OperationMode; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.EnumSet; @@ -23,7 +24,7 @@ /** * Contains metadata about registered licenses */ -public class LicensesMetaData extends AbstractNamedDiffable implements MetaData.Custom, +public class LicensesMetaData extends AbstractNamedDiffable implements XPackPlugin.XPackMetaDataCustom, MergableCustomMetaData { public static final String TYPE = "licenses"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 0b22cd86fe6a0..a96de96fd4f44 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.license.DeleteLicenseAction; @@ -28,6 +27,7 @@ import org.elasticsearch.license.PostStartBasicAction; import org.elasticsearch.license.PostStartTrialAction; import org.elasticsearch.license.PutLicenseAction; +import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -61,7 +61,6 @@ import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction; -import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; @@ -69,6 +68,7 @@ import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction; import org.elasticsearch.xpack.core.ml.action.KillProcessAction; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.PersistJobAction; import org.elasticsearch.xpack.core.ml.action.PostCalendarEventsAction; @@ -91,7 +91,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskStatus; import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage; -import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage; import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction; @@ -133,6 +132,8 @@ import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; +import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction; +import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction; import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage; import org.elasticsearch.xpack.core.watcher.WatcherMetaData; import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction; @@ -143,18 +144,25 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction; import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction; -import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction; -import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin { + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + static Optional X_PACK_FEATURE = Optional.of("x-pack"); + + @Override + protected Optional getFeature() { + return X_PACK_FEATURE; + } + private final Settings settings; public XPackClientPlugin(final Settings settings) { @@ -185,11 +193,10 @@ public Settings additionalSettings() { static Settings additionalSettings(final Settings settings, final boolean enabled, final boolean transportClientMode) { if (enabled && transportClientMode) { - final Settings.Builder builder = Settings.builder(); - builder.put(SecuritySettings.addTransportSettings(settings)); - builder.put(SecuritySettings.addUserSettings(settings)); - builder.put(ThreadContext.PREFIX + "." + "has_xpack", true); - return builder.build(); + return Settings.builder() + .put(SecuritySettings.addTransportSettings(settings)) + .put(SecuritySettings.addUserSettings(settings)) + .build(); } else { return Settings.EMPTY; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 9568a36551c83..602f4bdbc079b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -59,19 +59,15 @@ import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.watcher.WatcherMetaData; -import javax.security.auth.DestroyFailedException; - -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.security.AccessController; -import java.security.GeneralSecurityException; import java.security.PrivilegedAction; import java.time.Clock; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -316,4 +312,23 @@ public static Path resolveConfigFile(Environment env, String name) { } return config; } + + public interface XPackClusterStateCustom extends ClusterState.Custom { + + @Override + default Optional getRequiredFeature() { + return XPackClientPlugin.X_PACK_FEATURE; + } + + } + + public interface XPackMetaDataCustom extends MetaData.Custom { + + @Override + default Optional getRequiredFeature() { + return XPackClientPlugin.X_PACK_FEATURE; + } + + } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 6af323f1510e4..861f386a90966 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -27,6 +27,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; @@ -53,7 +54,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -public class MlMetadata implements MetaData.Custom { +public class MlMetadata implements XPackPlugin.XPackMetaDataCustom { private static final ParseField JOBS_FIELD = new ParseField("jobs"); private static final ParseField DATAFEEDS_FIELD = new ParseField("datafeeds"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java index 6bd6228f2efe1..46111b9b16cd1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java @@ -12,13 +12,14 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -public final class TokenMetaData extends AbstractNamedDiffable implements ClusterState.Custom { +public final class TokenMetaData extends AbstractNamedDiffable implements XPackPlugin.XPackClusterStateCustom { /** * The type of {@link ClusterState} data. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java index 3a490f08b79e5..9f014dee843c5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java @@ -13,12 +13,13 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.EnumSet; import java.util.Objects; -public class WatcherMetaData extends AbstractNamedDiffable implements MetaData.Custom { +public class WatcherMetaData extends AbstractNamedDiffable implements XPackPlugin.XPackMetaDataCustom { public static final String TYPE = "watcher"; From 82ed4d4ecb7a3bf842f1acddec6a267592e1c57e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 05:59:47 -0400 Subject: [PATCH 02/23] Remove irrelevant lines from test --- .../elasticsearch/client/transport/TransportClientTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java index 583dca95f65bd..1dc30e951b6d3 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java @@ -75,8 +75,6 @@ public void testSettingsContainsTransportClient() { final Settings settings = TcpTransport.DEFAULT_FEATURES_SETTING.get(client.settings()); assertThat(settings.keySet(), hasItem("transport_client")); assertThat(settings.get("transport_client"), equalTo("true")); - final ThreadContext threadContext = client.threadPool().getThreadContext(); - assertEquals("true", threadContext.getHeader("test")); } } From 6d529fee20a88a87592337e9a49fd24a3bb06b05 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:02:41 -0400 Subject: [PATCH 03/23] Add comment on use of TreeSet --- .../src/main/java/org/elasticsearch/transport/TcpTransport.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 82b7aa8aed428..d3d1b93810d68 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -255,6 +255,7 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo throw new IllegalArgumentException("feature settings must have default [true] value"); } }); + // use a sorted set to present the features in a consistent order this.features = new TreeSet<>(defaultFeatures.names()).toArray(new String[defaultFeatures.names().size()]); } } From 66f2ea95dc29438b1772321fc94afd7e13cae9d7 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:07:49 -0400 Subject: [PATCH 04/23] Remove unneeded parser in test --- .../elasticsearch/cluster/ClusterStateIT.java | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java index a66136c8f8234..aed3417939dae 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java @@ -106,25 +106,6 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeInt(value); } - public static int fromXContent(final XContentParser parser) throws IOException { - XContentParser.Token token; - int value = 0; - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - switch (token) { - case FIELD_NAME: - currentFieldName = parser.currentName(); - break; - case VALUE_BOOLEAN: - if (VALUE.match(currentFieldName, parser.getDeprecationHandler())) { - value = parser.intValue(); - } - break; - } - } - return value; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(VALUE.getPreferredName(), value); @@ -276,7 +257,12 @@ public Optional getFeature() { @Override protected void registerBuiltinWritables() { - registerMetaDataCustom(NodeCustom.TYPE, NodeCustom::new, parser -> new NodeCustom(NodeCustom.fromXContent(parser))); + registerMetaDataCustom( + NodeCustom.TYPE, + NodeCustom::new, + parser -> { + throw new IOException(new UnsupportedOperationException()); + }); } @Override @@ -308,7 +294,9 @@ protected void registerBuiltinWritables() { registerMetaDataCustom( NodeAndTransportClientCustom.TYPE, NodeAndTransportClientCustom::new, - parser -> new NodeAndTransportClientCustom(NodeAndTransportClientCustom.fromXContent(parser))); + parser -> { + throw new IOException(new UnsupportedOperationException()); + }); } @Override From 25a861c2a24bcf2ebf96daf1f12d80c45b9e9f28 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:08:43 -0400 Subject: [PATCH 05/23] Remove unneeded fields --- .../java/org/elasticsearch/cluster/ClusterStateIT.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java index aed3417939dae..d93a0680fbf98 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java @@ -246,11 +246,8 @@ public void onFailure(String source, Exception e) { public static class NodePlugin extends CustomPlugin { - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - static final Optional NODE_PLUGIN_FEATURE = Optional.of("node"); - public Optional getFeature() { - return NODE_PLUGIN_FEATURE; + return Optional.of("node"); } static final int VALUE = randomInt(); @@ -279,12 +276,9 @@ protected Custom getInstance() { public static class NodeAndTransportClientPlugin extends CustomPlugin { - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - static final Optional FEATURE = Optional.of("node-and-transport-client"); - @Override protected Optional getFeature() { - return FEATURE; + return Optional.of("node-and-transport-client"); } static final int VALUE = randomInt(); From 0b39ce997889035d44100c3a7cb8394eee155d76 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:11:07 -0400 Subject: [PATCH 06/23] Guard feature de-serialization by version --- .../org/elasticsearch/transport/netty4/ESLoggingHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java index 18ab5cc0bd169..62e52a8726fa2 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java @@ -105,7 +105,9 @@ else if (readableBytes >= TcpHeader.HEADER_SIZE) { context.readHeaders(in); } // now we decode the features - in.readStringArray(); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + in.readStringArray(); + } // now we can decode the action name sb.append(", action: ").append(in.readString()); } From 26071ff630e5b20be17a4f909416ce2e06691d59 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:20:01 -0400 Subject: [PATCH 07/23] Add Javadocs --- .../common/io/stream/StreamOutput.java | 15 +++++++++++++++ .../java/org/elasticsearch/plugins/Plugin.java | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index f9617be4b56ed..e8c4d197fda62 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -30,6 +30,8 @@ import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; @@ -116,11 +118,24 @@ public void setVersion(Version version) { this.version = version; } + /** + * Test if the stream has the specified feature. Features are used when serializing {@link ClusterState.Custom} or + * {@link MetaData.Custom}; see also {@link ClusterState.FeatureAware}. + * + * @param feature the feature to test + * @return true if the stream has the specified feature + */ public boolean hasFeature(final String feature) { return this.features.contains(feature); } + /** + * Set the features on the stream. See {@link StreamOutput#hasFeature(String)}. + * + * @param features the features on the stream + */ public void setFeatures(final Set features) { + assert this.features.isEmpty() : this.features; this.features = Collections.unmodifiableSet(new HashSet<>(features)); } diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 6294f942d08cb..0ef703448b799 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -81,8 +82,8 @@ public abstract class Plugin implements Closeable { /** - * A feature exposed by the plugin. This should be used if a plugin exposes {@link org.elasticsearch.cluster.ClusterState.Custom} or - * {@link MetaData.Custom}; see also {@link org.elasticsearch.cluster.ClusterState.FeatureAware}. + * A feature exposed by the plugin. This should be used if a plugin exposes {@link ClusterState.Custom} or {@link MetaData.Custom}; see + * also {@link ClusterState.FeatureAware}. * * @return a feature set represented by this plugin, or the empty optional if the plugin does not expose cluster state or metadata * customs From 8be176c24b6aa9e838e51b7c4a1ee0daeffaaecb Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:28:33 -0400 Subject: [PATCH 08/23] Randomly add feature to test --- .../test/java/org/elasticsearch/cluster/FeatureAwareTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java index c4ecca1516e82..696bf2f82faaa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java @@ -123,6 +123,9 @@ public void testVersion() { final Version beforeVersion = randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(version)); out.setVersion(beforeVersion); + if (custom.getRequiredFeature().isPresent() && randomBoolean()) { + out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); + } assertFalse(FeatureAware.shouldSerializeCustom(out, custom)); } } From 1f6682970001ac2430ece164630e74493ffc820b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 1 Jun 2018 06:35:52 -0400 Subject: [PATCH 09/23] Change prefix of feature setting --- .../src/main/java/org/elasticsearch/transport/TcpTransport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index d3d1b93810d68..df46c945540bf 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -191,7 +191,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9); private static final BytesReference EMPTY_BYTES_REFERENCE = new BytesArray(new byte[0]); - public static final String FEATURE_PREFIX = "client.features"; + public static final String FEATURE_PREFIX = "transport.features"; public static final Setting DEFAULT_FEATURES_SETTING = Setting.groupSetting(FEATURE_PREFIX + ".", Setting.Property.NodeScope); private final String[] features; From 4b87ed8488d30dbcfec7f7249c4cd93faf44eeb1 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 1 Jun 2018 15:14:35 +0200 Subject: [PATCH 10/23] add version and feature support to persistent tasks --- .../elasticsearch/cluster/ClusterState.java | 8 ++-- .../elasticsearch/cluster/NamedDiffable.java | 11 +----- .../cluster/RestoreInProgress.java | 8 +++- .../cluster/SnapshotsInProgress.java | 5 +++ .../cluster/metadata/IndexGraveyard.java | 9 +++-- .../cluster/metadata/MetaData.java | 4 +- .../metadata/RepositoriesMetaData.java | 6 +++ .../io/stream/VersionedNamedWriteable.java | 38 +++++++++++++++++++ .../elasticsearch/ingest/IngestMetadata.java | 6 +++ .../persistent/PersistentTaskParams.java | 5 ++- .../PersistentTasksCustomMetaData.java | 7 +++- .../elasticsearch/script/ScriptMetaData.java | 5 +++ .../cluster/ClusterChangedEventTests.java | 10 +++++ .../elasticsearch/cluster/ClusterStateIT.java | 13 ++++++- .../cluster/FeatureAwareTests.java | 12 +++--- .../cluster/SimpleClusterStateIT.java | 7 +++- .../cluster/service/ClusterSerivceTests.java | 6 +++ .../discovery/zen/ZenDiscoveryIT.java | 5 +++ .../gateway/GatewayMetaStateTests.java | 10 +++++ .../persistent/TestPersistentTasksPlugin.java | 16 +++++++- .../DedicatedClusterSnapshotRestoreIT.java | 27 ++++++++++++- .../xpack/core/XPackClientPlugin.java | 2 +- .../xpack/core/ml/action/OpenJobAction.java | 14 ++++++- .../xpack/core/rollup/job/RollupJob.java | 13 +++++++ 24 files changed, 210 insertions(+), 37 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 6bc555eae0bd9..276e00a2ba3db 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -22,7 +22,6 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -50,6 +49,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -122,7 +122,7 @@ default Optional getRequiredFeature() { * @param the type of the custom * @return true if the custom should be serialized and false otherwise */ - static boolean shouldSerializeCustom(final StreamOutput out, final T custom) { + static boolean shouldSerialize(final StreamOutput out, final T custom) { if (out.getVersion().before(custom.getMinimalSupportedVersion())) { return false; } @@ -748,13 +748,13 @@ public void writeTo(StreamOutput out) throws IOException { // filter out custom states not supported by the other node int numberOfCustoms = 0; for (final ObjectCursor cursor : customs.values()) { - if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { + if (FeatureAware.shouldSerialize(out, cursor.value)) { numberOfCustoms++; } } out.writeVInt(numberOfCustoms); for (final ObjectCursor cursor : customs.values()) { - if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { + if (FeatureAware.shouldSerialize(out, cursor.value)) { out.writeNamedWriteable(cursor.value); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java index b548b49fe1910..dfccd259780b8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java @@ -19,17 +19,10 @@ package org.elasticsearch.cluster; -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; /** * Diff that also support NamedWriteable interface */ -public interface NamedDiffable extends Diffable, NamedWriteable { - /** - * The minimal version of the recipient this custom object can be sent to - */ - default Version getMinimalSupportedVersion() { - return Version.CURRENT.minimumIndexCompatibilityVersion(); - } +public interface NamedDiffable extends Diffable, VersionedNamedWriteable { } diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index 5c036f94285e0..ad922ad36df65 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -20,14 +20,15 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState.Custom; -import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.Snapshot; import java.io.IOException; import java.util.ArrayList; @@ -382,6 +383,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { return readDiffFrom(Custom.class, TYPE, in); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 74b748e19a7ee..a48164ffe828e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -395,6 +395,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { return readDiffFrom(Custom.class, TYPE, in); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java index 9167b28a67b86..6abbf86edd972 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.common.ParseField; @@ -34,8 +35,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.ArrayList; @@ -44,7 +43,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Objects; -import java.util.concurrent.TimeUnit; /** * A collection of tombstones for explicitly marking indices as deleted in the cluster state. @@ -97,6 +95,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + @Override public EnumSet context() { return MetaData.API_AND_GATEWAY; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 8c73222519502..5657873987e18 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -786,13 +786,13 @@ public void writeTo(StreamOutput out) throws IOException { // filter out custom states not supported by the other node int numberOfCustoms = 0; for (final ObjectCursor cursor : customs.values()) { - if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { + if (FeatureAware.shouldSerialize(out, cursor.value)) { numberOfCustoms++; } } out.writeVInt(numberOfCustoms); for (final ObjectCursor cursor : customs.values()) { - if (FeatureAware.shouldSerializeCustom(out, cursor.value)) { + if (FeatureAware.shouldSerialize(out, cursor.value)) { out.writeNamedWriteable(cursor.value); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index c813ba76e82dd..abaca292ed6d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData.Custom; @@ -103,6 +104,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + public RepositoriesMetaData(StreamInput in) throws IOException { RepositoryMetaData[] repository = new RepositoryMetaData[in.readVInt()]; for (int i = 0; i < repository.length; i++) { diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java new file mode 100644 index 0000000000000..9eea2c00d56a6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.io.stream; + +import org.elasticsearch.Version; + +/** + * A {@link NamedWriteable} that has a minimum version associated with it. + */ +public interface VersionedNamedWriteable extends NamedWriteable { + + /** + * Returns the name of the writeable object + */ + String getWriteableName(); + + /** + * The minimal version of the recipient this object can be sent to + */ + Version getMinimalSupportedVersion(); +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java index ca8a5df845014..6bca478f52b7b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest; +import org.elasticsearch.Version; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.NamedDiff; @@ -69,6 +70,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + public Map getPipelines() { return pipelines; } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java index a475a7cde174a..c91727a913f3a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java @@ -19,12 +19,13 @@ package org.elasticsearch.persistent; -import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.common.xcontent.ToXContentObject; /** * Parameters used to start persistent task */ -public interface PersistentTaskParams extends NamedWriteable, ToXContentObject { +public interface PersistentTaskParams extends VersionedNamedWriteable, ToXContentObject, ClusterState.FeatureAware { } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index b9c5b32306397..de6ab0f7b879b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -49,8 +49,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -500,7 +500,10 @@ public PersistentTasksCustomMetaData(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeLong(lastAllocationId); - out.writeMap(tasks, StreamOutput::writeString, (stream, value) -> value.writeTo(stream)); + Map> filteredTasks = tasks.values().stream() + .filter(t -> ClusterState.FeatureAware.shouldSerialize(out, t.getParams())) + .collect(Collectors.toMap(PersistentTask::getId, Function.identity())); + out.writeMap(filteredTasks, StreamOutput::writeString, (stream, value) -> value.writeTo(stream)); } public static NamedDiff readDiffFrom(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java index 9505875ae1ebc..ea230b9e3a669 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -383,6 +383,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + @Override public EnumSet context() { return MetaData.ALL_CONTEXTS; diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java index b7ea45dd13a3d..f79fef74e917f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java @@ -308,6 +308,11 @@ public String getWriteableName() { return "2"; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); @@ -324,6 +329,11 @@ public String getWriteableName() { return "1"; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java index d93a0680fbf98..a2bcc424db51b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexGraveyard; @@ -32,7 +33,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -42,7 +42,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; @@ -131,6 +130,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public Optional getRequiredFeature() { return Optional.of("node"); @@ -155,6 +159,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public Optional getRequiredFeature() { return Optional.of("node-and-transport-client"); diff --git a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java index 696bf2f82faaa..8f6afee6b9e03 100644 --- a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java @@ -116,7 +116,7 @@ public void testVersion() { if (custom.getRequiredFeature().isPresent()) { out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); } - assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + assertTrue(FeatureAware.shouldSerialize(out, custom)); } { final BytesStreamOutput out = new BytesStreamOutput(); @@ -126,7 +126,7 @@ public void testVersion() { if (custom.getRequiredFeature().isPresent() && randomBoolean()) { out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); } - assertFalse(FeatureAware.shouldSerializeCustom(out, custom)); + assertFalse(FeatureAware.shouldSerialize(out, custom)); } } } @@ -141,7 +141,7 @@ public void testFeature() { out.setVersion(afterVersion); assertTrue(custom.getRequiredFeature().isPresent()); out.setFeatures(Collections.singleton(custom.getRequiredFeature().get())); - assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + assertTrue(FeatureAware.shouldSerialize(out, custom)); } { // the feature is present and the client is a transport client @@ -149,7 +149,7 @@ public void testFeature() { out.setVersion(afterVersion); assertTrue(custom.getRequiredFeature().isPresent()); out.setFeatures(new HashSet<>(Arrays.asList(custom.getRequiredFeature().get(), TransportClient.TRANSPORT_CLIENT_FEATURE))); - assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + assertTrue(FeatureAware.shouldSerialize(out, custom)); } } @@ -161,14 +161,14 @@ public void testMissingFeature() { // the feature is missing but we should serialize it anyway because the client is not a transport client final BytesStreamOutput out = new BytesStreamOutput(); out.setVersion(afterVersion); - assertTrue(FeatureAware.shouldSerializeCustom(out, custom)); + assertTrue(FeatureAware.shouldSerialize(out, custom)); } { // the feature is missing and we should not serialize it because the client is a transport client final BytesStreamOutput out = new BytesStreamOutput(); out.setVersion(afterVersion); out.setFeatures(Collections.singleton(TransportClient.TRANSPORT_CLIENT_FEATURE)); - assertFalse(FeatureAware.shouldSerializeCustom(out, custom)); + assertFalse(FeatureAware.shouldSerialize(out, custom)); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 8b246ecc2d3de..0ba3de4381891 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.support.IndicesOptions; @@ -37,7 +38,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; @@ -304,6 +304,11 @@ public String getWriteableName() { return "test"; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeInt(value); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java index e7cbd04ce4b50..2cebd41a52c43 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterSerivceTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.cluster.service; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.io.stream.StreamOutput; @@ -43,6 +44,11 @@ public String getWriteableName() { return null; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public void writeTo(StreamOutput out) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index e51177c318ca8..b7177fdf867af 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -239,6 +239,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index cef3502a077c5..14f3c212c464c 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -492,6 +492,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); @@ -510,6 +515,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 63ef871c0e3f4..8501095089771 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.persistent; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -49,6 +50,8 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; @@ -57,8 +60,6 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.ArrayList; @@ -67,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -194,6 +196,16 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(executorNodeAttr, responseNode, testParam); } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + + @Override + public Optional getRequiredFeature() { + return Optional.empty(); + } } public static class Status implements Task.Status { diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 1dc853db59467..5d2abdd149223 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -21,9 +21,9 @@ import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.IntSet; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; -import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; @@ -1162,6 +1162,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + public static SnapshottableMetadata readFrom(StreamInput in) throws IOException { return readFrom(SnapshottableMetadata::new, in); } @@ -1193,6 +1198,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + public static NonSnapshottableMetadata readFrom(StreamInput in) throws IOException { return readFrom(NonSnapshottableMetadata::new, in); } @@ -1223,6 +1233,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + public static SnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException { return readFrom(SnapshottableGatewayMetadata::new, in); } @@ -1253,6 +1268,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + public static NonSnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException { return readFrom(NonSnapshottableGatewayMetadata::new, in); } @@ -1284,6 +1304,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + public static SnapshotableGatewayNoApiMetadata readFrom(StreamInput in) throws IOException { return readFrom(SnapshotableGatewayNoApiMetadata::new, in); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a96de96fd4f44..8e1ccdc5767ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -156,7 +156,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin { @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - static Optional X_PACK_FEATURE = Optional.of("x-pack"); + public static Optional X_PACK_FEATURE = Optional.of("x-pack"); @Override protected Optional getFeature() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index d6e351f763787..28b2889fcd318 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -22,14 +22,16 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.tasks.Task; +import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.persistent.PersistentTaskParams; import java.io.IOException; import java.util.Objects; +import java.util.Optional; public class OpenJobAction extends Action { @@ -237,6 +239,16 @@ public boolean equals(Object obj) { public String toString() { return Strings.toString(this); } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_4_0; + } + + @Override + public Optional getRequiredFeature() { + return XPackClientPlugin.X_PACK_FEATURE; + } } public static class Response extends AcknowledgedResponse { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java index e71186b60e020..fd32b4548c1b9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.rollup.job; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; @@ -14,11 +15,13 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.xpack.core.XPackClientPlugin; import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * This class is the main wrapper object that is serialized into the PersistentTask's cluster state. @@ -110,4 +113,14 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(config, headers); } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_6_3_0; + } + + @Override + public Optional getRequiredFeature() { + return XPackClientPlugin.X_PACK_FEATURE; + } } From aeee8fff527355d4f9d8ea9b7ac8a647b6bc8fc4 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 1 Jun 2018 15:38:54 +0200 Subject: [PATCH 11/23] add version and feature support to persistent tasks --- .../xpack/core/ml/action/StartDatafeedAction.java | 15 ++++++++++++++- .../xpack/core/watcher/WatcherMetaData.java | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index fc8c17f462a8a..d564d2d814d14 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.action; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; @@ -24,13 +25,15 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.persistent.PersistentTaskParams; import java.io.IOException; import java.util.Objects; +import java.util.Optional; import java.util.function.LongSupplier; public class StartDatafeedAction extends Action { @@ -231,6 +234,16 @@ public String getWriteableName() { return TASK_NAME; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_4_0; + } + + @Override + public Optional getRequiredFeature() { + return XPackPlugin.X_PACK_FEATURE; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(datafeedId); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java index 9f014dee843c5..279b8dd4f5829 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.watcher; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; @@ -38,6 +39,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_3_0; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); From 18a9a6f0e596487def9150b70faab9e4f5b0fbef Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 1 Jun 2018 16:02:22 +0200 Subject: [PATCH 12/23] add version and feature support to persistent tasks --- .../java/org/elasticsearch/license/LicensesMetaData.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java index d9f7068b2181e..f579792d9385b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java @@ -108,6 +108,11 @@ public String getWriteableName() { return TYPE; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_0_0; + } + @Override public EnumSet context() { return EnumSet.of(MetaData.XContentContext.GATEWAY); From 35463b39201e9ec0f68e801083129bad6d9b95b9 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 1 Jun 2018 17:33:32 +0200 Subject: [PATCH 13/23] unit tests --- .../PersistentTasksCustomMetaDataTests.java | 69 +++++++++++++++++++ .../persistent/TestPersistentTasksPlugin.java | 19 ++++- .../org/elasticsearch/test/VersionUtils.java | 24 +++++-- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 67962b800d2cf..1beef0cd73de0 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.persistent; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; +import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; @@ -26,6 +28,8 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable; @@ -43,13 +47,22 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractDiffableSerializationTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; +import static org.elasticsearch.test.VersionUtils.getFirstVersion; +import static org.elasticsearch.test.VersionUtils.getNextVersion; +import static org.elasticsearch.test.VersionUtils.getPreviousVersion; +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; +import static org.hamcrest.Matchers.equalTo; public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase { @@ -228,7 +241,63 @@ public void testBuilder() { assertEquals(changed, builder.isChanged()); persistentTasks = builder.build(); } + } + + public void testMinVersionSerialization() throws IOException { + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + + Version minVersion = getFirstVersion(); + final Version streamVersion = randomVersionBetween(random(), minVersion, getPreviousVersion(Version.CURRENT)); + tasks.addTask("test_compatible_version", TestPersistentTasksExecutor.NAME, + new TestParams(null, randomVersionBetween(random(), minVersion, streamVersion), + randomBoolean() ? Optional.empty() : Optional.of("test")), + randomAssignment()); + tasks.addTask("test_incompatible_version", TestPersistentTasksExecutor.NAME, + new TestParams(null, randomVersionBetween(random(), getNextVersion(streamVersion), Version.CURRENT), + randomBoolean() ? Optional.empty() : Optional.of("test")), + randomAssignment()); + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(streamVersion); + Set features = new HashSet<>(); + if (randomBoolean()) { + features.add("test"); + } + if (randomBoolean()) { + features.add(TransportClient.TRANSPORT_CLIENT_FEATURE); + } + tasks.build().writeTo(out); + + PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry())); + Set expectedIds = new HashSet<>(tasks.getCurrentTaskIds()); + expectedIds.remove("test_incompatible_version"); + assertThat(read.taskMap().keySet(), equalTo(expectedIds)); + } + + public void testFeatureSerialization() throws IOException { + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + + Version minVersion = getFirstVersion(); + tasks.addTask("test_compatible", TestPersistentTasksExecutor.NAME, + new TestParams(null, randomVersionBetween(random(), minVersion, Version.CURRENT), + randomBoolean() ? Optional.empty() : Optional.of("existing")), + randomAssignment()); + tasks.addTask("test_incompatible", TestPersistentTasksExecutor.NAME, + new TestParams(null, randomVersionBetween(random(), minVersion, Version.CURRENT), Optional.of("non_existing")), + randomAssignment()); + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(Version.CURRENT); + Set features = new HashSet<>(); + features.add("existing"); + features.add(TransportClient.TRANSPORT_CLIENT_FEATURE); + out.setFeatures(features); + tasks.build().writeTo(out); + PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry())); + Set expectedIds = new HashSet<>(tasks.getCurrentTaskIds()); + expectedIds.remove("test_incompatible"); + assertThat(read.taskMap().keySet(), equalTo(expectedIds)); } private Assignment randomAssignment() { diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 8501095089771..97b3407938768 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -122,6 +122,9 @@ public static class TestParams implements PersistentTaskParams { REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); } + private final Version minVersion; + private final Optional feature; + private String executorNodeAttr = null; private String responseNode = null; @@ -129,17 +132,25 @@ public static class TestParams implements PersistentTaskParams { private String testParam = null; public TestParams() { - + this((String)null); } public TestParams(String testParam) { + this(testParam, Version.CURRENT, Optional.empty()); + } + + public TestParams(String testParam, Version minVersion, Optional feature) { this.testParam = testParam; + this.minVersion = minVersion; + this.feature = feature; } public TestParams(StreamInput in) throws IOException { executorNodeAttr = in.readOptionalString(); responseNode = in.readOptionalString(); testParam = in.readOptionalString(); + minVersion = Version.readVersion(in); + feature = Optional.ofNullable(in.readOptionalString()); } @Override @@ -168,6 +179,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(executorNodeAttr); out.writeOptionalString(responseNode); out.writeOptionalString(testParam); + Version.writeVersion(minVersion, out); + out.writeOptionalString(feature.orElse(null)); } @Override @@ -199,12 +212,12 @@ public int hashCode() { @Override public Version getMinimalSupportedVersion() { - return Version.CURRENT; + return minVersion; } @Override public Optional getRequiredFeature() { - return Optional.empty(); + return feature; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 792f3fba123da..28c5c3a872f55 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -19,22 +19,19 @@ package org.elasticsearch.test; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; + import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; - /** Utilities for selecting versions in tests */ public class VersionUtils { @@ -159,6 +156,19 @@ public static Version getPreviousVersion(Version version) { throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]"); } + /** + * Get the released version after {@code version}. + */ + public static Version getNextVersion(Version version) { + for (int i = 0; i < ALL_VERSIONS.size(); i++) { + Version v = ALL_VERSIONS.get(i); + if (v.after(version)) { + return v; + } + } + throw new IllegalArgumentException("couldn't find any released versions after [" + version + "]"); + } + /** * Get the released version before {@link Version#CURRENT}. */ From e6e5413cf9f556ebf9f4ab821ed4eda9b2f9d1fe Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 2 Jun 2018 11:44:30 +0200 Subject: [PATCH 14/23] require params --- .../persistent/NodePersistentTasksExecutor.java | 2 +- .../PersistentTasksClusterService.java | 5 ++--- .../PersistentTasksCustomMetaData.java | 14 ++++++++++---- .../persistent/PersistentTasksExecutor.java | 6 +++--- .../persistent/PersistentTasksService.java | 3 +-- .../persistent/StartPersistentTaskAction.java | 16 ++++++++++++---- .../PersistentTasksCustomMetaDataTests.java | 8 ++++++-- .../PersistentTasksExecutorFullRestartIT.java | 10 ++++------ .../decider/EnableAssignmentDeciderIT.java | 9 +-------- 9 files changed, 40 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index efed0aef9b807..bf42733ff54ac 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -35,7 +35,7 @@ public NodePersistentTasksExecutor(ThreadPool threadPool) { this.threadPool = threadPool; } - public void executeTask(@Nullable Params params, + public void executeTask(Params params, @Nullable Task.Status status, AllocatedPersistentTask task, PersistentTasksExecutor executor) { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index cf44556ee5ddc..1464279a814d5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; @@ -65,7 +64,7 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * @param taskParams the task's parameters * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String taskId, String taskName, @Nullable Params taskParams, + public void createPersistentTask(String taskId, String taskName, Params taskParams, ActionListener> listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override @@ -225,7 +224,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @return a new {@link Assignment} */ private Assignment createAssignment(final String taskName, - final @Nullable Params taskParams, + final Params taskParams, final ClusterState currentState) { PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 5aeceda656383..31c283ee2512d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -49,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; @@ -265,7 +264,6 @@ public static class PersistentTask

implements Wr private final String id; private final long allocationId; private final String taskName; - @Nullable private final P params; @Nullable private final Status status; @@ -315,7 +313,11 @@ public PersistentTask(StreamInput in) throws IOException { id = in.readString(); allocationId = in.readLong(); taskName = in.readString(); - params = (P) in.readOptionalNamedWriteable(PersistentTaskParams.class); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + params = (P) in.readNamedWriteable(PersistentTaskParams.class); + } else { + params = (P) in.readOptionalNamedWriteable(PersistentTaskParams.class); + } status = in.readOptionalNamedWriteable(Task.Status.class); assignment = new Assignment(in.readOptionalString(), in.readString()); allocationIdOnLastStatusUpdate = in.readOptionalLong(); @@ -326,7 +328,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeLong(allocationId); out.writeString(taskName); - out.writeOptionalNamedWriteable(params); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeNamedWriteable(params); + } else { + out.writeOptionalNamedWriteable(params); + } out.writeOptionalNamedWriteable(status); out.writeOptionalString(assignment.executorNode); out.writeString(assignment.explanation); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 0a1e2095934ef..de75b1ff54085 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -24,10 +24,10 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.util.Map; import java.util.function.Predicate; @@ -118,7 +118,7 @@ protected String getDescription(PersistentTask taskInProgress) { * NOTE: The nodeOperation has to throw an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to * indicate that the persistent task has finished. */ - protected abstract void nodeOperation(AllocatedPersistentTask task, @Nullable Params params, @Nullable Task.Status status); + protected abstract void nodeOperation(AllocatedPersistentTask task, Params params, @Nullable Task.Status status); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 7a4a956fcfd5b..01c28dd5cd634 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -21,7 +21,6 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.support.ContextPreservingActionListener; @@ -69,7 +68,7 @@ public PersistentTasksService(Settings settings, ClusterService clusterService, */ public void sendStartRequest(final String taskId, final String taskName, - final @Nullable Params taskParams, + final Params taskParams, final ActionListener> listener) { @SuppressWarnings("unchecked") final ActionListener> wrappedListener = diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 4e6b11205fdfc..2a1a59d9b08ae 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; @@ -36,9 +37,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; @@ -66,7 +67,6 @@ public static class Request extends MasterNodeRequest { private String taskId; - @Nullable private String taskName; private PersistentTaskParams params; @@ -86,7 +86,11 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); taskId = in.readString(); taskName = in.readString(); - params = in.readOptionalNamedWriteable(PersistentTaskParams.class); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + params = in.readNamedWriteable(PersistentTaskParams.class); + } else { + params = in.readOptionalNamedWriteable(PersistentTaskParams.class); + } } @Override @@ -94,7 +98,11 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(taskId); out.writeString(taskName); - out.writeOptionalNamedWriteable(params); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeNamedWriteable(params); + } else { + out.writeOptionalNamedWriteable(params); + } } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 1beef0cd73de0..c25e3d0cc8607 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; @@ -267,8 +268,11 @@ public void testMinVersionSerialization() throws IOException { } tasks.build().writeTo(out); - PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData( - new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry())); + final StreamInput input = out.bytes().streamInput(); + input.setVersion(streamVersion); + PersistentTasksCustomMetaData read = + new PersistentTasksCustomMetaData(new NamedWriteableAwareStreamInput(input, getNamedWriteableRegistry())); + Set expectedIds = new HashSet<>(tasks.getCurrentTaskIds()); expectedIds.remove("test_incompatible_version"); assertThat(read.taskMap().keySet(), equalTo(expectedIds)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index b67b7678332b7..0a7168ad9b287 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -20,12 +20,12 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import java.util.ArrayList; import java.util.Collection; @@ -35,8 +35,6 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase { @@ -65,7 +63,7 @@ public void testFullClusterRestart() throws Exception { PlainActionFuture> future = new PlainActionFuture<>(); futures.add(future); taskIds[i] = UUIDs.base64UUID(); - service.sendStartRequest(taskIds[i], TestPersistentTasksExecutor.NAME, randomBoolean() ? null : new TestParams("Blah"), future); + service.sendStartRequest(taskIds[i], TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); } for (int i = 0; i < numberOfTasks; i++) { diff --git a/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java b/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java index cf1cc89b3a18a..aeb4d9b3a9bfb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java @@ -71,7 +71,7 @@ public void testEnableAssignmentAfterRestart() throws Exception { final CountDownLatch latch = new CountDownLatch(numberOfTasks); for (int i = 0; i < numberOfTasks; i++) { PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); - service.sendStartRequest("task_" + i, TestPersistentTasksExecutor.NAME, randomTaskParams(), + service.sendStartRequest("task_" + i, TestPersistentTasksExecutor.NAME, new TestParams(randomAlphaOfLength(10)), new ActionListener>() { @Override public void onResponse(PersistentTask task) { @@ -163,11 +163,4 @@ private void resetPersistentTasksAssignment() { assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings)); } - /** Returns a random task parameter **/ - private static PersistentTaskParams randomTaskParams() { - if (randomBoolean()) { - return null; - } - return new TestParams(randomAlphaOfLength(10)); - } } From 5e6de982278377a7dc9552b631c470df97aeda0e Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 2 Jun 2018 12:17:02 +0200 Subject: [PATCH 15/23] fix tests --- .../StartPersistentActionRequestTests.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index 3b0fc2a3d0495..e4c5a26de9c0c 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -22,8 +22,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.persistent.StartPersistentTaskAction.Request; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.test.AbstractStreamableTestCase; import java.util.Collections; @@ -32,17 +32,12 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas @Override protected Request createTestInstance() { - TestParams testParams; + TestParams testParams = new TestParams(); + if (randomBoolean()) { + testParams.setTestParam(randomAlphaOfLengthBetween(1, 20)); + } if (randomBoolean()) { - testParams = new TestParams(); - if (randomBoolean()) { - testParams.setTestParam(randomAlphaOfLengthBetween(1, 20)); - } - if (randomBoolean()) { - testParams.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); - } - } else { - testParams = null; + testParams.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); } return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), testParams); } From ce99bbe0c0dbddf49ceeb3092d544fcd8400c7ed Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 2 Jun 2018 16:00:09 +0200 Subject: [PATCH 16/23] java docs --- .../src/main/java/org/elasticsearch/cluster/NamedDiffable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java index dfccd259780b8..729523233d73d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.io.stream.VersionedNamedWriteable; /** - * Diff that also support NamedWriteable interface + * Diff that also support {@link VersionedNamedWriteable} interface */ public interface NamedDiffable extends Diffable, VersionedNamedWriteable { } From 1f8392688819677aec211b87f51ad09dceec44ec Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 2 Jun 2018 16:12:52 +0200 Subject: [PATCH 17/23] feedback --- .../PersistentTasksCustomMetaDataTests.java | 7 +++---- .../java/org/elasticsearch/test/VersionUtils.java | 2 +- .../elasticsearch/xpack/core/XPackClientPlugin.java | 2 +- .../org/elasticsearch/xpack/core/XPackPlugin.java | 9 +++++++++ .../xpack/core/ml/action/OpenJobAction.java | 11 ++--------- .../xpack/core/ml/action/StartDatafeedAction.java | 9 +-------- .../xpack/core/rollup/job/RollupJob.java | 11 ++--------- 7 files changed, 19 insertions(+), 32 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index c25e3d0cc8607..d86f855d72a35 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -266,6 +266,7 @@ public void testMinVersionSerialization() throws IOException { if (randomBoolean()) { features.add(TransportClient.TRANSPORT_CLIENT_FEATURE); } + out.setFeatures(features); tasks.build().writeTo(out); final StreamInput input = out.bytes().streamInput(); @@ -273,8 +274,7 @@ public void testMinVersionSerialization() throws IOException { PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData(new NamedWriteableAwareStreamInput(input, getNamedWriteableRegistry())); - Set expectedIds = new HashSet<>(tasks.getCurrentTaskIds()); - expectedIds.remove("test_incompatible_version"); + Set expectedIds = Collections.singleton("test_compatible_version"); assertThat(read.taskMap().keySet(), equalTo(expectedIds)); } @@ -299,8 +299,7 @@ public void testFeatureSerialization() throws IOException { PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData( new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry())); - Set expectedIds = new HashSet<>(tasks.getCurrentTaskIds()); - expectedIds.remove("test_incompatible"); + Set expectedIds = Collections.singleton("test_compatible"); assertThat(read.taskMap().keySet(), equalTo(expectedIds)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 28c5c3a872f55..bba1f07520a1f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -157,7 +157,7 @@ public static Version getPreviousVersion(Version version) { } /** - * Get the released version after {@code version}. + * Get the next version after {@code version}. */ public static Version getNextVersion(Version version) { for (int i = 0; i < ALL_VERSIONS.size(); i++) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 8e1ccdc5767ba..a96de96fd4f44 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -156,7 +156,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin { @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - public static Optional X_PACK_FEATURE = Optional.of("x-pack"); + static Optional X_PACK_FEATURE = Optional.of("x-pack"); @Override protected Optional getFeature() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 602f4bdbc079b..db36aabf7ac6a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -40,6 +40,7 @@ import org.elasticsearch.license.LicensesMetaData; import org.elasticsearch.license.Licensing; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.rest.RestController; @@ -331,4 +332,12 @@ default Optional getRequiredFeature() { } + public interface XPackPersistentTaskParams extends PersistentTaskParams { + + @Override + default Optional getRequiredFeature() { + return XPackClientPlugin.X_PACK_FEATURE; + } + } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index 28b2889fcd318..f45a95423aa41 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -22,16 +22,14 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.tasks.Task; -import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; import java.util.Objects; -import java.util.Optional; public class OpenJobAction extends Action { @@ -129,7 +127,7 @@ public String toString() { } } - public static class JobParams implements PersistentTaskParams { + public static class JobParams implements XPackPlugin.XPackPersistentTaskParams { /** TODO Remove in 7.0.0 */ public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); @@ -244,11 +242,6 @@ public String toString() { public Version getMinimalSupportedVersion() { return Version.V_5_4_0; } - - @Override - public Optional getRequiredFeature() { - return XPackClientPlugin.X_PACK_FEATURE; - } } public static class Response extends AcknowledgedResponse { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index d564d2d814d14..3e24882551b92 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; @@ -33,7 +32,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.Optional; import java.util.function.LongSupplier; public class StartDatafeedAction extends Action { @@ -141,7 +139,7 @@ public boolean equals(Object obj) { } } - public static class DatafeedParams implements PersistentTaskParams { + public static class DatafeedParams implements XPackPlugin.XPackPersistentTaskParams { public static ObjectParser PARSER = new ObjectParser<>(TASK_NAME, DatafeedParams::new); @@ -239,11 +237,6 @@ public Version getMinimalSupportedVersion() { return Version.V_5_4_0; } - @Override - public Optional getRequiredFeature() { - return XPackPlugin.X_PACK_FEATURE; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(datafeedId); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java index fd32b4548c1b9..7afcdb71b11cc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java @@ -14,21 +14,19 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.persistent.PersistentTaskParams; -import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.Optional; /** * This class is the main wrapper object that is serialized into the PersistentTask's cluster state. * It holds the config (RollupJobConfig) and a map of authentication headers. Only RollupJobConfig * is ever serialized to the user, so the headers should never leak */ -public class RollupJob extends AbstractDiffable implements PersistentTaskParams { +public class RollupJob extends AbstractDiffable implements XPackPlugin.XPackPersistentTaskParams { public static final String NAME = "xpack/rollup/job"; @@ -118,9 +116,4 @@ public int hashCode() { public Version getMinimalSupportedVersion() { return Version.V_6_3_0; } - - @Override - public Optional getRequiredFeature() { - return XPackClientPlugin.X_PACK_FEATURE; - } } From 5bc8a0a79cd1f9f2d2b0815d0acc23cd375f8c5c Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 17:32:37 +0200 Subject: [PATCH 18/23] min compat version --- .../main/java/org/elasticsearch/cluster/RestoreInProgress.java | 2 +- .../java/org/elasticsearch/cluster/SnapshotsInProgress.java | 2 +- .../java/org/elasticsearch/cluster/metadata/IndexGraveyard.java | 2 +- .../elasticsearch/cluster/metadata/RepositoriesMetaData.java | 2 +- .../src/main/java/org/elasticsearch/ingest/IngestMetadata.java | 2 +- .../src/main/java/org/elasticsearch/script/ScriptMetaData.java | 2 +- .../main/java/org/elasticsearch/license/LicensesMetaData.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index ad922ad36df65..138788251c90a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -385,7 +385,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } public static NamedDiff readDiffFrom(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index a48164ffe828e..87563c968af17 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -397,7 +397,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } public static NamedDiff readDiffFrom(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java index 6abbf86edd972..74789aada3a46 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java @@ -97,7 +97,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index abaca292ed6d4..7bb72be0e1e18 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -106,7 +106,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } public RepositoriesMetaData(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java index 6bca478f52b7b..1e262adf8cf8d 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java @@ -72,7 +72,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } public Map getPipelines() { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java index ea230b9e3a669..59d824eb313e0 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -385,7 +385,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java index f579792d9385b..6d001dea516ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetaData.java @@ -110,7 +110,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_0_0; + return Version.CURRENT.minimumCompatibilityVersion(); } @Override From 9870fdd921b5ced587c9901f9c5a1fe242e3a06a Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 17:37:13 +0200 Subject: [PATCH 19/23] compatibleFutureVersion --- .../PersistentTasksCustomMetaDataTests.java | 4 ++-- .../org/elasticsearch/test/VersionUtils.java | 20 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index d86f855d72a35..6c5154e57cd10 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -59,8 +59,8 @@ import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; +import static org.elasticsearch.test.VersionUtils.compatibleFutureVersion; import static org.elasticsearch.test.VersionUtils.getFirstVersion; -import static org.elasticsearch.test.VersionUtils.getNextVersion; import static org.elasticsearch.test.VersionUtils.getPreviousVersion; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.equalTo; @@ -254,7 +254,7 @@ public void testMinVersionSerialization() throws IOException { randomBoolean() ? Optional.empty() : Optional.of("test")), randomAssignment()); tasks.addTask("test_incompatible_version", TestPersistentTasksExecutor.NAME, - new TestParams(null, randomVersionBetween(random(), getNextVersion(streamVersion), Version.CURRENT), + new TestParams(null, randomVersionBetween(random(), compatibleFutureVersion(streamVersion), Version.CURRENT), randomBoolean() ? Optional.empty() : Optional.of("test")), randomAssignment()); final BytesStreamOutput out = new BytesStreamOutput(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index bba1f07520a1f..84c480b8d510b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -156,19 +156,6 @@ public static Version getPreviousVersion(Version version) { throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]"); } - /** - * Get the next version after {@code version}. - */ - public static Version getNextVersion(Version version) { - for (int i = 0; i < ALL_VERSIONS.size(); i++) { - Version v = ALL_VERSIONS.get(i); - if (v.after(version)) { - return v; - } - } - throw new IllegalArgumentException("couldn't find any released versions after [" + version + "]"); - } - /** * Get the released version before {@link Version#CURRENT}. */ @@ -238,6 +225,13 @@ public static Version incompatibleFutureVersion(Version version) { return opt.get(); } + /** returns the first future compatible version */ + public static Version compatibleFutureVersion(Version version) { + final Optional opt = ALL_VERSIONS.stream().filter(version::before).filter(v -> v.isCompatible(version)).findAny(); + assert opt.isPresent() : "no future compatible version for " + version; + return opt.get(); + } + /** Returns the maximum {@link Version} that is compatible with the given version. */ public static Version maxCompatibleVersion(Version version) { final List compatible = ALL_VERSIONS.stream().filter(version::isCompatible).filter(version::onOrBefore) From 7b82a5b8d07300cbdc2ed04b8b0062f1a0658f24 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 17:38:51 +0200 Subject: [PATCH 20/23] inline --- .../persistent/PersistentTasksCustomMetaDataTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 6c5154e57cd10..267941d3cf87e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -274,8 +274,7 @@ public void testMinVersionSerialization() throws IOException { PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData(new NamedWriteableAwareStreamInput(input, getNamedWriteableRegistry())); - Set expectedIds = Collections.singleton("test_compatible_version"); - assertThat(read.taskMap().keySet(), equalTo(expectedIds)); + assertThat(read.taskMap().keySet(), equalTo(Collections.singleton("test_compatible_version"))); } public void testFeatureSerialization() throws IOException { @@ -299,8 +298,7 @@ public void testFeatureSerialization() throws IOException { PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData( new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry())); - Set expectedIds = Collections.singleton("test_compatible"); - assertThat(read.taskMap().keySet(), equalTo(expectedIds)); + assertThat(read.taskMap().keySet(), equalTo(Collections.singleton("test_compatible"))); } private Assignment randomAssignment() { From 2aa7a8583e24c81e4bd3e4844ae0bda9d1129d12 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 17:53:48 +0200 Subject: [PATCH 21/23] v_5_4_0 back --- .../elasticsearch/persistent/PersistentTasksCustomMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 31c283ee2512d..4b1ba3e11e372 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -190,7 +190,7 @@ public long getNumberOfTasksOnNode(String nodeId, String taskName) { @Override public Version getMinimalSupportedVersion() { - return Version.V_6_3_0; + return Version.V_5_4_0; } @Override From ef015fa851ded2161090e1bc4d16c9e48dedf1a3 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 17:59:37 +0200 Subject: [PATCH 22/23] V_5.x -> minimum compat --- .../org/elasticsearch/xpack/core/ml/action/OpenJobAction.java | 2 +- .../elasticsearch/xpack/core/ml/action/StartDatafeedAction.java | 2 +- .../org/elasticsearch/xpack/core/watcher/WatcherMetaData.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java index f45a95423aa41..429f36d00c2ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java @@ -240,7 +240,7 @@ public String toString() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_4_0; + return Version.CURRENT.minimumCompatibilityVersion(); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index 3e24882551b92..1a1d95820979e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -234,7 +234,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_4_0; + return Version.CURRENT.minimumCompatibilityVersion(); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java index 279b8dd4f5829..bddeb5f5e3281 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetaData.java @@ -41,7 +41,7 @@ public String getWriteableName() { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_3_0; + return Version.CURRENT.minimumCompatibilityVersion(); } @Override From 307b7143f9d024c31cb61c5fb44566459276bc44 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 3 Jun 2018 19:18:14 +0200 Subject: [PATCH 23/23] fix testSnapshotDeletionsInProgressSerialization --- .../cluster/serialization/ClusterSerializationTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index d64b4a66ee78a..b2878008f7055 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -133,7 +132,7 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception { // serialize with current version BytesStreamOutput outStream = new BytesStreamOutput(); - Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT); + Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT); outStream.setVersion(version); diffs.writeTo(outStream); StreamInput inStream = outStream.bytes().streamInput();