From 95696150506b3328c036bd881ab8af75ae709c34 Mon Sep 17 00:00:00 2001 From: Greg Marzouka Date: Tue, 14 Feb 2017 13:07:36 -0500 Subject: [PATCH] Fix #2573 bwc support for new cluster allocation explain response --- .../ClusterAllocationExplainResponse.cs | 184 ++++++++++++++++-- ...rAllocationExplainResponseJsonConverter.cs | 152 +++++++++++++++ src/Nest/Nest.csproj | 1 + .../ClientConcepts/LowLevel/PostData.doc.cs | 2 +- .../ClusterAllocationExplainApiTests.cs | 54 ++++- .../EndpointTests/Bootstrappers/Seeder.cs | 25 ++- .../Clusters/UnbalancedCluster.cs | 11 ++ src/Tests/Tests.csproj | 1 + 8 files changed, 399 insertions(+), 31 deletions(-) create mode 100644 src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponseJsonConverter.cs create mode 100644 src/Tests/Framework/EndpointTests/Clusters/UnbalancedCluster.cs diff --git a/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponse.cs b/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponse.cs index 85f2891a2f1..ad0e2eade8e 100644 --- a/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponse.cs +++ b/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponse.cs @@ -1,57 +1,147 @@ using System; using System.Collections.Generic; using System.Runtime.Serialization; +using System.Security.Cryptography.X509Certificates; using Newtonsoft.Json; using Newtonsoft.Json.Converters; namespace Nest { + [JsonConverter(typeof(ClusterAllocationExplainResponseJsonConverter))] public interface IClusterAllocationExplainResponse : IResponse { - [JsonProperty("shard")] - ShardAllocationExplanation Shard { get; } + [JsonProperty("index")] + string Index { get; } - [JsonProperty("assigned")] - bool Assigned { get; } + [JsonIgnore] + // TODO rename to Shard in 6.0 + int ShardId { get; } - [JsonProperty("assigned_node_id")] - string AssignedNodeId { get; } + [JsonProperty("primary")] + bool Primary { get; } - [JsonProperty("shard_state_fetch_pending")] - bool ShardStateFetchPending { get; } + [JsonProperty("current_state")] + string CurrentState { get; } [JsonProperty("unassigned_info")] UnassignedInformation UnassignedInformation { get; } + [JsonProperty("can_allocate")] + Decision? CanAllocate { get; } + + [JsonProperty("allocate_explanation")] + string AllocateExplanation { get; } + + [JsonProperty("configured_delay")] + string ConfiguredDelay { get; } + + [JsonProperty("configured_delay_in_mills")] + long ConfiguredDelayInMilliseconds { get; } + + [JsonProperty("current_node")] + CurrentNode CurrentNode { get; } + + [JsonProperty("can_remain_on_current_node")] + Decision? CanRemainOnCurrentNode { get; } + + [JsonProperty("can_remain_decisions")] + IReadOnlyCollection CanRemainDecisions { get; } + + [JsonProperty("can_rebalance_cluster")] + Decision? CanRebalanceCluster { get; } + + [JsonProperty("can_rebalance_to_other_nodes")] + Decision? CanRebalanceToOtherNode { get; } + + [JsonProperty("can_rebalance_cluster_decisions")] + IReadOnlyCollection CanRebalanceClusterDecisions { get; } + + [JsonProperty("rebalance_explanation")] + string RebalanceExplanation { get; } + + [JsonProperty("node_allocation_decisions")] + IReadOnlyCollection NodeAllocationDecisions { get; } + + [JsonProperty("can_move_to_other_node")] + Decision? CanMoveToOtherNode { get; } + + [JsonProperty("move_explanation")] + string MoveExplanation { get; } + [JsonProperty("allocation_delay")] string AllocationDelay { get; } - [JsonProperty("allocation_delay_ms")] + [JsonProperty("allocation_delay_in_millis")] long AllocationDelayInMilliseconds { get; } [JsonProperty("remaining_delay")] string RemainingDelay { get; } - [JsonProperty("remaining_delay_ms")] + [JsonProperty("remaining_delay_in_millis")] long RemainingDelayInMilliseconds { get; } + [JsonIgnore] + [Obsolete("Removed in Elasticsearch 5.2. Use properties on root object instead.")] + ShardAllocationExplanation Shard { get; } + + [JsonProperty("assigned")] + [Obsolete("Removed in Elasticsearch 5.2.")] + bool Assigned { get; } + + [JsonProperty("assigned_node_id")] + [Obsolete("Removed in Elasticsearch 5.2.")] + string AssignedNodeId { get; } + + [JsonProperty("shard_state_fetch_pending")] + [Obsolete("Removed in Elasticsearch 5.2.")] + bool ShardStateFetchPending { get; } + [JsonProperty("nodes")] + [Obsolete("Removed in Elasticsearch 5.2. Use node_allocation_decisions instead.")] [JsonConverter(typeof(VerbatimDictionaryKeysJsonConverter))] IReadOnlyDictionary Nodes { get; } } public class ClusterAllocationExplainResponse : ResponseBase, IClusterAllocationExplainResponse { - public ShardAllocationExplanation Shard { get; internal set; } + public string Index { get; internal set; } - public bool Assigned { get; internal set; } + public int ShardId { get; internal set; } - public string AssignedNodeId { get; internal set; } + public bool Primary { get; internal set; } - public bool ShardStateFetchPending { get; internal set; } + public string CurrentState { get; internal set; } public UnassignedInformation UnassignedInformation { get; internal set; } + public Decision? CanAllocate { get; internal set; } + + public string AllocateExplanation { get; internal set; } + + public string ConfiguredDelay { get; internal set; } + + public long ConfiguredDelayInMilliseconds { get; internal set; } + + public CurrentNode CurrentNode { get; internal set; } + + public Decision? CanRemainOnCurrentNode { get; internal set; } + + public IReadOnlyCollection CanRemainDecisions { get; internal set; } + + public Decision? CanRebalanceCluster { get; internal set; } + + public Decision? CanRebalanceToOtherNode { get; internal set; } + + public IReadOnlyCollection CanRebalanceClusterDecisions { get; internal set; } = EmptyReadOnly.Collection; + + public string RebalanceExplanation { get; internal set; } + + public IReadOnlyCollection NodeAllocationDecisions { get; internal set; } + + public Decision? CanMoveToOtherNode { get; internal set; } + + public string MoveExplanation { get; internal set; } + public string AllocationDelay { get; internal set; } public long AllocationDelayInMilliseconds { get; internal set; } @@ -60,9 +150,41 @@ public class ClusterAllocationExplainResponse : ResponseBase, IClusterAllocation public long RemainingDelayInMilliseconds { get; internal set; } + [Obsolete("Removed in Elasticsearch 5.2. Use properties on root object instead.")] + public ShardAllocationExplanation Shard { get; internal set; } + + [Obsolete("Removed in Elasticsearch 5.2.")] + public bool Assigned { get; internal set; } + + [Obsolete("Removed in Elasticsearch 5.2.")] + public string AssignedNodeId { get; internal set; } + + [Obsolete("Removed in Elasticsearch 5.2.")] + public bool ShardStateFetchPending { get; internal set; } + + [Obsolete("Removed in Elasticsearch 5.2. Usage NodeAllocationDecisions instead.")] public IReadOnlyDictionary Nodes { get; internal set; } = EmptyReadOnly.Dictionary; } + [JsonObject] + public class CurrentNode + { + [JsonProperty("id")] + public string Id { get; internal set; } + + [JsonProperty("name")] + public string Name { get; internal set; } + + [JsonProperty("transport_address")] + public string TransportAddress { get; internal set; } + + [JsonProperty("weight_ranking")] + public string WeightRanking { get; internal set; } + + [JsonProperty("attributes")] + public IReadOnlyDictionary NodeAttributes { get; set; } = EmptyReadOnly.Dictionary; + } + [JsonConverter(typeof(StringEnumConverter))] public enum AllocationExplainDecision { @@ -82,9 +204,18 @@ public enum AllocationExplainDecision [JsonObject] public class NodeAllocationExplanation { + [JsonProperty("node_id")] + public string NodeId { get; set; } + [JsonProperty("node_name")] public string NodeName { get; set; } + [JsonProperty("transport_address")] + public string TransportAddress { get; set; } + + [JsonProperty("node_decision")] + public Decision? NodeDecision { get; set; } + [JsonProperty("node_attributes")] public IReadOnlyDictionary NodeAttributes { get; set; } = EmptyReadOnly.Dictionary; @@ -92,19 +223,40 @@ public class NodeAllocationExplanation public AllocationStore Store { get; set; } [JsonProperty("final_decision")] + [Obsolete("Removed in Elasticsearch 5.2.")] public FinalDecision FinalDecision { get; set; } [JsonProperty("final_explanation")] + [Obsolete("Removed in Elasticsearch 5.2.")] public string FinalExplanation { get; set; } [JsonProperty("weight")] + [Obsolete("Removed in Elasticsearch 5.2. Use WeightRanking instead.")] public float Weight { get; set; } + [JsonProperty("weight_ranking")] + public int? WeightRanking { get; set; } + [JsonProperty("decisions")] + [Obsolete("Removed in Elasticsearch 5.2. Use Deciders instead.")] public IReadOnlyCollection Decisions { get; set; } = EmptyReadOnly.Collection; + + [JsonProperty("deciders")] + public IReadOnlyCollection Deciders { get; set; } = EmptyReadOnly.Collection; + } + + [JsonConverter(typeof(StringEnumConverter))] + public enum Decision + { + [EnumMember(Value = "yes")] + Yes, + + [EnumMember(Value = "no")] + No } [JsonConverter(typeof(StringEnumConverter))] + [Obsolete("Removed in Elasticsearch 5.2")] public enum FinalDecision { [EnumMember(Value = "YES")] @@ -159,7 +311,6 @@ public class AllocationDecision public string Explanation { get; set; } } - public class UnassignedInformation { [JsonProperty("reason")] @@ -167,6 +318,9 @@ public class UnassignedInformation [JsonProperty("at")] public DateTime At { get; set; } + + [JsonProperty("last_allocation_status")] + public string LastAllocationStatus { get; set; } } public class ShardAllocationExplanation diff --git a/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponseJsonConverter.cs b/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponseJsonConverter.cs new file mode 100644 index 00000000000..c2fdea4fbac --- /dev/null +++ b/src/Nest/Cluster/ClusterAllocationExplain/ClusterAllocationExplainResponseJsonConverter.cs @@ -0,0 +1,152 @@ +using System; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Collections.ObjectModel; +using System.Collections.Generic; + +namespace Nest +{ + // TODO this custom converter is in place because the response changed from ES 5.0 to ES 5.2 + // so we are supporting both formats. In 6.0 we should remove this entirely and only support + // the new format. + public class ClusterAllocationExplainResponseJsonConverter : JsonConverter + { + public override bool CanRead { get; } = true; + + public override bool CanWrite { get; } = false; + + public override bool CanConvert(Type objectType) => true; + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var o = JObject.Load(reader); + var response = new ClusterAllocationExplainResponse(); + var newResponseStructure = false; + foreach (var p in o.Properties()) + { + switch (p.Name) + { + case "shard": + if (p.Value.Type == JTokenType.Object) + { + response.Shard = p.Value.ToObject(); + } + else if (p.Value.Type == JTokenType.Integer) + { + newResponseStructure = true; + response.ShardId = p.Value.ToObject(); + } + break; + case "index": + response.Index = p.Value.ToString(); + break; + case "primary": + response.Primary = p.Value.ToObject(); + break; + case "can_allocate": + response.CanAllocate = p.Value.ToObject(); + break; + case "allocate_explanation": + response.AllocateExplanation = p.Value.ToString(); + break; + case "configured_delay": + response.ConfiguredDelay = p.Value.ToString(); + break; + case "configured_delay_in_millis": + response.ConfiguredDelayInMilliseconds = p.Value.ToObject(); + break; + case "can_remain_decisions": + response.CanRemainDecisions = p.Value.ToObject>(); + break; + case "can_move_to_other_node": + response.CanMoveToOtherNode = p.Value.ToObject(); + break; + case "move_explanation": + response.MoveExplanation = p.Value.ToString(); + break; + case "current_state": + response.CurrentState = p.Value.ToString(); + break; + case "current_node": + response.CurrentNode = p.Value.ToObject(); + break; + case "can_remain_on_current_node": + response.CanRemainOnCurrentNode = p.Value.ToObject(); + break; + case "can_rebalance_cluster": + response.CanRebalanceCluster = p.Value.ToObject(); + break; + case "can_rebalance_cluster_decisions": + response.CanRebalanceClusterDecisions = p.Value.ToObject>(); + break; + case "can_rebalance_to_other_node": + response.CanRebalanceToOtherNode = p.Value.ToObject(); + break; + case "rebalance_explanation": + response.RebalanceExplanation = p.Value.ToString(); + break; + case "assigned": + response.Assigned = p.Value.ToObject(); + break; + case "assigned_node_id": + response.AssignedNodeId = p.Value.ToString(); + break; + case "shard_state_fetch_pending": + response.ShardStateFetchPending = p.Value.ToObject(); + break; + case "unassigned_info": + response.UnassignedInformation = p.Value.ToObject(); + break; + case "allocation_delay": + response.AllocationDelay = p.Value.ToString(); + break; + case "allocation_delay_in_millis": + case "allocation_delay_ms": + response.AllocationDelayInMilliseconds = p.Value.ToObject(); + break; + case "remaining_delay": + response.RemainingDelay = p.Value.ToString(); + break; + case "remaining_delay_in_millis": + case "remaining_delay_ms": + response.RemainingDelayInMilliseconds = p.Value.ToObject(); + break; + case "node_allocation_decisions": + response.NodeAllocationDecisions = p.Value.ToObject>(); + break; + case "nodes": + response.Nodes = p.Value.ToObject>(); + break; + } + } + + if (newResponseStructure) + { + // Fill in old properties + response.Shard = new ShardAllocationExplanation + { + Id = response.ShardId, + Index = response.Index, + Primary = response.Primary + }; + + if (response.NodeAllocationDecisions != null) + { + var nodes = new Dictionary(); + foreach (var explanation in response.NodeAllocationDecisions) + { + explanation.Decisions = explanation.Deciders; + nodes.Add(explanation.NodeId, explanation); + } + } + } + + return response; + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotSupportedException(); + } + } +} diff --git a/src/Nest/Nest.csproj b/src/Nest/Nest.csproj index c75f3dccac3..d5d717da687 100644 --- a/src/Nest/Nest.csproj +++ b/src/Nest/Nest.csproj @@ -339,6 +339,7 @@ + diff --git a/src/Tests/ClientConcepts/LowLevel/PostData.doc.cs b/src/Tests/ClientConcepts/LowLevel/PostData.doc.cs index 3aff26452c0..844e4026ff2 100644 --- a/src/Tests/ClientConcepts/LowLevel/PostData.doc.cs +++ b/src/Tests/ClientConcepts/LowLevel/PostData.doc.cs @@ -112,7 +112,7 @@ private async Task AssertOn(IConnectionConfigurationValues settings) /** If you want to maintain a copy of the request that went out, use `DisableDirectStreaming` */ settings = new ConnectionSettings().DisableDirectStreaming(); - /** by forcing `DisableDirectStreaming` on connection settings, serialization happens first in a private `MemoryStream` + /** by forcing `DisableDirectStreaming` on connection indexCreationSettings, serialization happens first in a private `MemoryStream` * so we can get hold of the serialized bytes */ await Post(() => listOfObjects, writes: multiObjectJson, storesBytes: true, settings: settings); diff --git a/src/Tests/Cluster/ClusterAllocationExplain/ClusterAllocationExplainApiTests.cs b/src/Tests/Cluster/ClusterAllocationExplain/ClusterAllocationExplainApiTests.cs index ef268deb4a2..f0d0d56172d 100644 --- a/src/Tests/Cluster/ClusterAllocationExplain/ClusterAllocationExplainApiTests.cs +++ b/src/Tests/Cluster/ClusterAllocationExplain/ClusterAllocationExplainApiTests.cs @@ -9,9 +9,10 @@ namespace Tests.Cluster.ClusterAllocationExplain { - public class ClusterAllocationExplainApiTests : ApiIntegrationTestBase + public class ClusterAllocationExplainApiTests : ApiIntegrationTestBase { - public ClusterAllocationExplainApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + public ClusterAllocationExplainApiTests(UnbalancedCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + protected override LazyResponses ClientUsage() => Calls( fluent: (client, f) => client.ClusterAllocationExplain(f), fluentAsync: (client, f) => client.ClusterAllocationExplainAsync(f), @@ -51,10 +52,7 @@ protected override void ExpectResponse(IClusterAllocationExplainResponse respons { response.Shard.Primary.Should().BeTrue(); response.Shard.Id.Should().Be(0); - response.Shard.IndexUniqueId.Should().NotBeNullOrEmpty(); - response.Assigned.Should().BeTrue(); - response.AssignedNodeId.Should().NotBeNullOrWhiteSpace(); - response.ShardStateFetchPending.Should().BeFalse(); + response.Shard.Index.Should().NotBeNull(); foreach (var node in response.Nodes) { @@ -62,7 +60,7 @@ protected override void ExpectResponse(IClusterAllocationExplainResponse respons explanation.NodeName.Should().NotBeNullOrEmpty(); explanation.Weight.Should().BeGreaterOrEqualTo(0); - explanation.NodeAttributes.Should().NotBeEmpty(); + explanation.NodeAttributes.Should().NotBeNull(); explanation.Store.Should().NotBeNull(); explanation.Store.ShardCopy.Should().Be(StoreCopy.Available); explanation.FinalExplanation.Should().NotBeNullOrEmpty(); @@ -70,4 +68,46 @@ protected override void ExpectResponse(IClusterAllocationExplainResponse respons } } } + + [SkipVersion(">5.1.2", "")] + public class ClusterAllocationExplainOldApiTests : ClusterAllocationExplainApiTests + { + public ClusterAllocationExplainOldApiTests(UnbalancedCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override void ExpectResponse(IClusterAllocationExplainResponse response) + { + base.ExpectResponse(response); + response.Shard.IndexUniqueId.Should().NotBeNullOrEmpty(); + response.Assigned.Should().BeTrue(); + response.AssignedNodeId.Should().NotBeNullOrWhiteSpace(); + response.ShardStateFetchPending.Should().BeFalse(); + } + } + + [SkipVersion("<5.2.0", "")] + public class ClusterAllocationExplainNewApiTests : ClusterAllocationExplainApiTests + { + public ClusterAllocationExplainNewApiTests(UnbalancedCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override void ExpectResponse(IClusterAllocationExplainResponse response) + { + response.Primary.Should().BeTrue(); + response.ShardId.Should().Be(0); + response.Index.Should().NotBeNullOrEmpty(); + response.CurrentState.Should().NotBeNullOrEmpty(); + response.CurrentNode.Should().NotBeNull(); + response.CanRemainOnCurrentNode.Should().NotBeNull(); + response.CanRebalanceCluster.Should().NotBeNull(); + response.CanRebalanceClusterDecisions.Should().NotBeNullOrEmpty(); + + foreach( var decision in response.CanRebalanceClusterDecisions) + { + decision.Decider.Should().NotBeNullOrEmpty(); + decision.Explanation.Should().NotBeNullOrEmpty(); + } + + response.CanRebalanceToOtherNode.Should().NotBeNull(); + response.RebalanceExplanation.Should().NotBeNullOrEmpty(); + } + } } diff --git a/src/Tests/Framework/EndpointTests/Bootstrappers/Seeder.cs b/src/Tests/Framework/EndpointTests/Bootstrappers/Seeder.cs index ac1a0599f49..6bf0445b3b8 100644 --- a/src/Tests/Framework/EndpointTests/Bootstrappers/Seeder.cs +++ b/src/Tests/Framework/EndpointTests/Bootstrappers/Seeder.cs @@ -11,11 +11,22 @@ public class Seeder { private IElasticClient Client { get; } - public Seeder(ElasticsearchNode node) + private readonly IIndexSettings DefaultIndexSettings = new IndexSettings + { + NumberOfShards = 2, + NumberOfReplicas = 0 + }; + + private IIndexSettings IndexSettings { get; } + + public Seeder(ElasticsearchNode node, IIndexSettings indexSettings) { this.Client = node.Client; + this.IndexSettings = indexSettings ?? DefaultIndexSettings; } + public Seeder(ElasticsearchNode node) : this(node, null) { } + public void SeedNode() { if (TestClient.Configuration.ForceReseed || !AlreadySeeded()) @@ -78,13 +89,11 @@ private void CreateIndicesAndSeedIndexData() private void CreateIndexTemplate() { - var putTemplateResult = this.Client.PutIndexTemplate("nest_tests", p => p - .Template("*") //match on all created indices - .Settings(s => s - .NumberOfReplicas(0) - .NumberOfShards(2) - ) - ); + var putTemplateResult = this.Client.PutIndexTemplate(new PutIndexTemplateRequest("nest_test") + { + Template = "*", + Settings = this.IndexSettings + }); putTemplateResult.IsValid.Should().BeTrue(); } diff --git a/src/Tests/Framework/EndpointTests/Clusters/UnbalancedCluster.cs b/src/Tests/Framework/EndpointTests/Clusters/UnbalancedCluster.cs new file mode 100644 index 00000000000..c03070145ac --- /dev/null +++ b/src/Tests/Framework/EndpointTests/Clusters/UnbalancedCluster.cs @@ -0,0 +1,11 @@ +using System; +using Nest; + +namespace Tests.Framework.Integration +{ + public class UnbalancedCluster : ReadOnlyCluster + { + public override void Bootstrap() => + new Seeder(this.Node, new IndexSettings { NumberOfShards = 3, NumberOfReplicas = 2 }).SeedNode(); + } +} diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index 01375077b0c..ae6ba47d023 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -262,6 +262,7 @@ +