Implement hidden indices and aliases

Relates: elastic/elasticsearch#52547
Relates: elastic/elasticsearch#53248

This commit adds the ability to mark indices
and aliases as hidden.

src/Nest/IndexModules/IndexSettings/Settings/FixedIndexSettings.cs

@@ -10,6 +10,12 @@ public static class FixedIndexSettings
 		public const string NumberOfShards = "index.number_of_shards";
 		public const string RoutingPartitionSize = "index.routing_partition_size";
 
+		/// <summary>
+		/// Indicates whether the index should be hidden by default.
+		/// Hidden indices are not returned by default when using a wildcard expression.
+		/// </summary>
+		public const string Hidden = "index.hidden";
+
 		/// <summary>
 		/// If a field referred to in a percolator query does not exist,
 		/// it will be handled as a default text field so that adding the percolator query doesn't fail.

src/Nest/IndexModules/IndexSettings/Settings/IndexSettings.cs

@@ -39,6 +39,12 @@ public interface IIndexSettings : IDynamicIndexSettings
 		/// </summary>
 		int? RoutingPartitionSize { get; set; }
 
+		/// <summary>
+		/// Indicates whether the index should be hidden by default.
+		/// Hidden indices are not returned by default when using a wildcard expression.
+		/// </summary>
+		bool? Hidden { get; set; }
+
 		/// <summary>
 		///  Settings associated with index sorting.
 		/// https://www.elastic.co/guide/en/elasticsearch/reference/6.0/index-modules-index-sorting.html
@@ -71,6 +77,9 @@ public class IndexSettings : DynamicIndexSettings, IIndexSettings
 		/// <inheritdoc cref="IIndexSettings.RoutingPartitionSize" />
 		public int? RoutingPartitionSize { get; set; }
 
+		/// <inheritdoc cref="IIndexSettings.Hidden" />
+		public bool? Hidden { get; set; }
+
 		/// <inheritdoc cref="IIndexSettings.Sorting" />
 		public ISortingSettings Sorting { get; set; }
 
@@ -95,6 +104,10 @@ public class IndexSettingsDescriptor : DynamicIndexSettingsDescriptorBase<IndexS
 		public IndexSettingsDescriptor RoutingPartitionSize(int? routingPartitionSize) =>
 			Assign(routingPartitionSize, (a, v) => a.RoutingPartitionSize = v);
 
+		/// <inheritdoc cref="IIndexSettings.Hidden" />
+		public IndexSettingsDescriptor Hidden(bool? hidden = true) =>
+			Assign(hidden, (a, v) => a.Hidden = v);
+
 		/// <inheritdoc cref="IIndexSettings.FileSystemStorageImplementation" />
 		public IndexSettingsDescriptor FileSystemStorageImplementation(FileSystemStorageImplementation? fs) =>
 			Assign(fs, (a, v) => a.FileSystemStorageImplementation = v);

src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsFormatter.cs

@@ -124,6 +124,7 @@ void Set(string knownKey, object newValue)
 				Set(NumberOfShards, indexSettings.NumberOfShards);
 				Set(NumberOfRoutingShards, indexSettings.NumberOfRoutingShards);
 				Set(RoutingPartitionSize, indexSettings.RoutingPartitionSize);
+				Set(Hidden, indexSettings.Hidden);
 				if (indexSettings.SoftDeletes != null)
 				{
 #pragma warning disable 618
@@ -159,7 +160,7 @@ private static object AsArrayOrSingleItem<T>(IEnumerable<T> items)
 			Dictionary<string, object> current = null
 		)
 		{
-			current = current ?? new Dictionary<string, object>();
+			current ??= new Dictionary<string, object>();
 			foreach (var property in original)
 			{
 				if (property.Value is Dictionary<string, object> objects &&
@@ -251,6 +252,7 @@ private static void SetKnownIndexSettings(ref JsonReader reader, IJsonFormatterR
 			Set<int?>(s, settings, NumberOfShards, v => s.NumberOfShards = v, formatterResolver);
 			Set<int?>(s, settings, NumberOfRoutingShards, v => s.NumberOfRoutingShards = v, formatterResolver);
 			Set<int?>(s, settings, RoutingPartitionSize, v => s.RoutingPartitionSize = v, formatterResolver);
+			Set<bool?>(s, settings, Hidden, v => s.Hidden = v, formatterResolver);
 			Set<FileSystemStorageImplementation?>(s, settings, StoreType, v => s.FileSystemStorageImplementation = v, formatterResolver);
 
 			var sorting = s.Sorting = new SortingSettings();

src/Nest/Indices/AliasManagement/Alias.cs

@@ -23,10 +23,21 @@ public interface IAlias
 		[DataMember(Name = "index_routing")]
 		Routing IndexRouting { get; set; }
 
-		/// <inheritdoc cref="AliasAddOperation.IsWriteIndex" />
+		/// <summary>
+		/// If an alias points to multiple indices, Elasticsearch will reject the write operations
+		/// unless one is explicitly marked as the write alias using this property.
+		/// </summary>
 		[DataMember(Name = "is_write_index")]
 		bool? IsWriteIndex { get; set; }
 
+		/// <summary>
+		/// If true, the alias will be excluded from wildcard expressions by default, unless overriden in the request using
+		/// the expand_wildcards parameter, similar to hidden indices.
+		/// This property must be set to the same value on all indices that share an alias. Defaults to false.
+		/// </summary>
+		[DataMember(Name = "is_hidden")]
+		bool? IsHidden { get; set; }
+
 		/// <summary>
 		/// Associates routing values with aliases for both index and search operations. This feature can be used together
 		/// with filtering aliases in order to avoid unnecessary shard operations.
@@ -51,6 +62,9 @@ public class Alias : IAlias
 		public Routing IndexRouting { get; set; }
 		/// <inheritdoc />
 		public bool? IsWriteIndex { get; set; }
+		/// <inheritdoc />
+		public bool? IsHidden { get; set; }
+
 		/// <inheritdoc />
 		public Routing Routing { get; set; }
 		/// <inheritdoc />
@@ -65,6 +79,7 @@ public class AliasDescriptor : DescriptorBase<AliasDescriptor, IAlias>, IAlias
 		bool? IAlias.IsWriteIndex { get; set; }
 		Routing IAlias.Routing { get; set; }
 		Routing IAlias.SearchRouting { get; set; }
+		bool? IAlias.IsHidden { get; set; }
 
 		/// <inheritdoc cref="IAlias.Filter" />
 		public AliasDescriptor Filter<T>(Func<QueryContainerDescriptor<T>, QueryContainer> filterSelector) where T : class =>
@@ -76,6 +91,9 @@ public class AliasDescriptor : DescriptorBase<AliasDescriptor, IAlias>, IAlias
 		/// <inheritdoc cref="IAlias.IsWriteIndex" />
 		public AliasDescriptor IsWriteIndex(bool? isWriteIndex = true) => Assign(isWriteIndex, (a, v) => a.IsWriteIndex = v);
 
+		/// <inheritdoc cref="IAlias.IsHidden" />
+		public AliasDescriptor IsHidden(bool? isHidden = true) => Assign(isHidden, (a, v) => a.IsHidden = v);
+
 		/// <inheritdoc cref="IAlias.Routing" />
 		public AliasDescriptor Routing(Routing routing) => Assign(routing, (a, v) => a.Routing = v);
 

src/Nest/Indices/AliasManagement/Alias/Actions/AliasAdd.cs

@@ -22,54 +22,70 @@ public class AliasAddDescriptor : DescriptorBase<AliasAddDescriptor, IAliasAddAc
 
 		AliasAddOperation IAliasAddAction.Add { get; set; }
 
+		/// <inheritdoc cref="AliasAddOperation.Index"/>
 		public AliasAddDescriptor Index(string index)
 		{
 			Self.Add.Index = index;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.Index"/>
 		public AliasAddDescriptor Index(Type index)
 		{
 			Self.Add.Index = index;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.Index"/>
 		public AliasAddDescriptor Index<T>() where T : class
 		{
 			Self.Add.Index = typeof(T);
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.Alias"/>
 		public AliasAddDescriptor Alias(string alias)
 		{
 			Self.Add.Alias = alias;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.Routing"/>
 		public AliasAddDescriptor Routing(string routing)
 		{
 			Self.Add.Routing = routing;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.IndexRouting"/>
 		public AliasAddDescriptor IndexRouting(string indexRouting)
 		{
 			Self.Add.IndexRouting = indexRouting;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.SearchRouting"/>
 		public AliasAddDescriptor SearchRouting(string searchRouting)
 		{
 			Self.Add.SearchRouting = searchRouting;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.IsWriteIndex"/>
 		public AliasAddDescriptor IsWriteIndex(bool? isWriteIndex = true)
 		{
 			Self.Add.IsWriteIndex = isWriteIndex;
 			return this;
 		}
 
+		/// <inheritdoc cref="AliasAddOperation.IsHidden"/>
+		public AliasAddDescriptor IsHidden(bool? isHidden = true)
+		{
+			Self.Add.IsHidden = isHidden;
+			return this;
+		}
+
+		/// <inheritdoc cref="AliasAddOperation.Filter"/>
 		public AliasAddDescriptor Filter<T>(Func<QueryContainerDescriptor<T>, QueryContainer> filterSelector)
 			where T : class
 		{

src/Nest/Indices/AliasManagement/Alias/Actions/AliasAddOperation.cs

@@ -4,28 +4,42 @@ namespace Nest
 {
 	public class AliasAddOperation
 	{
+		/// <summary>
+		/// The name of the alias
+		/// </summary>
 		[DataMember(Name ="alias")]
 		public string Alias { get; set; }
 
+		/// <summary>
+		/// Filter query used to limit the index alias.
+		/// If specified, the index alias only applies to documents returned by the filter.
+		/// </summary>
 		[DataMember(Name ="filter")]
 		public QueryContainer Filter { get; set; }
 
+		/// <summary>
+		/// The index to which to add the alias
+		/// </summary>
 		[DataMember(Name ="index")]
 		public IndexName Index { get; set; }
 
+		/// <inheritdoc cref="IAlias.IndexRouting"/>
 		[DataMember(Name ="index_routing")]
 		public string IndexRouting { get; set; }
 
-		/// <summary>
-		/// If an alias points to multiple indices, Elasticsearch will reject the write operations
-		/// unless one is explicitly marked as the write alias using this property.
-		/// </summary>
+		/// <inheritdoc cref="IAlias.IsWriteIndex"/>
 		[DataMember(Name ="is_write_index")]
 		public bool? IsWriteIndex { get; set; }
 
+		/// <inheritdoc cref="IAlias.IsHidden"/>
+		[DataMember(Name ="is_hidden")]
+		public bool? IsHidden { get; set; }
+
+		/// <inheritdoc cref="IAlias.Routing"/>
 		[DataMember(Name ="routing")]
 		public string Routing { get; set; }
 
+		/// <inheritdoc cref="IAlias.SearchRouting"/>
 		[DataMember(Name ="search_routing")]
 		public string SearchRouting { get; set; }
 	}

tests/Tests/Indices/IndexManagement/CreateIndex/CreateIndexApiTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using Elastic.Xunit.XunitPlumbing;
 using Elasticsearch.Net;
 using FluentAssertions;
 using Nest;
@@ -317,4 +318,90 @@ protected override void ExpectResponse(CreateIndexResponse response)
 			aliases[CallIsolatedValue + "-alias"].IsWriteIndex.Should().BeTrue();
 		}
 	}
+
+
+	[SkipVersion("<7.7.0", "hidden indices and aliases introduced in 7.7.0")]
+	public class CreateHiddenIndexApiTests
+		: ApiIntegrationTestBase<WritableCluster, CreateIndexResponse, ICreateIndexRequest, CreateIndexDescriptor, CreateIndexRequest>
+	{
+		public CreateHiddenIndexApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override bool ExpectIsValid => true;
+
+		protected override object ExpectJson => new
+		{
+			settings = new Dictionary<string, object>
+			{
+				{ "index.number_of_replicas", 0 },
+				{ "index.number_of_shards", 1 },
+				{ "index.hidden", true }
+			},
+			aliases = new Dictionary<string, object>
+			{
+				{ CallIsolatedValue + "-alias", new { is_write_index = true, is_hidden = true } }
+			}
+		};
+
+		protected override int ExpectStatusCode => 200;
+
+		protected override Func<CreateIndexDescriptor, ICreateIndexRequest> Fluent => d => d
+			.Settings(s => s
+				.NumberOfReplicas(0)
+				.NumberOfShards(1)
+				.Hidden()
+			)
+			.Aliases(a => a
+				.Alias(CallIsolatedValue + "-alias", aa => aa
+					.IsWriteIndex()
+					.IsHidden()
+				)
+			);
+
+		protected override HttpMethod HttpMethod => HttpMethod.PUT;
+
+		protected override CreateIndexRequest Initializer => new CreateIndexRequest(CallIsolatedValue)
+		{
+			Settings = new Nest.IndexSettings
+			{
+				NumberOfReplicas = 0,
+				NumberOfShards = 1,
+				Hidden = true
+			},
+			Aliases = new Aliases
+			{
+				{ CallIsolatedValue + "-alias", new Alias { IsWriteIndex = true, IsHidden = true} }
+			}
+		};
+
+		protected override string UrlPath => $"/{CallIsolatedValue}";
+
+		protected override LazyResponses ClientUsage() => Calls(
+			(client, f) => client.Indices.Create(CallIsolatedValue, f),
+			(client, f) => client.Indices.CreateAsync(CallIsolatedValue, f),
+			(client, r) => client.Indices.Create(r),
+			(client, r) => client.Indices.CreateAsync(r)
+		);
+
+		protected override CreateIndexDescriptor NewDescriptor() => new CreateIndexDescriptor(CallIsolatedValue);
+
+		protected override void ExpectResponse(CreateIndexResponse response)
+		{
+			response.ShouldBeValid();
+			response.Acknowledged.Should().BeTrue();
+			response.ShardsAcknowledged.Should().BeTrue();
+
+			var indexResponse = Client.Indices.Get(CallIsolatedValue);
+
+			indexResponse.ShouldBeValid();
+			indexResponse.Indices.Should().NotBeEmpty().And.ContainKey(CallIsolatedValue);
+			var index = indexResponse.Indices[CallIsolatedValue];
+
+			index.Settings.Hidden.Should().BeTrue();
+
+			var aliases = indexResponse.Indices[CallIsolatedValue].Aliases;
+			aliases.Count.Should().Be(1);
+			aliases[CallIsolatedValue + "-alias"].IsWriteIndex.Should().BeTrue();
+			aliases[CallIsolatedValue + "-alias"].IsHidden.Should().BeTrue();
+		}
+	}
 }