diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index bd1b23b16c..b54c4837c4 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -29,6 +29,7 @@ internal sealed class ConnectionPolicy private Protocol connectionProtocol; private ObservableCollection preferredLocations; + private ObservableCollection accountInitializationCustomEndpoints; /// /// Initializes a new instance of the class to connect to the Azure Cosmos DB service. @@ -43,6 +44,7 @@ public ConnectionPolicy() this.MediaReadMode = MediaReadMode.Buffered; this.UserAgentContainer = new UserAgentContainer(clientId: 0); this.preferredLocations = new ObservableCollection(); + this.accountInitializationCustomEndpoints = new ObservableCollection(); this.EnableEndpointDiscovery = true; this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit; this.RetryOptions = new RetryOptions(); @@ -90,6 +92,27 @@ public void SetPreferredLocations(IReadOnlyList regions) } } + /// + /// Sets the custom private endpoints required to fetch account information from + /// private domain names. + /// + /// An instance of containing the custom DNS endpoints + /// provided by the customer. + public void SetAccountInitializationCustomEndpoints( + IEnumerable customEndpoints) + { + if (customEndpoints == null) + { + throw new ArgumentNullException(nameof(customEndpoints)); + } + + this.accountInitializationCustomEndpoints.Clear(); + foreach (Uri endpoint in customEndpoints) + { + this.accountInitializationCustomEndpoints.Add(endpoint); + } + } + /// /// Gets or sets the maximum number of concurrent fanout requests sent to the Azure Cosmos DB service. /// @@ -270,6 +293,24 @@ public Collection PreferredLocations } } + /// + /// Gets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// + /// + /// + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints + /// provided in the customAccountEndpoints list. + /// + /// + public Collection AccountInitializationCustomEndpoints + { + get + { + return this.accountInitializationCustomEndpoints; + } + } + /// /// Gets or sets the flag to enable endpoint discovery for geo-replicated database accounts in the Azure Cosmos DB service. /// diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 293712a64e..0c2d73a315 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -191,6 +191,42 @@ public string ApplicationName /// /// High availability on regional outages public IReadOnlyList ApplicationPreferredRegions { get; set; } + + /// + /// Gets and sets the custom endpoints to use for account initialization for geo-replicated database accounts in the Azure Cosmos DB service. + /// + /// + /// + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints provided in . + /// + /// + /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured an endpoint with a custom DNS hostname + /// (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. + /// + /// + /// See also Diagnose + /// and troubleshoot the availability of Cosmos SDKs for more details. + /// + /// + /// + /// + /// () + /// { + /// new Uri("custom.p-1.documents.azure.com"), + /// new Uri("custom.p-2.documents.azure.com") + /// } + /// }; + /// + /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions); + /// ]]> + /// + /// + /// High availability on regional outages + public IEnumerable AccountInitializationCustomEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target @@ -847,6 +883,11 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) List mappedRegions = this.ApplicationPreferredRegions.Select(s => mapper.GetCosmosDBRegionName(s)).ToList(); connectionPolicy.SetPreferredLocations(mappedRegions); + } + + if (this.AccountInitializationCustomEndpoints != null) + { + connectionPolicy.SetAccountInitializationCustomEndpoints(this.AccountInitializationCustomEndpoints); } if (this.MaxRetryAttemptsOnRateLimitedRequests != null) diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index a4d0381e6c..c56fe7feda 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -308,6 +308,41 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList return this; } + /// + /// Sets the custom endpoints to use for account initialization for geo-replicated database accounts in the Azure Cosmos DB service. + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints + /// provided in the customAccountEndpoints list. + /// + /// An instance of of Uri containing the custom private endpoints for the cosmos db account. + /// + /// This function is optional and is recommended for implementation when a customer has configured one or more endpoints with a custom DNS + /// hostname (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. + /// + /// + /// The example below creates a new instance of with the regional endpoints. + /// + /// () + /// { + /// new Uri("https://region-1.documents-test.windows-int.net:443/"), + /// new Uri("https://region-2.documents-test.windows-int.net:443/") + /// }); + /// CosmosClient client = cosmosClientBuilder.Build(); + /// ]]> + /// + /// + /// The current . + /// + public CosmosClientBuilder WithCustomAccountEndpoints(IEnumerable customAccountEndpoints) + { + this.clientOptions.AccountInitializationCustomEndpoints = customAccountEndpoints; + return this; + } + /// /// Limits the operations to the provided endpoint on the CosmosClientBuilder constructor. /// diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 5413103fee..1a2c3e14a4 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.Globalization; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -89,6 +88,7 @@ public async Task InitializeReaderAsync() AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( defaultEndpoint: this.serviceEndpoint, locations: this.connectionPolicy.PreferredLocations, + accountInitializationCustomEndpoints: this.connectionPolicy.AccountInitializationCustomEndpoints, getDatabaseAccountFn: this.GetDatabaseAccountAsync, cancellationToken: this.cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index db1a01cd61..2d60e57adb 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -116,13 +116,15 @@ public Uri GetHubUri() /// public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, - IList? locations, + IList? locations, + IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, - locations?.GetEnumerator(), + locations, + accountInitializationCustomEndpoints, getDatabaseAccountFn, cancellationToken)) { @@ -137,7 +139,8 @@ private class GetAccountPropertiesHelper : IDisposable { private readonly CancellationTokenSource CancellationTokenSource; private readonly Uri DefaultEndpoint; - private readonly IEnumerator? Locations; + private readonly bool LimitToGlobalEndpointOnly; + private readonly IEnumerator ServiceEndpointEnumerator; private readonly Func> GetDatabaseAccountFn; private readonly List TransientExceptions = new List(); private AccountProperties? AccountProperties = null; @@ -146,24 +149,31 @@ private class GetAccountPropertiesHelper : IDisposable public GetAccountPropertiesHelper( Uri defaultEndpoint, - IEnumerator? locations, + IList? locations, + IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; - this.Locations = locations; - this.GetDatabaseAccountFn = getDatabaseAccountFn; - this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.LimitToGlobalEndpointOnly = (locations == null || locations.Count == 0) && (accountInitializationCustomEndpoints == null || accountInitializationCustomEndpoints.Count == 0); + this.GetDatabaseAccountFn = getDatabaseAccountFn; + this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.ServiceEndpointEnumerator = GetAccountPropertiesHelper + .GetServiceEndpoints( + defaultEndpoint, + locations, + accountInitializationCustomEndpoints) + .GetEnumerator(); } public async Task GetAccountPropertiesAsync() { - // If there are no preferred regions then just wait for the global endpoint results - if (this.Locations == null) + // If there are no preferred regions or private endpoints, then just wait for the global endpoint results + if (this.LimitToGlobalEndpointOnly) { return await this.GetOnlyGlobalEndpointAsync(); } - + Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); // Start a timer to start secondary requests in parallel. @@ -219,9 +229,9 @@ public async Task GetAccountPropertiesAsync() private async Task GetOnlyGlobalEndpointAsync() { - if (this.Locations != null) + if (!this.LimitToGlobalEndpointOnly) { - throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other regions"); + throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other private endpoints or regions"); } await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); @@ -250,52 +260,53 @@ private async Task GetOnlyGlobalEndpointAsync() } /// - /// This is done in a thread safe way to allow multiple tasks to iterate over the - /// list of locations. + /// This is done in a thread safe way to allow multiple tasks to iterate over the list of service endpoints. /// private async Task TryGetAccountPropertiesFromAllLocationsAsync() - { - while (this.TryMoveNextLocationThreadSafe( - out string? location)) + { + while (this.TryMoveNextServiceEndpointhreadSafe( + out Uri? serviceEndpoint)) { - if (location == null) + if (serviceEndpoint == null) { - DefaultTrace.TraceCritical("GlobalEndpointManager: location is null for TryMoveNextLocationThreadSafe"); + DefaultTrace.TraceCritical("GlobalEndpointManager: serviceEndpoint is null for TryMoveNextServiceEndpointhreadSafe."); return; } - await this.TryGetAccountPropertiesFromRegionalEndpointsAsync(location); + await this.GetAndUpdateAccountPropertiesAsync( + endpoint: serviceEndpoint); } - } - - private bool TryMoveNextLocationThreadSafe( - out string? location) + } + + /// + /// We first iterate through all the private endpoints to fetch the account information. + /// If all the attempt fails to fetch the metadata from the private endpoints, we will + /// attempt to retrieve the account information from the regional endpoints constructed + /// using the preferred regions list. + /// + /// An instance of that will contain the service endpoint. + /// A boolean flag indicating if the was advanced in a thread safe manner. + private bool TryMoveNextServiceEndpointhreadSafe( + out Uri? serviceEndpoint) { - if (this.CancellationTokenSource.IsCancellationRequested - || this.Locations == null) + if (this.CancellationTokenSource.IsCancellationRequested) { - location = null; + serviceEndpoint = null; return false; } - lock (this.Locations) + lock (this.ServiceEndpointEnumerator) { - if (!this.Locations.MoveNext()) + if (!this.ServiceEndpointEnumerator.MoveNext()) { - location = null; + serviceEndpoint = null; return false; } - location = this.Locations.Current; + serviceEndpoint = this.ServiceEndpointEnumerator.Current; return true; } - } - - private Task TryGetAccountPropertiesFromRegionalEndpointsAsync(string location) - { - return this.GetAndUpdateAccountPropertiesAsync( - LocationHelper.GetLocationEndpoint(this.DefaultEndpoint, location)); - } + } private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { @@ -305,7 +316,7 @@ private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { lock (this.TransientExceptions) { - this.TransientExceptions.Add(new OperationCanceledException("GlobalEndpointManager: Get account information canceled")); + this.TransientExceptions.Add(new OperationCanceledException($"GlobalEndpointManager: Get account information canceled for URI: {endpoint}")); } return; @@ -349,6 +360,40 @@ private static bool IsNonRetriableException(Exception exception) return false; } + /// + /// Returns an instance of containing the private and regional service endpoints to iterate over. + /// + /// An instance of containing the default global endpoint. + /// An instance of containing the preferred serviceEndpoint names. + /// An instance of containing the custom private endpoints. + /// An instance of containing the service endpoints. + private static IEnumerable GetServiceEndpoints( + Uri defaultEndpoint, + IList? locations, + IList? accountInitializationCustomEndpoints) + { + // We first iterate over all the private endpoints and yield return them. + if (accountInitializationCustomEndpoints?.Count > 0) + { + foreach (Uri customEndpoint in accountInitializationCustomEndpoints) + { + // Yield return all of the custom private endpoints first. + yield return customEndpoint; + } + } + + // The next step is to iterate over the preferred locations, construct and yield return the regional endpoints one by one. + // The regional endpoints will be constructed by appending the preferred region name as a suffix to the default global endpoint. + if (locations?.Count > 0) + { + foreach (string location in locations) + { + // Yield return all of the regional endpoints once the private custom endpoints are visited. + yield return LocationHelper.GetLocationEndpoint(defaultEndpoint, location); + } + } + } + public void Dispose() { if (Interlocked.Increment(ref this.disposeCounter) == 1) @@ -390,7 +435,7 @@ public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() } /// - /// Returns location corresponding to the endpoint + /// Returns serviceEndpoint corresponding to the endpoint /// /// public string GetLocation(Uri endpoint) @@ -530,7 +575,7 @@ private async void StartLocationBackgroundRefreshLoop() return; } - DefaultTrace.TraceCritical("GlobalEndpointManager: StartLocationBackgroundRefreshWithTimer() - Unable to refresh database account from any location. Exception: {0}", ex.ToString()); + DefaultTrace.TraceCritical("GlobalEndpointManager: StartLocationBackgroundRefreshWithTimer() - Unable to refresh database account from any serviceEndpoint. Exception: {0}", ex.ToString()); } // Call itself to create a loop to continuously do background refresh every 5 minutes @@ -549,7 +594,7 @@ private void OnPreferenceChanged(object sender, NotifyCollectionChangedEventArgs } /// - /// Thread safe refresh account and location info. + /// Thread safe refresh account and serviceEndpoint info. /// private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) { @@ -607,7 +652,8 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh obsoleteValue: null, singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, - this.connectionPolicy.PreferredLocations, + this.connectionPolicy.PreferredLocations, + this.connectionPolicy.AccountInitializationCustomEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), cancellationToken: this.cancellationTokenSource.Token, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 3ce587dcd3..d7335051ae 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -2770,6 +2770,18 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.CosmosSerializer Serializer;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.CosmosSerializer get_Serializer();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Serializer(Microsoft.Azure.Cosmos.CosmosSerializer);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Collections.Generic.IEnumerable`1[System.Uri] AccountInitializationCustomEndpoints": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.Uri] AccountInitializationCustomEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions": { "Type": "Property", "Attributes": [], @@ -2993,6 +3005,13 @@ "Attributes": [], "MethodInfo": "[Void .ctor(), Void .ctor()]" }, + "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_AllowBulkExecution(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -4603,6 +4622,11 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithContentResponseOnWrite(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri])": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomSerializer(Microsoft.Azure.Cosmos.CosmosSerializer)": { "Type": "Method", "Attributes": [], diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 274723f24e..293c3b14ab 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -162,7 +162,13 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsFalse(policy.EnablePartitionLevelFailover); Assert.IsTrue(clientOptions.EnableAdvancedReplicaSelectionForTcp.Value); - IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; + IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; + ISet regionalEndpoints = new HashSet() + { + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/") + }; + //Verify Direct Mode settings cosmosClientBuilder = new CosmosClientBuilder( accountEndpoint: endpoint, @@ -174,7 +180,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() maxTcpConnectionsPerEndpoint, portReuseMode, enableTcpConnectionEndpointRediscovery) - .WithApplicationPreferredRegions(preferredLocations) + .WithApplicationPreferredRegions(preferredLocations) + .WithCustomAccountEndpoints(regionalEndpoints) .WithClientTelemetryOptions(new CosmosClientTelemetryOptions() { DisableDistributedTracing = false, @@ -194,7 +201,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(maxTcpConnectionsPerEndpoint, clientOptions.MaxTcpConnectionsPerEndpoint); Assert.AreEqual(portReuseMode, clientOptions.PortReuseMode); Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery); - CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray()); + CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray()); + CollectionAssert.AreEqual(regionalEndpoints.ToArray(), clientOptions.AccountInitializationCustomEndpoints.ToArray()); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.PointOperationLatencyThreshold); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.NonPointOperationLatencyThreshold); Assert.IsFalse(clientOptions.CosmosClientTelemetryOptions.DisableDistributedTracing); @@ -208,6 +216,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(portReuseMode, policy.PortReuseMode); Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery); CollectionAssert.AreEqual(preferredLocations.ToArray(), policy.PreferredLocations.ToArray()); + CollectionAssert.AreEqual(regionalEndpoints.ToArray(), policy.AccountInitializationCustomEndpoints.ToArray()); } /// @@ -331,6 +340,13 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Regions.NorthCentralUS, Regions.WestUS, Regions.EastAsia, + }) + .WithCustomAccountEndpoints( + new HashSet() + { + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed3.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/"), }); CosmosClientOptions clientOptions = cosmosClientBuilder.Build().ClientOptions; @@ -348,7 +364,8 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Assert.IsFalse(clientOptions.AllowBulkExecution); Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsTrue(clientOptions.EnablePartitionLevelFailover); - Assert.IsNotNull(clientOptions.ApplicationPreferredRegions); + Assert.IsNotNull(clientOptions.ApplicationPreferredRegions); + Assert.IsNotNull(clientOptions.AccountInitializationCustomEndpoints); } finally { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs index 6740001f53..353e94b642 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs @@ -138,7 +138,7 @@ public async Task InvalidKey_ExceptionFullStacktrace(string endpoint, string key } catch (Exception ex) { - Assert.IsTrue(ex.StackTrace.Contains("GatewayAccountReader.GetDatabaseAccountAsync"), ex.StackTrace); + Assert.IsTrue(ex.StackTrace.Contains("GatewayAccountReader.InitializeReaderAsync"), ex.StackTrace); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index cf54b12a52..41e447ac35 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -16,6 +16,7 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.Tests; using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Cosmos.Tracing.TraceData; + using System.Collections.Generic; /// /// Tests for . @@ -100,6 +101,157 @@ public void DocumentClient_BuildHttpClientFactory_WithFactory() .Verify(f => f(), Times.Once); } + [TestMethod] + [Owner("dkunda")] + [DataRow(true, DisplayName = "Validate that when custom endpoints are provided in the connection policy, the request will be retried in the regional endpoints.")] + [DataRow(false, DisplayName = "Validate that when custom endpoints are not provided in the connection policy, the request will be failed in the primary endpoint.")] + public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryWithPrivateCustomEndpointsWhenPrimaryFails( + bool customEndpointsProvided) + { + string accountPropertiesResponse = "{\r\n \"_self\": \"\",\r\n \"id\": \"localhost\",\r\n \"_rid\": \"127.0.0.1\",\r\n \"media\": \"//media/\",\r\n \"addresses\": \"//addresses/\",\r\n \"_dbs\": \"//dbs/\",\r\n \"writableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }" + + "\r\n ],\r\n \"readableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }\r\n ],\r\n \"enableMultipleWriteLocations\": false,\r\n \"userReplicationPolicy\": {\r\n \"asyncReplication\": false,\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"userConsistencyPolicy\": {\r\n " + + "\"defaultConsistencyLevel\": \"Session\"\r\n },\r\n \"systemReplicationPolicy\": {\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"readPolicy\": {\r\n \"primaryReadCoefficient\": 1,\r\n \"secondaryReadCoefficient\": 1\r\n },\r\n \"queryEngineConfiguration\": \"{\\\"maxSqlQueryInputLength\\\":262144,\\\"maxJoinsPerSqlQuery\\\":5," + + "\\\"maxLogicalAndPerSqlQuery\\\":500,\\\"maxLogicalOrPerSqlQuery\\\":500,\\\"maxUdfRefPerSqlQuery\\\":10,\\\"maxInExpressionItemsCount\\\":16000,\\\"queryMaxInMemorySortDocumentCount\\\":500,\\\"maxQueryRequestTimeoutFraction\\\":0.9,\\\"sqlAllowNonFiniteNumbers\\\":false,\\\"sqlAllowAggregateFunctions\\\":true,\\\"sqlAllowSubQuery\\\":true,\\\"sqlAllowScalarSubQuery\\\":true,\\\"allowNewKeywords\\\":true,\\\"" + + "sqlAllowLike\\\":true,\\\"sqlAllowGroupByClause\\\":true,\\\"maxSpatialQueryCells\\\":12,\\\"spatialMaxGeometryPointCount\\\":256,\\\"sqlDisableOptimizationFlags\\\":0,\\\"sqlAllowTop\\\":true,\\\"enableSpatialIndexing\\\":true}\"\r\n}"; + + Uri globalEndpoint = new("https://testfed1.documents-test.windows-int.net:443/"); + Uri privateEndpoint1 = new ("https://testfed2.documents-test.windows-int.net:443/"); + Uri privateEndpoint2 = new ("https://testfed3.documents-test.windows-int.net:443/"); + Uri privateEndpoint3 = new ("https://testfed4.documents-test.windows-int.net:443/"); + Uri endpointSucceeded = default; + + StringContent content = new(accountPropertiesResponse); + HttpResponseMessage responseMessage = new() + { + StatusCode = HttpStatusCode.OK, + Content = content, + }; + + Mock mockHttpClient = new(); + + GatewayAccountReaderTests.SetupMockToThrowException( + mockHttpClient: mockHttpClient, + endpoints: new List() + { + globalEndpoint, + privateEndpoint1, + privateEndpoint2, + }); + + mockHttpClient + .Setup(x => x.GetAsync( + privateEndpoint3, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(( + Uri serviceEndpoint, + INameValueCollection _, + ResourceType _, + HttpTimeoutPolicy _, + IClientSideRequestStatistics _, + CancellationToken _) => endpointSucceeded = serviceEndpoint) + .ReturnsAsync(responseMessage); + + ConnectionPolicy connectionPolicy = new() + { + ConnectionMode = ConnectionMode.Direct, + }; + + if (customEndpointsProvided) + { + connectionPolicy.SetAccountInitializationCustomEndpoints( + new HashSet() + { + privateEndpoint1, + privateEndpoint2, + privateEndpoint3, + }); + } + + GatewayAccountReader accountReader = new GatewayAccountReader( + serviceEndpoint: globalEndpoint, + cosmosAuthorization: Mock.Of(), + connectionPolicy: connectionPolicy, + httpClient: mockHttpClient.Object); + + if (customEndpointsProvided) + { + AccountProperties accountProperties = await accountReader.InitializeReaderAsync(); + + Assert.IsNotNull(accountProperties); + Assert.AreEqual("localhost", accountProperties.Id); + Assert.AreEqual("127.0.0.1", accountProperties.ResourceId); + Assert.AreEqual(endpointSucceeded, privateEndpoint3); + } + else + { + Exception exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + Assert.IsNull(endpointSucceeded); + Assert.IsNotNull(exception); + Assert.AreEqual("Service is Unavailable at the Moment.", exception.Message); + } + } + + [TestMethod] + [Owner("dkunda")] + public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThrowAggregateExceptionWithAllEndpointsFail() + { + Mock mockHttpClient = new(); + mockHttpClient + .Setup(x => x.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); + + ConnectionPolicy connectionPolicy = new() + { + ConnectionMode = ConnectionMode.Direct, + }; + + connectionPolicy.SetAccountInitializationCustomEndpoints( + new HashSet() + { + new ("https://testfed2.documents-test.windows-int.net:443/"), + new ("https://testfed3.documents-test.windows-int.net:443/"), + new ("https://testfed4.documents-test.windows-int.net:443/"), + }); + + GatewayAccountReader accountReader = new GatewayAccountReader( + serviceEndpoint: new Uri("https://testfed1.documents-test.windows-int.net:443/"), + cosmosAuthorization: Mock.Of(), + connectionPolicy: connectionPolicy, + httpClient: mockHttpClient.Object); + + AggregateException exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + Assert.IsNotNull(exception); + Assert.AreEqual("Service is Unavailable at the Moment.", exception.InnerException.Message); + } + + private static void SetupMockToThrowException( + Mock mockHttpClient, + IList endpoints) + { + foreach(Uri endpoint in endpoints) + { + mockHttpClient + .Setup(x => x.GetAsync( + endpoint, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); + } + } + public class CustomMessageHandler : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index 30c0d4e457..931e1fc8b4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -118,6 +118,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), cancellationTokenSource.Token); @@ -147,6 +148,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -177,6 +179,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: async (uri) => { count++; @@ -208,6 +211,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -239,6 +243,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -307,6 +312,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -329,6 +335,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); stopwatch.Stop(); @@ -352,6 +359,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -373,6 +381,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -398,6 +407,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "westus6", "westus7", }, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default);