Merged with master

Directory.Build.props

@@ -3,7 +3,7 @@
 		<ClientOfficialVersion>3.37.1</ClientOfficialVersion>
 		<ClientPreviewVersion>3.37.1</ClientPreviewVersion>
 		<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
-		<DirectVersion>3.31.5</DirectVersion>
+		<DirectVersion>3.32.0</DirectVersion>
 		<EncryptionOfficialVersion>2.0.4</EncryptionOfficialVersion>
 		<EncryptionPreviewVersion>2.1.0</EncryptionPreviewVersion>
 		<EncryptionPreviewSuffixVersion>preview4</EncryptionPreviewSuffixVersion>

Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs

@@ -898,6 +898,19 @@ public async Task EncryptionReadManyItemAsync()
             }
         }
 
+        // ISSUE-TODO-VipulVishal - This test passes locally, but often fails in pre-checkin validation.
+        // Re-enable once the test is stabilized.
+        // Here's one example failure for reference:
+        // Test method Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.EncryptionChangeFeedDecryptionSuccessful threw exception:
+        // System.NullReferenceException: Object reference not set to an instance of an object.
+        //   at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.<>c__DisplayClass51_0.<ValidateChangeFeedProcessorResponse>b__2(TestDoc doc) in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 3152
+        // at System.Linq.Enumerable.WhereListIterator`1.MoveNext()
+        // at System.Linq.Enumerable.TryGetFirst[TSource](IEnumerable`1 source, Boolean& found)
+        // at System.Linq.Enumerable.FirstOrDefault[TSource] (IEnumerable`1 source)
+        // at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.ValidateChangeFeedProcessorResponse(Container container, TestDoc testDoc1, TestDoc testDoc2) in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 3152
+        // at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.EncryptionChangeFeedDecryptionSuccessful() in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 911
+        // at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.ThreadOperations.ExecuteWithAbortSafety(Action action)
+        [Ignore]
         [TestMethod]
         public async Task EncryptionChangeFeedDecryptionSuccessful()
         {

Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs

@@ -161,6 +161,17 @@ private async Task ProcessPartitionAsync(DocumentServiceLease lease)
             {
                 DefaultTrace.TraceVerbose("Lease with token {0}: processing canceled", lease.CurrentLeaseToken);
             }
+            catch (LeaseLostException leaseLostException)
+            {
+                // LeaseLostException by itself is not loggable, unless it contains a related inner exception
+                // For cases when the lease or container has been deleted or the lease has been stolen
+                if (leaseLostException.InnerException != null)
+                {
+                    await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, leaseLostException.InnerException);
+                }
+
+                DefaultTrace.TraceVerbose("Lease with token {0}: lease was lost", lease.CurrentLeaseToken);
+            }
             catch (Exception ex)
             {
                 await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex);

Microsoft.Azure.Cosmos/src/CosmosClient.cs

@@ -183,7 +183,9 @@ protected CosmosClient()
         /// </code>
         /// </example>
         /// <remarks>
-        /// The returned reference doesn't guarantee credentials or connectivity validations because creation doesn't make any network calls.
+        /// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;". 
+        /// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
+        /// NOTE: DO NOT use this flag in production (only for emulator)
         /// </remarks>
         /// <seealso cref="CosmosClientOptions"/>
         /// <seealso cref="Fluent.CosmosClientBuilder"/>
@@ -195,7 +197,7 @@ protected CosmosClient()
             : this(
                   CosmosClientOptions.GetAccountEndpoint(connectionString),
                   CosmosClientOptions.GetAccountKey(connectionString),
-                  clientOptions)
+                  CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, clientOptions))
         {
         }
 
@@ -495,6 +497,11 @@ protected CosmosClient()
         /// ]]>
         /// </code>
         /// </example>
+        /// <remarks>
+        /// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;". 
+        /// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
+        /// NOTE: DO NOT use this flag in production (only for emulator)
+        /// </remarks>
         public static async Task<CosmosClient> CreateAndInitializeAsync(string connectionString,
                                                                         IReadOnlyList<(string databaseId, string containerId)> containers,
                                                                         CosmosClientOptions cosmosClientOptions = null,
@@ -504,6 +511,7 @@ protected CosmosClient()
             {
                 throw new ArgumentNullException(nameof(containers));
             }
+            cosmosClientOptions = CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, cosmosClientOptions);
 
             CosmosClient cosmosClient = new CosmosClient(connectionString,
                                                          cosmosClientOptions);

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -51,6 +51,7 @@ public class CosmosClientOptions
 
         private const string ConnectionStringAccountEndpoint = "AccountEndpoint";
         private const string ConnectionStringAccountKey = "AccountKey";
+        private const string ConnectionStringDisableServerCertificateValidation = "DisableServerCertificateValidation";
 
         private const ApiType DefaultApiType = ApiType.None;
 
@@ -278,8 +279,8 @@ public ConnectionMode ConnectionMode
         /// If this is not set the database account consistency level will be used for all requests.
         /// </summary>
         public ConsistencyLevel? ConsistencyLevel { get; set; }
-
-        /// <summary>
+
+        /// <summary>
         /// Sets the priority level for requests created using cosmos client.
         /// </summary>
         /// <remarks>
@@ -666,7 +667,9 @@ internal Protocol ConnectionProtocol
         /// </summary>
         /// <remarks>
         /// <para>
-        /// Customizing SSL verification is not recommended in production environments.
+        /// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;". 
+        /// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
+        /// NOTE: DO NOT use this flag in production (only for emulator)
         /// </para>
         /// </remarks>
         public Func<X509Certificate2, X509Chain, SslPolicyErrors, bool> ServerCertificateCustomValidationCallback { get; set; }
@@ -744,6 +747,11 @@ internal Protocol ConnectionProtocol
         /// </summary>
         internal bool? EnableCpuMonitor { get; set; }
 
+        /// <summary>
+        /// Flag indicates the value of DisableServerCertificateValidation flag set at connection string level.Default it is false.
+        /// </summary>
+        internal bool DisableServerCertificateValidation { get; set; }
+
         /// <summary>
         /// Gets or sets Client Telemetry Options like feature flags and corresponding options
         /// </summary>
@@ -770,6 +778,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
             this.ValidateDirectTCPSettings();
             this.ValidateLimitToEndpointSettings();
             this.ValidatePartitionLevelFailoverSettings();
+            this.ValidateAndSetServerCallbackSettings();
 
             ConnectionPolicy connectionPolicy = new ConnectionPolicy()
             {
@@ -857,18 +866,34 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
 
             return (Documents.ConsistencyLevel)this.ConsistencyLevel.Value;
         }
-
+
         internal static string GetAccountEndpoint(string connectionString)
         {
-            return CosmosClientOptions.GetValueFromConnectionString(connectionString, CosmosClientOptions.ConnectionStringAccountEndpoint);
+            return CosmosClientOptions.GetValueFromConnectionString<string>(connectionString, CosmosClientOptions.ConnectionStringAccountEndpoint, null);
         }
 
         internal static string GetAccountKey(string connectionString)
         {
-            return CosmosClientOptions.GetValueFromConnectionString(connectionString, CosmosClientOptions.ConnectionStringAccountKey);
+            return CosmosClientOptions.GetValueFromConnectionString<string>(connectionString, CosmosClientOptions.ConnectionStringAccountKey, null);
+        }
+
+        internal static bool IsConnectionStringDisableServerCertificateValidationFlag(string connectionString)
+        {
+            return Convert.ToBoolean(CosmosClientOptions.GetValueFromConnectionString<bool>(connectionString, CosmosClientOptions.ConnectionStringDisableServerCertificateValidation, false));
+        }
+
+        internal static CosmosClientOptions GetCosmosClientOptionsWithCertificateFlag(string connectionString, CosmosClientOptions clientOptions)
+        {
+            clientOptions ??= new CosmosClientOptions();
+            if (CosmosClientOptions.IsConnectionStringDisableServerCertificateValidationFlag(connectionString))
+            {
+                clientOptions.DisableServerCertificateValidation = true;
+            }
+
+            return clientOptions;
         }
 
-        private static string GetValueFromConnectionString(string connectionString, string keyName)
+        private static T GetValueFromConnectionString<T>(string connectionString, string keyName, T defaultValue)
         {
             if (connectionString == null)
             {
@@ -881,8 +906,20 @@ private static string GetValueFromConnectionString(string connectionString, stri
                 string keyNameValue = value as string;
                 if (!string.IsNullOrEmpty(keyNameValue))
                 {
-                    return keyNameValue;
-                }
+                    try
+                    {
+                        return (T)Convert.ChangeType(value, typeof(T));
+                    }
+                    catch (InvalidCastException)
+                    {
+                        throw new ArgumentException("The connection string contains invalid property: " + keyName);
+                    }
+                }
+            }
+
+            if (defaultValue != null)
+            {
+                return defaultValue;
             }
 
             throw new ArgumentException("The connection string is missing a required property: " + keyName);
@@ -915,6 +952,19 @@ private void ValidatePartitionLevelFailoverSettings()
             }
         }
 
+        private void ValidateAndSetServerCallbackSettings()
+        {
+            if (this.DisableServerCertificateValidation && this.ServerCertificateCustomValidationCallback != null)
+            {
+                throw new ArgumentException($"Cannot specify {nameof(this.DisableServerCertificateValidation)} flag in Connection String and {nameof(this.ServerCertificateCustomValidationCallback)}. Only one can be set.");
+            }
+
+            if (this.DisableServerCertificateValidation)
+            {
+                this.ServerCertificateCustomValidationCallback = (_, _, _) => true;
+            }
+        }
+
         private void ValidateDirectTCPSettings()
         {
             string settingName = string.Empty;

Microsoft.Azure.Cosmos/src/DocumentClient.cs

@@ -115,7 +115,9 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider
         private readonly bool IsLocalQuorumConsistency = false;
         private readonly bool isReplicaAddressValidationEnabled;
 
-        private readonly IChaosInterceptor chaosInterceptor;
+        //Fault Injection
+        private readonly IChaosInterceptorFactory chaosInterceptorFactory;
+        private IChaosInterceptor chaosInterceptor;
 
         //Auth
         internal readonly AuthorizationTokenProvider cosmosAuthorization;
@@ -487,7 +489,7 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider
             this.transportClientHandlerFactory = transportClientHandlerFactory;
             this.IsLocalQuorumConsistency = isLocalQuorumConsistency;
             this.initTaskCache = new AsyncCacheNonBlocking<string, bool>(cancellationToken: this.cancellationTokenSource.Token);
-            this.chaosInterceptor = chaosInterceptorFactory?.CreateInterceptor(this);
+            this.chaosInterceptorFactory = chaosInterceptorFactory;
 
             this.Initialize(
                 serviceEndpoint: serviceEndpoint,
@@ -1015,6 +1017,14 @@ private async Task OpenPrivateAsync(CancellationToken cancellationToken)
         // Always called from under the lock except when called from Intilialize method during construction.
         private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeClientFactory)
         {
+            //Create the chaos interceptor if using fault injection
+            //Creating the chaos interceptor requires async calls, so we do it here instead of in the constructor
+            //This must also be done before creating the storeClientFactory for direct mode
+            if (this.chaosInterceptorFactory != null)
+            {
+                this.chaosInterceptor = await this.chaosInterceptorFactory.CreateInterceptorAsync(this);
+            }
+
             await this.InitializeGatewayConfigurationReaderAsync();
 
             if (this.desiredConsistencyLevel.HasValue)

Microsoft.Azure.Cosmos/src/FaultInjection/IChaosInterceptorFactory.cs

@@ -3,6 +3,7 @@
 //------------------------------------------------------------
 namespace Microsoft.Azure.Cosmos
 {
+    using System.Threading.Tasks;
     using Microsoft.Azure.Documents.FaultInjection;
 
     /// <summary>
@@ -15,6 +16,6 @@ internal interface IChaosInterceptorFactory
         /// Creates the IChaosInterceptor interceptor that will be used to inject fault injection rules. 
         /// </summary>
         /// <param name="documentClient"></param>
-        public IChaosInterceptor CreateInterceptor(DocumentClient documentClient);
+        public Task<IChaosInterceptor> CreateInterceptorAsync(DocumentClient documentClient);
     }
 }

Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs

@@ -124,6 +124,11 @@ public class CosmosClientBuilder
         /// </summary>
         /// <example>"AccountEndpoint=https://mytestcosmosaccount.documents.azure.com:443/;AccountKey={SecretAccountKey};"</example>
         /// <param name="connectionString">The connection string must contain AccountEndpoint and AccountKey or ResourceToken.</param>
+        /// <remarks>
+        /// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;". 
+        /// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
+        /// NOTE: DO NOT use this flag in production (only for emulator)
+        /// </remarks>
         public CosmosClientBuilder(string connectionString)
         {
             if (connectionString == null)
@@ -133,6 +138,8 @@ public CosmosClientBuilder(string connectionString)
 
             this.accountEndpoint = CosmosClientOptions.GetAccountEndpoint(connectionString);
             this.accountKey = CosmosClientOptions.GetAccountKey(connectionString);
+
+            this.clientOptions = CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, this.clientOptions);
         }
 
         /// <summary>

Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs

@@ -83,7 +83,7 @@ public ResponseMessage()
             this.CosmosException = cosmosException;
             this.Headers = headers ?? new Headers();
 
-            this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers, true);
+            this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers, isBase64Encoded: true);
 
             if (requestMessage != null && requestMessage.Trace != null)
             {
@@ -269,9 +269,9 @@ static internal Lazy<string> DecodeIndexMetrics(Headers responseMessageHeaders,
 
                             return stringBuilder.ToString();
                         }
-
-                        // Return the JSON from the response header
-                        return responseMessageHeaders.IndexUtilizationText;
+                        
+                        // Return the JSON from the response header after url decode
+                        return System.Web.HttpUtility.UrlDecode(responseMessageHeaders.IndexUtilizationText, Encoding.UTF8); 
                     });
             }
 

Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/AggregateItem.cs

@@ -5,6 +5,7 @@
 namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators
 {
     using System;
+    using System.Text.RegularExpressions;
     using Microsoft.Azure.Cosmos.CosmosElements;
 
     internal readonly struct AggregateItem
@@ -16,17 +17,31 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators
 
         public AggregateItem(CosmosElement cosmosElement)
         {
-            if (cosmosElement == null)
-            {
-                throw new ArgumentNullException($"{nameof(cosmosElement)} must not be null.");
-            }
+            // If the query is not a select value query then the top level is a an object
+            CosmosObject cosmosObject = cosmosElement as CosmosObject;
 
-            if (!(cosmosElement is CosmosObject cosmosObject))
+            if (cosmosObject == null)
             {
-                throw new ArgumentException($"{nameof(cosmosElement)} must not be an object.");
+                // In case of Aggregate query with VALUE query plan, the top level is an array of one item after it is rewritten
+                // For example, if the query is
+                // SELECT VALUE {"age": c.age}
+                // FROM c
+                // GROUP BY c.age
+                // Fhe rewritten query is 
+                // SELECT [{"item": c.age}] AS groupByItems, {"age": c.age} AS payload
+                // FROM c
+                // GROUP BY c.age
+
+                // In this case, the top level is an array of one item [{"item": c.age}]
+                CosmosArray cosmosArray = cosmosElement as CosmosArray;
+                if (cosmosArray.Count == 1)
+                {
+                    cosmosObject = cosmosArray[0] as CosmosObject;
+                }
             }
 
-            this.cosmosObject = cosmosObject;
+            // If the object is still null, then we have an invalid aggregate item
+            this.cosmosObject = cosmosObject ?? throw new ArgumentException($"Unsupported aggregate item. Expected CosmosObject"); 
         }
 
         public CosmosElement Item