Fixes for N:1 Issues over WebAPI, Adding new Property hint for recommended degree's of Parallelism

src/Build.Shared.props

@@ -5,8 +5,9 @@
     <PackageVersion_AppInsights>2.9.1</PackageVersion_AppInsights>
     <PackageVersion_Adal>3.19.8</PackageVersion_Adal>
     <PackageVersion_MSAL>4.35.1</PackageVersion_MSAL>
-    <PackageVersion_CdsSdk>4.6.4784-weekly-2107.4</PackageVersion_CdsSdk>
-    <PackageVersion_CDSServerNuget>4.6.4784-weekly-2107.4</PackageVersion_CDSServerNuget>
+    <PackageVersion_CdsSdk>4.6.6061-weekly-2108.5</PackageVersion_CdsSdk>
+    <PackageVersion_CDSServerNuget>4.6.6061-weekly-2108.5</PackageVersion_CDSServerNuget>
+    <PackageVersion_CdsSdkProxy>4.7.6346-master</PackageVersion_CdsSdkProxy>
     <PackageVersion_Newtonsoft>11.0.2</PackageVersion_Newtonsoft>
     <PackageVersion_RestClientRuntime>2.3.20</PackageVersion_RestClientRuntime>
     <PackageVersion_XrmSdk>9.0.2.42</PackageVersion_XrmSdk>

src/GeneralTools/DataverseClient/Client/ConnectionService.cs

@@ -232,6 +232,17 @@ internal sealed class ConnectionService : IConnectionService, IDisposable
         /// </summary>
         private IOrganizationService _testSupportIOrg;
 
+        /// <summary>
+        /// Value used by the retry system while the code is running,
+        /// this value can scale up and down based on throttling limits.
+        /// </summary>
+        private TimeSpan _retryPauseTimeRunning;
+
+        /// <summary>
+        /// Known types factory
+        /// </summary>
+        private KnownTypesFactory _knownTypesFactory = new KnownTypesFactory();
+
         #endregion
 
         #region Properties
@@ -580,17 +591,11 @@ internal bool EnableCookieRelay
         /// Cookies that are being passed though clients, when cookies are used
         /// </summary>
         internal Dictionary<string, string> CurrentCookieCollection { get; set; } = null;
-
+        
         /// <summary>
-        /// Value used by the retry system while the code is running,
-        /// this value can scale up and down based on throttling limits.
-        /// </summary>
-        private TimeSpan _retryPauseTimeRunning;
-
-        /// <summary>
-        /// Known types factory
+        /// Server Hint for the number of concurrent threads that would provbide optimal processing. 
         /// </summary>
-        private KnownTypesFactory _knownTypesFactory = new KnownTypesFactory();
+        internal int RecommendedDegreesOfParallelism { get; set; } = 5; // Default value. 
 
         #endregion
 
@@ -601,7 +606,7 @@ internal bool EnableCookieRelay
         /// <param name="baseConnectUrl"></param>
         /// <param name="mockClient"></param>
         /// <param name="logger"></param>
-        internal ConnectionService(IOrganizationService testIOrganziationSvc , string baseConnectUrl, HttpClient mockClient, ILogger logger)
+        internal ConnectionService(IOrganizationService testIOrganziationSvc, string baseConnectUrl, HttpClient mockClient, ILogger logger)
         {
             _testSupportIOrg = testIOrganziationSvc;
             WebApiHttpClient = mockClient;
@@ -1371,7 +1376,7 @@ private async Task RefreshInstanceDetails(IOrganizationService dvService, Uri ur
                         {
                             resp = (RetrieveCurrentOrganizationResponse)dvService.Execute(request);
                         }
-                        catch(Exception ex)
+                        catch (Exception ex)
                         {
                             dtQueryTimer.Stop();
                             logEntry.Log(string.Format(CultureInfo.InvariantCulture, "Failed to Executed Command - RetrieveCurrentOrganizationRequest : RequestId={1} : total duration: {0}", dtQueryTimer.Elapsed.ToString(), trackingID.ToString()), TraceEventType.Error);
@@ -1601,7 +1606,7 @@ internal async Task<OrganizationResponse> Command_WebAPIProcess_ExecuteAsync(Org
             if (cReq != null)
             {
                 cancellationToken.ThrowIfCancellationRequested();
-                requestBodyObject = Utilities.ToExpandoObject(cReq, metadataUtlity, methodToExecute , logEntry);
+                requestBodyObject = Utilities.ToExpandoObject(cReq, metadataUtlity, methodToExecute, logEntry);
                 if (cReq.RelatedEntities != null && cReq.RelatedEntities.Count > 0)
                     requestBodyObject = Utilities.ReleatedEntitiesToExpandoObject(requestBodyObject, cReq.LogicalName, cReq.RelatedEntities, metadataUtlity, methodToExecute, logEntry);
             }
@@ -2055,7 +2060,15 @@ internal async Task<HttpResponseMessage> Command_WebExecuteAsync(string queryStr
 
             if (resp != null)
             {
-                CurrentCookieCollection = Utilities.GetAllCookiesFromHeader(resp.Headers.SingleOrDefault(header => header.Key == "Set-Cookie").Value?.ToArray(), CurrentCookieCollection);
+                CurrentCookieCollection = Utilities.GetAllCookiesFromHeader(resp.Headers.SingleOrDefault(header => header.Key == Utilities.ResponseHeaders.SETCOOKIE).Value?.ToArray(), CurrentCookieCollection);
+                string dregreeofparallelismHint = resp.Headers.SingleOrDefault(header => header.Key == Utilities.ResponseHeaders.RECOMMENDEDDEGREESOFPARALLELISM).Value?.FirstOrDefault();
+                if (!string.IsNullOrEmpty(dregreeofparallelismHint))
+                {
+                    if (int.TryParse(dregreeofparallelismHint, out int idop))
+                    {
+                        RecommendedDegreesOfParallelism = idop;
+                    }
+                }
             }
 
             return resp;

src/GeneralTools/DataverseClient/Client/DataverseTelemetryBehaviors.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections;
 using System.Configuration;
 using System.Linq;
@@ -155,9 +155,23 @@ public void AfterReceiveReply(ref Message reply, object correlationState)
 
                 if (httpResponseMessage != null && httpResponseMessage.Headers.Count > 0)
                 {
-                    var a = httpResponseMessage.Headers["Set-Cookie"];
-                    if (a != null)
-                        _callerCdsConnectionServiceHandler.CurrentCookieCollection = Utilities.GetAllCookiesFromHeader(httpResponseMessage.Headers["Set-Cookie"] , _callerCdsConnectionServiceHandler.CurrentCookieCollection);
+                    string cookieHeader = httpResponseMessage.Headers[Utilities.ResponseHeaders.SETCOOKIE];
+                    if (cookieHeader != null)
+                    {
+                        _callerCdsConnectionServiceHandler.CurrentCookieCollection = Utilities.GetAllCookiesFromHeader(httpResponseMessage.Headers[Utilities.ResponseHeaders.SETCOOKIE] , _callerCdsConnectionServiceHandler.CurrentCookieCollection);
+                    }
+
+                    string dregreeofparallelismHint = httpResponseMessage.Headers[Utilities.ResponseHeaders.RECOMMENDEDDEGREESOFPARALLELISM];
+                    if (!string.IsNullOrEmpty(dregreeofparallelismHint))
+                    {
+                        if(int.TryParse(dregreeofparallelismHint, out int idop))
+                        {
+                            if (_callerCdsConnectionServiceHandler != null)
+                            {
+                                _callerCdsConnectionServiceHandler.RecommendedDegreesOfParallelism = idop;
+                            }
+                        }
+                    }
                 }
             }
             finally { };

src/GeneralTools/DataverseClient/Client/Microsoft.PowerPlatform.Dataverse.Client.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <RootNamespace>Microsoft.PowerPlatform.Dataverse.Client</RootNamespace>
@@ -24,9 +24,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Cds.CoreAssemblies.Internal" Version="$(PackageVersion_CDSServerNuget)">
-      <IncludeAssets></IncludeAssets>
-    </PackageReference>
+    <PackageReference Include="Microsoft.Xrm.Sdk" Version="$(PackageVersion_CdsSdk)" />
+    <PackageReference Include="Microsoft.Cds.Sdk.Proxy.Internal" Version="$(PackageVersion_CdsSdkProxy)" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="$(PackageVersion_Microsoft_Extensions)" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="$(PackageVersion_Microsoft_Extensions)" />

src/GeneralTools/DataverseClient/Client/ServiceClient.cs

@@ -131,7 +131,7 @@ internal OrganizationWebProxyClientAsync OrganizationWebProxyClient
                     if (_connectionSvc.WebClient == null)
                     {
                         if (_logEntry != null)
-                            _logEntry.Log("OrganizationWebProxyClient is null", TraceEventType.Error);
+                            _logEntry.Log("OrganizationWebProxyClientAsync is null", TraceEventType.Error);
                         return null;
                     }
                     else
@@ -140,13 +140,12 @@ internal OrganizationWebProxyClientAsync OrganizationWebProxyClient
                 else
                 {
                     if (_logEntry != null)
-                        _logEntry.Log("OrganizationWebProxyClient is null", TraceEventType.Error);
+                        _logEntry.Log("OrganizationWebProxyClientAsync is null", TraceEventType.Error);
                     return null;
                 }
             }
         }
 
-
         /// <summary>
         /// Enabled Log Capture in memory
         /// This capability enables logs that would normally be sent to your configured
@@ -599,6 +598,11 @@ public bool UseWebApi
             set => _configuration.Value.UseWebApi = value;
         }
 
+        /// <summary>
+        /// Server Hint for the number of concurrent threads that would provbide optimal processing. 
+        /// </summary>
+        public int RecommendedDegreesOfParallelism => _connectionSvc.RecommendedDegreesOfParallelism;
+
         #endregion
 
         #region Constructor and Setup methods

src/GeneralTools/DataverseClient/Client/Utils/Utils.cs

@@ -1,4 +1,4 @@
-#region using
+#region using
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using Microsoft.PowerPlatform.Dataverse.Client.Model;
@@ -503,10 +503,14 @@ internal static ExpandoObject ToExpandoObject(Entity sourceEntity, MetadataUtili
                     if (attributeInfo is LookupAttributeMetadata attribData)
                     {
                         // Now get relationship to make sure we use the correct name.
-                        var eData = mUtil.GetEntityMetadata(EntityFilters.Relationships, sourceEntity.LogicalName);
-                        var ERNavName = eData.ManyToOneRelationships.FirstOrDefault(w => w.ReferencingAttribute.Equals(attribData.LogicalName) &&
+                        EntityMetadata eData = mUtil.GetEntityMetadata(EntityFilters.Relationships, sourceEntity.LogicalName);
+                        string ERNavName = eData.ManyToOneRelationships.FirstOrDefault(w => w.ReferencingAttribute.Equals(attribData.LogicalName) &&
                                                                                 w.ReferencedEntity.Equals(entityReference.LogicalName))
                                                                                 ?.ReferencingEntityNavigationPropertyName;
+                        if (string.IsNullOrEmpty(ERNavName ))
+                        {
+                            ERNavName = eData.ManyToOneRelationships.FirstOrDefault(w => w.ReferencingAttribute.Equals(attribData.LogicalName))?.ReferencingEntityNavigationPropertyName;
+                        }
 
                         if (!string.IsNullOrEmpty(ERNavName))
                         {
@@ -660,10 +664,9 @@ internal static ExpandoObject ToExpandoObject(Entity sourceEntity, MetadataUtili
             {
                 var sourceMdata = mUtil.GetEntityMetadata(sourceEntity.LogicalName);
                 if (sourceMdata != null )
-                    expandoObject.Add($"{sourceMdata.SchemaName}_activity_parties", partiesCollection);
+                    expandoObject.Add($"{sourceMdata.LogicalName}_activity_parties", partiesCollection);
             }
 
-
             return (ExpandoObject)expandoObject;
         }
 
@@ -743,33 +746,7 @@ internal static ExpandoObject ReleatedEntitiesToExpandoObject(ExpandoObject root
                 // Get the Entity relationship key and entity and reverse it back to the entity key name
                 var eData = mUtil.GetEntityMetadata(EntityFilters.Relationships, entItem.Value.Entities[0].LogicalName);
 
-                // Find the relationship that is referenced.
-                var ERM21 = eData.ManyToOneRelationships.FirstOrDefault(w1 => w1.SchemaName.ToLower().Equals(entItem.Key.SchemaName.ToLower()));
-                var ERM2M = eData.ManyToManyRelationships.FirstOrDefault(w2 => w2.SchemaName.ToLower().Equals(entItem.Key.SchemaName.ToLower()));
-                var ER12M = eData.OneToManyRelationships.FirstOrDefault(w3 => w3.SchemaName.ToLower().Equals(entItem.Key.SchemaName.ToLower()));
-
-                // Determine which one hit
-                if (ERM21 != null)
-                {
-                    isArrayRequired = true;
-                    key = ERM21.ReferencedEntityNavigationPropertyName;
-                }
-                else if (ERM2M != null)
-                {
-                    isArrayRequired = true;
-                    if (ERM2M.Entity1LogicalName.ToLower().Equals(entityName))
-                    {
-                        key = ERM2M.Entity1NavigationPropertyName;
-                    }
-                    else
-                    {
-                        key = ERM2M.Entity2NavigationPropertyName;
-                    }
-                }
-                else if (ER12M != null)
-                {
-                    key = ER12M.ReferencingAttribute;
-                }
+                key = ExtractKeyNameFromRelationship(entItem.Key.SchemaName.ToLower(), entityName, ref isArrayRequired, eData);
 
                 if (string.IsNullOrEmpty(key)) // Failed to find key
                 {
@@ -785,7 +762,7 @@ internal static ExpandoObject ReleatedEntitiesToExpandoObject(ExpandoObject root
                     }
 
                     // generate object.
-                    ExpandoObject ent1 = ToExpandoObject(ent, mUtil, requestedMethod , logger);
+                    ExpandoObject ent1 = ToExpandoObject(ent, mUtil, requestedMethod, logger);
 
                     if (((IDictionary<string, object>)childEntities).Count() > 0)
                     {
@@ -804,6 +781,49 @@ internal static ExpandoObject ReleatedEntitiesToExpandoObject(ExpandoObject root
             return rootExpando;
         }
 
+
+        /// <summary>
+        /// Helper to extract key name from one of the relationships.
+        /// </summary>
+        /// <param name="schemaName"></param>
+        /// <param name="entityName"></param>
+        /// <param name="isArrayRequired"></param>
+        /// <param name="eData"></param>
+        /// <returns></returns>
+        private static string ExtractKeyNameFromRelationship(string schemaName, string entityName,  ref bool isArrayRequired, EntityMetadata eData)
+        {
+            string key = ""; 
+            // Find the relationship that is referenced.
+            OneToManyRelationshipMetadata ERM21 = eData.ManyToOneRelationships.FirstOrDefault(w1 => w1.SchemaName.ToLower().Equals(schemaName.ToLower()));
+            ManyToManyRelationshipMetadata ERM2M = eData.ManyToManyRelationships.FirstOrDefault(w2 => w2.SchemaName.ToLower().Equals(schemaName.ToLower()));
+            OneToManyRelationshipMetadata ER12M = eData.OneToManyRelationships.FirstOrDefault(w3 => w3.SchemaName.ToLower().Equals(schemaName.ToLower()));
+
+            // Determine which one hit
+            if (ERM21 != null)
+            {
+                isArrayRequired = true;
+                key = ERM21.ReferencedEntityNavigationPropertyName;
+            }
+            else if (ERM2M != null)
+            {
+                isArrayRequired = true;
+                if (ERM2M.Entity1LogicalName.ToLower().Equals(entityName))
+                {
+                    key = ERM2M.Entity1NavigationPropertyName;
+                }
+                else
+                {
+                    key = ERM2M.Entity2NavigationPropertyName;
+                }
+            }
+            else if (ER12M != null)
+            {
+                key = ER12M.ReferencingAttribute;
+            }
+
+            return key; 
+        }
+
         /// <summary>
         /// Parses Key attribute collection for alt key support.
         /// </summary>
@@ -950,6 +970,19 @@ internal static class RequestHeaders
 
         }
 
+        internal static class ResponseHeaders
+        {
+            /// <summary>
+            /// Recomended number of client connection threads Hint 
+            /// </summary>
+            public const string RECOMMENDEDDEGREESOFPARALLELISM = "x-ms-dop-hint";
+
+            /// <summary>
+            /// header for Cookie's
+            /// </summary>
+            public const string SETCOOKIE = "Set-Cookie";
+        }
+
         /// <summary>
         /// Minim Version numbers for various features of Dataverse API's.
         /// </summary>

src/GeneralTools/DataverseClient/DataverseClient_PackageTestSolution.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.31729.503
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LivePackageTestsConsole", "UnitTests\LivePackageTestsConsole\LivePackageTestsConsole.csproj", "{AD21CFD4-EDA7-4F31-9A07-CC90C03B4C27}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{AD21CFD4-EDA7-4F31-9A07-CC90C03B4C27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD21CFD4-EDA7-4F31-9A07-CC90C03B4C27}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD21CFD4-EDA7-4F31-9A07-CC90C03B4C27}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD21CFD4-EDA7-4F31-9A07-CC90C03B4C27}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {BDA12603-BF5F-4103-81BE-24703407A609}
+	EndGlobalSection
+EndGlobal

src/GeneralTools/DataverseClient/Extensions/DynamicsExtension/Microsoft.PowerPlatform.Dataverse.Client.Dynamics.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <RootNamespace>Microsoft.PowerPlatform.Dataverse.Client.Dynamics</RootNamespace>
@@ -13,12 +13,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Cds.CoreAssemblies.Internal" Version="$(PackageVersion_CDSServerNuget)">
-      <IncludeAssets></IncludeAssets>
-    </PackageReference>
-    <PackageReference Include="Microsoft.Dynamics.Sdk.Messages.Internal" Version="$(PackageVersion_CDSServerNuget)">
-      <IncludeAssets></IncludeAssets>
-    </PackageReference>
+    <PackageReference Include="Microsoft.Xrm.Sdk" Version="$(PackageVersion_CdsSdk)" />
+    <PackageReference Include="Microsoft.Dynamics.Sdk.Messages.Internal" Version="$(PackageVersion_CdsSdkProxy)" />
   </ItemGroup>
 
   <ItemGroup>

src/GeneralTools/DataverseClient/Extensions/Microsoft.Dynamics.Sdk.Messages/Microsoft.Dynamics.Sdk.Messages.Shell.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RootNamespace>Microsoft.Dynamics.Sdk.Messages</RootNamespace>
     <ComponentAreaName>DataverseClient</ComponentAreaName>
@@ -8,7 +8,8 @@
   <Import Project="..\..\..\..\Build.Common.core.props" />
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Dynamics.Sdk.Messages.Internal" Version="$(PackageVersion_CDSServerNuget)" />
+    <PackageReference Include="Microsoft.Xrm.Sdk" Version="$(PackageVersion_CdsSdk)" />
+    <PackageReference Include="Microsoft.Dynamics.Sdk.Messages.Internal" Version="$(PackageVersion_CdsSdkProxy)" />
   </ItemGroup>
 
 </Project>