Correct Search Behavior

When Purview has a limit set to 1000, search scores are set to 1 and resulting in sub-optimal
search result ordering. The custom connector generic is showing up first when other assets
like sql tables and resource sets are showing up second.

This is solved by changing the default limit to 100 instead of 1000 (or any value less than 1000)
and Microsoft Purview search scores are populating correctly.

function-app/adb-to-purview/src/Function.Domain/Constants/Constants.cs

@@ -16,7 +16,9 @@ public struct AuthenticationConstants
 public struct PurviewAPIConstants
 { 
     //Purview API Constants
-    public const string DefaultSearchLimit = "1000";
+    // Setting to 1000 (the max) will force the search scores to be 1 and thus suboptimal search results
+    // Reducing from 1000 to 100 will enable better search results
+    public const string DefaultSearchLimit = "100";
     public const string DefaultOffset = "100";
 }
 

function-app/adb-to-purview/src/Function.Domain/Helpers/PurviewClientHelper.cs

@@ -170,7 +170,9 @@ public async Task<List<QueryValeuModel>> QueryEntities(string correlationId, str
             PurviewQueryResponseModel entityObjectModel = new PurviewQueryResponseModel();
             List<QueryValeuModel> entities = new List<QueryValeuModel>();
             int offset = 0;
-            int totalEntities = 1000;
+            // Setting to 1000 (the max) will force the search scores to be 1 and thus suboptimal search results
+            // Reducing from 1000 to 100 will enable better search results
+            int totalEntities = 100;
             int numberEntities = 1;
             bool printNumberEntitiesOnSearch = true;
             try

function-app/adb-to-purview/src/Function.Domain/Helpers/PurviewCustomType.cs

@@ -367,6 +367,7 @@ private async Task<List<QueryValeuModel>> SelectReturnEntity(List<QueryValeuMode
             bool resourceSetHasBeenSeen = false;
             foreach (QueryValeuModel entity in results)
             {
+                _logger.LogDebug($"Working on {entity.entityType} with score {entity.SearchScore}");
                 if (IsSpark_Entity(entity.entityType))
                     if (results[0].qualifiedName.ToLower().Trim('/') != this.properties!["attributes"]!["qualifiedName"]!.ToString().ToLower().Trim('/'))
                     {