Fix to support paging of query rule iteration

documentation/Get-PnPSearchConfiguration.md

@@ -6,96 +6,108 @@ applicable: SharePoint Online
 external help file: PnP.PowerShell.dll-Help.xml
 online version: https://pnp.github.io/powershell/cmdlets/Get-PnPSearchConfiguration.html
 ---
- 
+
 # Get-PnPSearchConfiguration
 
 ## SYNOPSIS
+
 Returns the search configuration
 
 ## SYNTAX
 
 ### Xml (Default)
+
 ```powershell
-Get-PnPSearchConfiguration [-Scope <SearchConfigurationScope>] [-Path <String>] 
+Get-PnPSearchConfiguration [-Scope <SearchConfigurationScope>] [-Path <String>]
  [-Connection <PnPConnection>] [<CommonParameters>]
 ```
 
 ### OutputFormat
+
 ```powershell
 Get-PnPSearchConfiguration [-Scope <SearchConfigurationScope>] [-OutputFormat <OutputFormat>]
  [-Connection <PnPConnection>] [<CommonParameters>]
 ```
 
 ### BookmarksCSV
+
 ```powershell
-Get-PnPSearchConfiguration [-Scope <SearchConfigurationScope>] [-PromotedResultsToBookmarkCSV] [-BookmarkStatus <BookmarkStatus>] [-Path <String>]
+Get-PnPSearchConfiguration [-Scope <SearchConfigurationScope>] [-PromotedResultsToBookmarkCSV] [-ExcludeVisualPromotedResults <Boolean>] [-BookmarkStatus <BookmarkStatus>] [-Path <String>]
  [-Connection <PnPConnection>] [<CommonParameters>]
 ```
 
-
 ## DESCRIPTION
 
 ## EXAMPLES
 
 ### EXAMPLE 1
+
 ```powershell
 Get-PnPSearchConfiguration
 ```
 
-Returns the search configuration for the current web
+Returns the search configuration for the current web.
 
 ### EXAMPLE 2
+
 ```powershell
 Get-PnPSearchConfiguration -Scope Site
 ```
 
-Returns the search configuration for the current site collection
+Returns the search configuration for the current site collection.
 
 ### EXAMPLE 3
+
 ```powershell
 Get-PnPSearchConfiguration -Scope Subscription
 ```
 
-Returns the search configuration for the current tenant
+Returns the search configuration for the current tenant.
 
 ### EXAMPLE 4
+
 ```powershell
 Get-PnPSearchConfiguration -Path searchconfig.xml -Scope Subscription
 ```
 
-Returns the search configuration for the current tenant and saves it to the specified file
+Returns the search configuration for the current tenant and saves it to the specified file.
 
 ### EXAMPLE 5
+
 ```powershell
 Get-PnPSearchConfiguration -Scope Site -OutputFormat ManagedPropertyMappings
 ```
 
-Returns all custom managed properties and crawled property mapping at the current site collection
+Returns all custom managed properties and crawled property mapping at the current site collection.
 
 ### EXAMPLE 6
+
 ```powershell
 Get-PnPSearchConfiguration -Scope Site -PromotedResultsToBookmarkCSV -Path bookmarks.csv
 ```
 
-Export promoted results from query rules on the site collection as a CSV file with the bookmarks in suggested status
+Export promoted results excluding visual ones from query rules on the site collection as a CSV file with the bookmarks in suggested status.
 
 ### EXAMPLE 7
+
 ```powershell
 Get-PnPSearchConfiguration -Scope Site -PromotedResultsToBookmarkCSV -Path bookmarks.csv -BookmarkStatus Published
 ```
 
-Export promoted results from query rules on the site collection as a CSV file with the bookmarks in published status
+Export promoted results excluding visual from query rules on the site collection as a CSV file with the bookmarks in published status.
 
 ### EXAMPLE 8
+
 ```powershell
-Get-PnPSearchConfiguration -Scope Subscription -PromotedResultsToBookmarkCSV
+Get-PnPSearchConfiguration -Scope Subscription -PromotedResultsToBookmarkCSV -ExcludeVisualPromotedResults $false
 ```
 
-Export promoted results from query rules on the tenant in CSV format with the bookmarks in suggested status.
+Export promoted results including visual ones from query rules on the tenant in CSV format with the bookmarks in suggested status.
 
 ## PARAMETERS
 
 ### -Connection
+
 Optional connection to be used by the cmdlet. Retrieve the value for this parameter by either specifying -ReturnConnection on Connect-PnPOnline or by executing Get-PnPConnection.
 
 ```yaml
@@ -110,6 +122,7 @@ Accept wildcard characters: False
 ```
 
 ### -OutputFormat
+
 Output format for of the configuration. Defaults to complete XML
 
 ```yaml
@@ -125,6 +138,7 @@ Accept wildcard characters: False
 ```
 
 ### -Path
+
 Local path where the search configuration will be saved
 
 ```yaml
@@ -139,6 +153,7 @@ Accept wildcard characters: False
 ```
 
 ### -Scope
+
 Scope to use. Either Web, Site, or Subscription. Defaults to Web
 
 ```yaml
@@ -154,13 +169,14 @@ Accept wildcard characters: False
 ```
 
 ### -PromotedResultsToBookmarkCSV
+
 Output promoted results to a compatible CSV file to be used as Bookmark import at https://admin.microsoft.com/#/MicrosoftSearch/bookmarks.
 
 Export details:
 
-* Promoted results marked as "Render the URL as a banner instead of as a hyperlink" and query rules with no triggers will be skipped.
-* Triggers set to "Advanced Query Text Match" and "Query Contains Action Term" will have "Match Similar Keywords" set to true for the Bookmark.
-* Multiple triggers on a query rule will be merged into one.
+- Promoted results marked as "Render the URL as a banner instead of as a hyperlink" (visual promoted results) and query rules with no triggers will be skipped by default.
+- Triggers set to "Advanced Query Text Match" and "Query Contains Action Term" will have "Match Similar Keywords" set to true for the Bookmark.
+- Multiple triggers on a query rule will be merged into a single trigger.
 
 ```yaml
 Type: SwitchParameter
@@ -173,6 +189,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -ExcludeVisualPromotedResults
+
+Exclude promoted results marked as "Render the URL as a banner instead of as a hyperlink". Defaults to true.
+
+```yaml
+Type: Boolean
+Parameter Sets: CSV
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -BookmarkStatus
 Output bookmarks to be in suggested or published status upon CSV import. Defaults to suggested status.
 
@@ -191,4 +222,3 @@ Accept wildcard characters: False
 ## RELATED LINKS
 
 [Microsoft 365 Patterns and Practices](https://aka.ms/m365pnp)
-

src/Commands/Search/GetSearchConfiguration.cs

@@ -50,64 +50,80 @@ public class GetSearchConfiguration : PnPWebCmdlet
         [Parameter(Mandatory = false, ParameterSetName = "CSV")]
         public BookmarkStatus BookmarkStatus = BookmarkStatus.Suggested;
 
+        [Parameter(Mandatory = false, ParameterSetName = "CSV")]
+        public bool ExcludeVisualPromotedResults = true;
+
         protected override void ExecuteCmdlet()
         {
             string output = string.Empty;
 
-            switch (Scope)
+            if (!PromotedResultsToBookmarkCSV.IsPresent)
             {
-                case SearchConfigurationScope.Web:
-                    {
-                        if (PromotedResultsToBookmarkCSV.IsPresent)
-                        {
-                            output = RestHelper.ExecuteGetRequest(ClientContext, "searchsetting/getpromotedresultqueryrules");
-                        }
-                        else
+                switch (Scope)
+                {
+                    case SearchConfigurationScope.Web:
                         {
+
                             output = CurrentWeb.GetSearchConfiguration();
+
+                            break;
                         }
-                        break;
-                    }
-                case SearchConfigurationScope.Site:
-                    {
-                        if (PromotedResultsToBookmarkCSV.IsPresent)
-                        {
-                            output = RestHelper.ExecuteGetRequest(ClientContext, "searchsetting/getpromotedresultqueryrules?sitecollectionlevel=true");
-                        }
-                        else
+                    case SearchConfigurationScope.Site:
                         {
+
                             output = ClientContext.Site.GetSearchConfiguration();
-                        }
-                        break;
-                    }
-                case SearchConfigurationScope.Subscription:
-                    {
-                        if (!ClientContext.Url.ToLower().Contains("-admin"))
-                        {
-                            throw new InvalidOperationException(Resources.CurrentSiteIsNoTenantAdminSite);
-                        }
 
-                        if (PromotedResultsToBookmarkCSV.IsPresent)
-                        {
-                            output = RestHelper.ExecuteGetRequest(ClientContext, "searchsetting/getpromotedresultqueryrules");
+                            break;
                         }
-                        else
+                    case SearchConfigurationScope.Subscription:
                         {
+                            if (!ClientContext.Url.ToLower().Contains("-admin"))
+                            {
+                                throw new InvalidOperationException(Resources.CurrentSiteIsNoTenantAdminSite);
+                            }
+
                             SearchObjectOwner owningScope = new SearchObjectOwner(ClientContext, SearchObjectLevel.SPSiteSubscription);
                             var config = new SearchConfigurationPortability(ClientContext);
                             ClientResult<string> configuration = config.ExportSearchConfiguration(owningScope);
                             ClientContext.ExecuteQueryRetry();
                             output = configuration.Value;
                         }
-                    }
-                    break;
+                        break;
+                }
             }
-
-            if (PromotedResultsToBookmarkCSV.IsPresent)
+            else
             {
-                output = ConvertToCSV(output);
-            }
+                string promotedResultsBaseUrl = "searchsetting/getpromotedresultqueryrules?";
+                if (Scope == SearchConfigurationScope.Site)
+                {
+                    promotedResultsBaseUrl += "sitecollectionlevel=true&";
+                }
 
+                int offset = 0;
+                const int numberOfRules = 50;
+                bool hasData;
+                List<string> queryRuleResponses = new List<string>();
+                do
+                {
+                    string runUrl = string.Format("{0}offset={1}&numberOfRules={2}", promotedResultsBaseUrl, offset, numberOfRules);
+                    string response = RestHelper.ExecuteGetRequest(ClientContext, runUrl);
+                    offset += numberOfRules;
+                    var config = JsonSerializer.Deserialize<Root>(response);
+                    hasData = config.Result != null && config.Result.Count > 0;
+                    if(hasData) queryRuleResponses.Add(response);
+                } while (hasData);
+
+                List<ExpandoObject> bookmarks = new List<ExpandoObject>(200);
+                foreach (var response in queryRuleResponses)
+                {
+                    var result = PromotedResultsToBookmarks(response);
+                    if (result != null && result.Count > 0)
+                    {
+                        bookmarks.AddRange(result);
+                    }
+                }
+                output = BookmarksToString(bookmarks);
+            }
 
             if (Path != null)
             {
@@ -144,28 +160,39 @@ protected override void ExecuteCmdlet()
             }
         }
 
-        private string ConvertToCSV(string json)
+        private List<ExpandoObject> PromotedResultsToBookmarks(string json)
         {
             var bookmarks = new List<ExpandoObject>();
             var config = JsonSerializer.Deserialize<Root>(json);
             if (config.Result != null)
             {
                 foreach (var rule in config.Result)
                 {
-                    //if (!rule.IsPromotedResultsOnly) continue;
-                    if (rule.QueryConditions == null || rule.PromotedResults == null) continue;
+                    if (rule.QueryConditions == null || rule.PromotedResults == null)
+                    {
+                        continue;
+                    }
                     foreach (var promoResult in rule.PromotedResults)
                     {
-                        if (promoResult.IsVisual) continue;
                         dynamic bookmark = new ExpandoObject();
                         bookmark.Title = promoResult.Title.Contains(" ") ? '"' + promoResult.Title + '"' : promoResult.Title;
                         bookmark.Url = promoResult.Url;
+
+                        if (promoResult.IsVisual && ExcludeVisualPromotedResults)
+                        {
+                            WriteWarning($"Skipping visual promoted result {bookmark.Title} ({bookmark.Url})");
+                            continue;
+                        }
+
                         List<string> triggerTerms = new List<string>();
                         bool matchSimilar = false;
                         foreach (var condition in rule.QueryConditions)
                         {
-                            if (condition.Terms == null) continue;
-                            if (condition.QueryConditionType != "Keyword") continue;
+                            if (condition.Terms == null || condition.QueryConditionType != "Keyword")
+                            {
+                                WriteWarning($"Skipping {bookmark.Title} due to no trigger conditions");
+                                continue;
+                            }
 
                             if (condition.MatchingOptions.Contains("ProperPrefix") || condition.MatchingOptions.Contains("ProperSuffix"))
                             {
@@ -177,7 +204,11 @@ private string ConvertToCSV(string json)
                                 triggerTerms.AddRange(term.Split(';').Select(s => s.Replace("Keywords:", "").Trim()).ToList());
                             }
                         }
-                        if (triggerTerms.Count == 0) continue;
+                        if (triggerTerms.Count == 0)
+                        {
+                            WriteWarning($"Skipping {bookmark.Title} due to no trigger terms");
+                            continue;
+                        }
 
                         var dict = bookmark as IDictionary<string, object>;
 
@@ -201,7 +232,11 @@ private string ConvertToCSV(string json)
                     }
                 }
             }
+            return bookmarks;
+        }
 
+        private static string BookmarksToString(List<ExpandoObject> bookmarks)
+        {
             StringBuilder sb = new StringBuilder();
             bool firstLine = true;
             foreach (var bookmark in bookmarks)