From 97e9ea4dcf40c1b0a3a671d89e8ece5bc8fb43df Mon Sep 17 00:00:00 2001 From: Gautam Sheth Date: Sun, 5 Mar 2023 15:36:06 +0200 Subject: [PATCH 1/3] Feature/Fix #2485 - improved recycle bin item handling in Restore/Clear cmdlets --- CHANGELOG.md | 3 +- documentation/Add-PnPFile.md | 2 +- documentation/Add-PnPListItem.md | 2 +- documentation/Add-PnPSiteTemplate.md | 2 +- documentation/Clear-PnpRecycleBinItem.md | 2 +- .../RecycleBin/ClearRecycleBinItem.cs | 10 +--- .../RecycleBin/RestoreRecycleBinItem.cs | 9 +-- src/Commands/Utilities/RecycleBinUtility.cs | 56 +++++++++++++++++-- 8 files changed, 63 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77004bf88..b81fd444f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Properties of `Get-PnPAzureADServicePrincipal` are now all typed instead of some of them returning unparsed JSON fragments. [#2717](https://github.com/pnp/powershell/pull/2717) - Changed `Add-PnPTeamsChannel` to no longer require an `-OwnerUPN` to be provided when specifying `-ChannelType Standard` [#2786](https://github.com/pnp/powershell/pull/2786) - Changed `Add-PnPFile` by default to upload a file as a draft with a minor version now instead of publishing it as a major version. `-CheckinType MajorCheckIn` can be used to still upload the file as a major published version [#2806](https://github.com/pnp/powershell/pull/2806) +- Improved `Restore-PnPRecycleBinItem` and `Clear-PnPRecycleBinItem` cmdlets to better work with large items in recycle bin. ### Removed @@ -87,7 +88,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Contributors -- Markus Hanisch[m-hanisch] +- Markus Hanisch [m-hanisch] - Kasper Larsen [kasperbolarsen] - Arnaud Rompen [rompenar] - [reusto] diff --git a/documentation/Add-PnPFile.md b/documentation/Add-PnPFile.md index c6af74948..a4f8031ae 100644 --- a/documentation/Add-PnPFile.md +++ b/documentation/Add-PnPFile.md @@ -360,7 +360,7 @@ Managed Metadata (multiple values with paths to terms): -Values @{"MetadataField Managed Metadata (multiple values with ids of terms): -Values @{"MetadataField" = "fe40a95b-2144-4fa2-b82a-0b3d0299d818","52d88107-c2a8-4bf0-adfa-04bc2305b593"} -Hyperlink or Picture: -Values @{"Hyperlink" = "https://github.com/OfficeDev/, OfficePnp"} +Hyperlink or Picture: -Values @{"Hyperlink" = "https://github.com/OfficeDev/, OfficePnP"} ```yaml Type: Hashtable diff --git a/documentation/Add-PnPListItem.md b/documentation/Add-PnPListItem.md index 979b9d252..49fbcd3d9 100644 --- a/documentation/Add-PnPListItem.md +++ b/documentation/Add-PnPListItem.md @@ -225,7 +225,7 @@ Managed Metadata (multiple values with paths to terms): -Values @{"MetadataField Managed Metadata (multiple values with ids of terms): -Values @{"MetadataField" = "fe40a95b-2144-4fa2-b82a-0b3d0299d818","52d88107-c2a8-4bf0-adfa-04bc2305b593"} -Hyperlink or Picture: -Values @{"Hyperlink" = "https://github.com/OfficeDev/, OfficePnp"} +Hyperlink or Picture: -Values @{"Hyperlink" = "https://github.com/OfficeDev/, OfficePnP"} ```yaml Type: Hashtable diff --git a/documentation/Add-PnPSiteTemplate.md b/documentation/Add-PnPSiteTemplate.md index aecff9661..6712e9316 100644 --- a/documentation/Add-PnPSiteTemplate.md +++ b/documentation/Add-PnPSiteTemplate.md @@ -26,7 +26,7 @@ Allows to add PnP Site Template object to a tenant template. ### EXAMPLE 1 ```powershell -Add-PnpSiteTemplate -TenantTemplate $tenanttemplate -SiteTemplate $sitetemplate +Add-PnPSiteTemplate -TenantTemplate $tenanttemplate -SiteTemplate $sitetemplate ``` Adds an existing site template to an existing tenant template object diff --git a/documentation/Clear-PnpRecycleBinItem.md b/documentation/Clear-PnpRecycleBinItem.md index 771ba78d0..f7fc22a13 100644 --- a/documentation/Clear-PnpRecycleBinItem.md +++ b/documentation/Clear-PnpRecycleBinItem.md @@ -34,7 +34,7 @@ Allows to permanently delete items from recycle bin. By default the command will ### EXAMPLE 1 ```powershell -Get-PnPRecycleBinItem | Where-Object LeafName -like "*.docx" | Clear-PnpRecycleBinItem +Get-PnPRecycleBinItem | Where-Object LeafName -like "*.docx" | Clear-PnPRecycleBinItem ``` Permanently deletes all the items in the first and second stage recycle bins of which the file names have the .docx extension diff --git a/src/Commands/RecycleBin/ClearRecycleBinItem.cs b/src/Commands/RecycleBin/ClearRecycleBinItem.cs index d328ded10..82cc5ca81 100644 --- a/src/Commands/RecycleBin/ClearRecycleBinItem.cs +++ b/src/Commands/RecycleBin/ClearRecycleBinItem.cs @@ -2,6 +2,7 @@ using Microsoft.SharePoint.Client; using PnP.PowerShell.Commands.Base.PipeBinds; +using PnP.PowerShell.Commands.Utilities; using Resources = PnP.PowerShell.Commands.Properties.Resources; namespace PnP.PowerShell.Commands.RecycleBin @@ -45,15 +46,10 @@ protected override void ExecuteCmdlet() if (ParameterSpecified(nameof(RowLimit))) { if (Force || ShouldContinue(SecondStageOnly ? Resources.ClearSecondStageRecycleBin : Resources.ClearBothRecycleBins, Resources.Confirm)) - { + { RecycleBinItemState recycleBinStage = SecondStageOnly ? RecycleBinItemState.SecondStageRecycleBin : RecycleBinItemState.None; - RecycleBinItemCollection items = ClientContext.Site.GetRecycleBinItems(null, RowLimit, false, RecycleBinOrderBy.DeletedDate, recycleBinStage); - ClientContext.Load(items); - ClientContext.ExecuteQueryRetry(); - - items.DeleteAll(); - ClientContext.ExecuteQueryRetry(); + RecycleBinUtility.RestoreOrClearRecycleBinItems(ClientContext, RowLimit, recycleBinStage, false); } } else diff --git a/src/Commands/RecycleBin/RestoreRecycleBinItem.cs b/src/Commands/RecycleBin/RestoreRecycleBinItem.cs index ec1d1971d..44cebed79 100644 --- a/src/Commands/RecycleBin/RestoreRecycleBinItem.cs +++ b/src/Commands/RecycleBin/RestoreRecycleBinItem.cs @@ -3,6 +3,7 @@ using Microsoft.SharePoint.Client; using PnP.PowerShell.Commands.Base.PipeBinds; +using PnP.PowerShell.Commands.Utilities; using Resources = PnP.PowerShell.Commands.Properties.Resources; namespace PnP.PowerShell.Commands.RecycleBin @@ -24,7 +25,6 @@ protected override void ExecuteCmdlet() { if (ParameterSpecified(nameof(Identity))) { - var recycleBinItem = Identity.GetRecycleBinItem(ClientContext.Site); if (Force || ShouldContinue(string.Format(Resources.RestoreRecycleBinItem, recycleBinItem.LeafName), Resources.Confirm)) @@ -39,12 +39,7 @@ protected override void ExecuteCmdlet() { if (Force || ShouldContinue(string.Format(Resources.Restore0RecycleBinItems, RowLimit), Resources.Confirm)) { - RecycleBinItemCollection items = ClientContext.Site.GetRecycleBinItems(null, RowLimit, false, RecycleBinOrderBy.DeletedDate, RecycleBinItemState.None); - ClientContext.Load(items); - ClientContext.ExecuteQueryRetry(); - - items.RestoreAll(); - ClientContext.ExecuteQueryRetry(); + RecycleBinUtility.RestoreOrClearRecycleBinItems(ClientContext, RowLimit, RecycleBinItemState.None, true); } } else diff --git a/src/Commands/Utilities/RecycleBinUtility.cs b/src/Commands/Utilities/RecycleBinUtility.cs index 0b5587ef0..510f51466 100644 --- a/src/Commands/Utilities/RecycleBinUtility.cs +++ b/src/Commands/Utilities/RecycleBinUtility.cs @@ -1,9 +1,7 @@ using Microsoft.SharePoint.Client; -using System; using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text; namespace PnP.PowerShell.Commands.Utilities { @@ -39,11 +37,11 @@ internal static List GetRecycleBinItems(ClientContext ctx, int? { iterationRowLimit = 5000; } - + items = ctx.Site.GetRecycleBinItems(pagingInfo, iterationRowLimit, false, RecycleBinOrderBy.DefaultOrderBy, recycleBinStage); ctx.Load(items); ctx.ExecuteQueryRetry(); - recycleBinItems.AddRange(items.ToList()); + recycleBinItems.AddRange(items.ToList()); // Paging magic (if needed) // Based on this work our good friends at Portiva did ❤ @@ -61,5 +59,55 @@ internal static List GetRecycleBinItems(ClientContext ctx, int? return recycleBinItems; } + + internal static void RestoreOrClearRecycleBinItems(ClientContext ctx, int? rowLimit = null, RecycleBinItemState recycleBinItemState = RecycleBinItemState.None, bool restore = true) + { + string pagingInfo = null; + RecycleBinItemCollection items; + + do + { + // We don't actually know what the List View Threshold for the Recycle Bin is, so we'll use the safe number (5000) and implement paging. + int iterationRowLimit; + if (rowLimit.HasValue && rowLimit.Value >= 5000) + { + // Subtract this page's count from the rowLimit (we don't want duplicates or go out of bounds) + if (rowLimit.HasValue) rowLimit -= 5000; + + iterationRowLimit = 5000; + } + else if (rowLimit.HasValue && rowLimit.Value > 0 && rowLimit.Value < 5000) + { + iterationRowLimit = rowLimit.Value; + } + else + { + iterationRowLimit = 5000; + } + + items = ctx.Site.GetRecycleBinItems(pagingInfo, iterationRowLimit, false, RecycleBinOrderBy.DefaultOrderBy, recycleBinItemState); + ctx.Load(items); + ctx.ExecuteQueryRetry(); + + if (items.Count > 0) + { + var nextId = items.Last().Id; + var nextTitle = WebUtility.UrlEncode(items.Last().Title); + pagingInfo = $"id={nextId}&title={nextTitle}"; + + if (restore) + { + items.RestoreAll(); + ctx.ExecuteQueryRetry(); + } + else + { + items.DeleteAll(); + ctx.ExecuteQueryRetry(); + } + } + } + while (items?.Count == 5000); + } } } From 0ef3b546fe068c901fa2ab0da572b2be9d6f9883 Mon Sep 17 00:00:00 2001 From: Koen Zomers Date: Mon, 6 Mar 2023 16:40:22 +0100 Subject: [PATCH 2/3] Update CHANGELOG.md Added PR reference --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b81fd444f..af09ae13c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Properties of `Get-PnPAzureADServicePrincipal` are now all typed instead of some of them returning unparsed JSON fragments. [#2717](https://github.com/pnp/powershell/pull/2717) - Changed `Add-PnPTeamsChannel` to no longer require an `-OwnerUPN` to be provided when specifying `-ChannelType Standard` [#2786](https://github.com/pnp/powershell/pull/2786) - Changed `Add-PnPFile` by default to upload a file as a draft with a minor version now instead of publishing it as a major version. `-CheckinType MajorCheckIn` can be used to still upload the file as a major published version [#2806](https://github.com/pnp/powershell/pull/2806) -- Improved `Restore-PnPRecycleBinItem` and `Clear-PnPRecycleBinItem` cmdlets to better work with large items in recycle bin. +- Improved `Restore-PnPRecycleBinItem` and `Clear-PnPRecycleBinItem` cmdlets to better work with large items in recycle bin. [#2866] (https://github.com/pnp/powershell/pull/2866) ### Removed From 831a378c23686bfd0a3c21eb6eb10629ce3ca029 Mon Sep 17 00:00:00 2001 From: Gautam Sheth Date: Thu, 9 Mar 2023 14:14:34 +0200 Subject: [PATCH 3/3] Improved implementation --- src/Commands/RecycleBin/ClearRecycleBinItem.cs | 8 +++++++- .../RecycleBin/RestoreRecycleBinItem.cs | 8 +++++++- src/Commands/Utilities/RecycleBinUtility.cs | 17 +++++------------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Commands/RecycleBin/ClearRecycleBinItem.cs b/src/Commands/RecycleBin/ClearRecycleBinItem.cs index 82cc5ca81..d9d3171ea 100644 --- a/src/Commands/RecycleBin/ClearRecycleBinItem.cs +++ b/src/Commands/RecycleBin/ClearRecycleBinItem.cs @@ -49,7 +49,13 @@ protected override void ExecuteCmdlet() { RecycleBinItemState recycleBinStage = SecondStageOnly ? RecycleBinItemState.SecondStageRecycleBin : RecycleBinItemState.None; - RecycleBinUtility.RestoreOrClearRecycleBinItems(ClientContext, RowLimit, recycleBinStage, false); + var recycleBinItemCollection = RecycleBinUtility.GetRecycleBinItemCollection(ClientContext, RowLimit, recycleBinStage); + for (var i = 0; i < recycleBinItemCollection.Count; i++) + { + var recycleBinItems = recycleBinItemCollection[i]; + recycleBinItems.DeleteAll(); + ClientContext.ExecuteQueryRetry(); + } } } else diff --git a/src/Commands/RecycleBin/RestoreRecycleBinItem.cs b/src/Commands/RecycleBin/RestoreRecycleBinItem.cs index 44cebed79..e51637575 100644 --- a/src/Commands/RecycleBin/RestoreRecycleBinItem.cs +++ b/src/Commands/RecycleBin/RestoreRecycleBinItem.cs @@ -39,7 +39,13 @@ protected override void ExecuteCmdlet() { if (Force || ShouldContinue(string.Format(Resources.Restore0RecycleBinItems, RowLimit), Resources.Confirm)) { - RecycleBinUtility.RestoreOrClearRecycleBinItems(ClientContext, RowLimit, RecycleBinItemState.None, true); + var recycleBinItemCollection = RecycleBinUtility.GetRecycleBinItemCollection(ClientContext, RowLimit, RecycleBinItemState.None); + for (var i = 0; i < recycleBinItemCollection.Count; i++) + { + var recycleBinItems = recycleBinItemCollection[i]; + recycleBinItems.RestoreAll(); + ClientContext.ExecuteQueryRetry(); + } } } else diff --git a/src/Commands/Utilities/RecycleBinUtility.cs b/src/Commands/Utilities/RecycleBinUtility.cs index 510f51466..464e39673 100644 --- a/src/Commands/Utilities/RecycleBinUtility.cs +++ b/src/Commands/Utilities/RecycleBinUtility.cs @@ -60,10 +60,11 @@ internal static List GetRecycleBinItems(ClientContext ctx, int? return recycleBinItems; } - internal static void RestoreOrClearRecycleBinItems(ClientContext ctx, int? rowLimit = null, RecycleBinItemState recycleBinItemState = RecycleBinItemState.None, bool restore = true) + internal static List GetRecycleBinItemCollection(ClientContext ctx, int? rowLimit = null, RecycleBinItemState recycleBinItemState = RecycleBinItemState.None) { string pagingInfo = null; RecycleBinItemCollection items; + var recycleBinItems = new List(); do { @@ -88,26 +89,18 @@ internal static void RestoreOrClearRecycleBinItems(ClientContext ctx, int? rowLi items = ctx.Site.GetRecycleBinItems(pagingInfo, iterationRowLimit, false, RecycleBinOrderBy.DefaultOrderBy, recycleBinItemState); ctx.Load(items); ctx.ExecuteQueryRetry(); + recycleBinItems.Add(items); if (items.Count > 0) { var nextId = items.Last().Id; var nextTitle = WebUtility.UrlEncode(items.Last().Title); pagingInfo = $"id={nextId}&title={nextTitle}"; - - if (restore) - { - items.RestoreAll(); - ctx.ExecuteQueryRetry(); - } - else - { - items.DeleteAll(); - ctx.ExecuteQueryRetry(); - } } } while (items?.Count == 5000); + + return recycleBinItems; } } }