From d95c382b5759a5e2a5ab785a5cde15f2ee0192de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=C3=A1n=20Ferguson?= Date: Sat, 3 Feb 2018 15:13:08 -0800 Subject: [PATCH] Support for retrieving multiple pages of history as an attachment. --- .gitignore | 1 + Common/Config/ConfigJson.cs | 4 ++ Common/Constants.cs | 2 +- Common/Migration/MigrationHelpers.cs | 14 ++-- .../Phase2/Processors/AttachmentsProcessor.cs | 5 +- .../RevisionHistoryAttachmentsProcessor.cs | 68 ++++++++++++------- .../Migration/RevisionHistoryAttachments.cs | 11 --- Common/WorkItemTrackingHelpers.cs | 4 +- WiMigrator/WiMigrator.csproj | 2 +- WiMigrator/sample-configuration.json | 5 +- 10 files changed, 68 insertions(+), 48 deletions(-) delete mode 100644 Common/Migration/RevisionHistoryAttachments.cs diff --git a/.gitignore b/.gitignore index 940794e..8b61648 100644 --- a/.gitignore +++ b/.gitignore @@ -286,3 +286,4 @@ __pycache__/ *.btm.cs *.odx.cs *.xsd.cs +/WiMigrator/configuration.json diff --git a/Common/Config/ConfigJson.cs b/Common/Config/ConfigJson.cs index 593370e..c478639 100644 --- a/Common/Config/ConfigJson.cs +++ b/Common/Config/ConfigJson.cs @@ -46,6 +46,10 @@ public class ConfigJson [DefaultValue(false)] public bool MoveHistory { get; set; } + [JsonProperty(PropertyName = "move-history-limit", DefaultValueHandling = DefaultValueHandling.Populate)] + [DefaultValue(200)] + public int MoveHistoryLimit { get; set; } + [JsonProperty(PropertyName = "move-git-links", DefaultValueHandling = DefaultValueHandling.Populate)] [DefaultValue(false)] public bool MoveGitLinks { get; set; } diff --git a/Common/Constants.cs b/Common/Constants.cs index 14c000b..0c6d3f2 100644 --- a/Common/Constants.cs +++ b/Common/Constants.cs @@ -5,7 +5,7 @@ namespace Common public class Constants { public const int BatchSize = 200; - public const int RevisionNumber = 200; + public const int PageSize = 200; public const string Hyperlink = "Hyperlink"; public const string AttachedFile = "AttachedFile"; diff --git a/Common/Migration/MigrationHelpers.cs b/Common/Migration/MigrationHelpers.cs index 0bc9e3b..2792ed3 100644 --- a/Common/Migration/MigrationHelpers.cs +++ b/Common/Migration/MigrationHelpers.cs @@ -131,18 +131,20 @@ public static JsonPatchOperation GetRelationAddOperation(WorkItemRelation relati return jsonPatchOperation; } - public static JsonPatchOperation GetRevisionHistoryAttachmentAddOperation(AttachmentReference attachmentReference, int workItemId) + public static JsonPatchOperation GetRevisionHistoryAttachmentAddOperation(AttachmentLink attachmentLink, int workItemId) { JsonPatchOperation jsonPatchOperation = new JsonPatchOperation(); jsonPatchOperation.Operation = Operation.Add; jsonPatchOperation.Path = $"/{Constants.Relations}/-"; - jsonPatchOperation.Value = new + jsonPatchOperation.Value = new WorkItemRelation { - rel = Constants.AttachedFile, - url = attachmentReference.Url, - attributes = new + Rel = Constants.AttachedFile, + Url = attachmentLink.AttachmentReference.Url, + Attributes = new Dictionary { - name = $"{Constants.WorkItemHistory}{workItemId}.json" + { Constants.RelationAttributeName, attachmentLink.FileName }, + { Constants.RelationAttributeResourceSize, attachmentLink.ResourceSize }, + { Constants.RelationAttributeComment, attachmentLink.Comment } } }; diff --git a/Common/Migration/Phase2/Processors/AttachmentsProcessor.cs b/Common/Migration/Phase2/Processors/AttachmentsProcessor.cs index e1ab85f..8562976 100644 --- a/Common/Migration/Phase2/Processors/AttachmentsProcessor.cs +++ b/Common/Migration/Phase2/Processors/AttachmentsProcessor.cs @@ -2,13 +2,12 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; +using Common.Config; +using Logging; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; using Microsoft.VisualStudio.Services.WebApi.Patch.Json; -using Logging; -using Common.Config; namespace Common.Migration { diff --git a/Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs b/Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs index c36c7aa..84a5ada 100644 --- a/Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs +++ b/Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs @@ -1,12 +1,12 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Common.Config; +using Logging; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; -using Newtonsoft.Json; using Microsoft.VisualStudio.Services.WebApi.Patch.Json; -using Logging; -using Common.Config; +using Newtonsoft.Json; namespace Common.Migration { @@ -28,37 +28,59 @@ public async Task Preprocess(IMigrationContext migrationContext, IBatchMigration public async Task> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem) { - IList jsonPatchOperations = new List(); - AttachmentReference aRef = await UploadAttachmentsToTarget(migrationContext, sourceWorkItem); - JsonPatchOperation revisionHistoryAttachmentAddOperation = MigrationHelpers.GetRevisionHistoryAttachmentAddOperation(aRef, sourceWorkItem.Id.Value); - jsonPatchOperations.Add(revisionHistoryAttachmentAddOperation); + var jsonPatchOperations = new List(); + var attachments = await UploadAttachmentsToTarget(migrationContext, sourceWorkItem); + foreach (var attachment in attachments) + { + JsonPatchOperation revisionHistoryAttachmentAddOperation = MigrationHelpers.GetRevisionHistoryAttachmentAddOperation(attachment, sourceWorkItem.Id.Value); + jsonPatchOperations.Add(revisionHistoryAttachmentAddOperation); + } - return jsonPatchOperations; // We could just return one item, but we make an IList to be consistent + return jsonPatchOperations; } - private async Task UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem) + private async Task> UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem) { - RevisionHistoryAttachments revisionHistoryAttachmentsItem = await GetWorkItemUpdates(migrationContext, sourceWorkItem); + var attachmentLinks = new List(); + int updateLimit = migrationContext.Config.MoveHistoryLimit; + int updateCount = 0; - string attachment = JsonConvert.SerializeObject(revisionHistoryAttachmentsItem.Updates); - AttachmentReference aRef; - using (MemoryStream stream = new MemoryStream()) + while (updateCount < updateLimit) { - var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachment); - await stream.WriteAsync(stringBytes, 0, stringBytes.Length); - stream.Position = 0; - //upload the attachment to the target for each workitem - aRef = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream); + var updates = await GetWorkItemUpdates(migrationContext, sourceWorkItem, skip: updateCount); + string attachmentContent = JsonConvert.SerializeObject(updates); + AttachmentReference attachmentReference; + using (MemoryStream stream = new MemoryStream()) + { + var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachmentContent); + await stream.WriteAsync(stringBytes, 0, stringBytes.Length); + stream.Position = 0; + //upload the attachment to the target for each workitem + attachmentReference = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream); + attachmentLinks.Add( + new AttachmentLink( + $"{Constants.WorkItemHistory}-{sourceWorkItem.Id}-{updateCount}.json", + attachmentReference, + stringBytes.Length, + comment: $"Update range from {updateCount} to {updateCount + updates.Count}")); + } + + updateCount += updates.Count; + + // if we got less than a page size, that means we're on the last + // page and shouldn't try and read another page. + if (updates.Count < Constants.PageSize) + { + break; + } } - return aRef; + return attachmentLinks; } - private async Task GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem) + private async Task> GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem, int skip = 0) { - IList revisionHistoryAttachments = new List(); - var wiUpdates = await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value); - return new RevisionHistoryAttachments { Workitem = sourceWorkItem, Updates = wiUpdates }; + return await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value, skip); } } } diff --git a/Common/Migration/RevisionHistoryAttachments.cs b/Common/Migration/RevisionHistoryAttachments.cs deleted file mode 100644 index de8d4eb..0000000 --- a/Common/Migration/RevisionHistoryAttachments.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Collections.Generic; -using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; - -namespace Common.Migration -{ - public class RevisionHistoryAttachments - { - public WorkItem Workitem { get; set; } - public List Updates { get; set; } - } -} diff --git a/Common/WorkItemTrackingHelpers.cs b/Common/WorkItemTrackingHelpers.cs index 9c1f983..31f7bdd 100644 --- a/Common/WorkItemTrackingHelpers.cs +++ b/Common/WorkItemTrackingHelpers.cs @@ -77,11 +77,11 @@ public async static Task> GetWorkItemsAsync(WorkItemTrackingHttp }, 5); } - public async static Task> GetWorkItemUpdatesAsync(WorkItemTrackingHttpClient client, int id) + public async static Task> GetWorkItemUpdatesAsync(WorkItemTrackingHttpClient client, int id, int skip = 0) { return await RetryHelper.RetryAsync(async () => { - return await client.GetUpdatesAsync(id, Constants.RevisionNumber); + return await client.GetUpdatesAsync(id, Constants.PageSize, skip: skip); }, 5); } diff --git a/WiMigrator/WiMigrator.csproj b/WiMigrator/WiMigrator.csproj index db8491b..a0c443e 100644 --- a/WiMigrator/WiMigrator.csproj +++ b/WiMigrator/WiMigrator.csproj @@ -23,7 +23,7 @@ - + Always diff --git a/WiMigrator/sample-configuration.json b/WiMigrator/sample-configuration.json index 2df3cec..a06c8ad 100644 --- a/WiMigrator/sample-configuration.json +++ b/WiMigrator/sample-configuration.json @@ -56,9 +56,12 @@ // when false, it will update any previously migrated work items // that have changed on the source since the migration was completed. "skip-existing": true, - // create a json file containing the last 200 updates of the source + // create a json file containing the updates of the source // work item and attach it to the migrated work item. "move-history": false, + // the limit to the number of updates of the source work item + // to attach to the migrated work item. + "move-history-limit": 200, // migrate git commit links as hyperlinks that point to the // web view of the commit on the source account. "move-git-links": false,