Support for retrieving multiple pages of history as an attachment.

.gitignore

@@ -286,3 +286,4 @@ __pycache__/
 *.btm.cs
 *.odx.cs
 *.xsd.cs
+/WiMigrator/configuration.json

Common/Config/ConfigJson.cs

@@ -46,6 +46,10 @@ public class ConfigJson
         [DefaultValue(false)]
         public bool MoveHistory { get; set; }
 
+        [JsonProperty(PropertyName = "move-history-limit", DefaultValueHandling = DefaultValueHandling.Populate)]
+        [DefaultValue(200)]
+        public int MoveHistoryLimit { get; set; }
+
         [JsonProperty(PropertyName = "move-git-links", DefaultValueHandling = DefaultValueHandling.Populate)]
         [DefaultValue(false)]
         public bool MoveGitLinks { get; set; }

Common/Constants.cs

@@ -5,7 +5,7 @@ namespace Common
     public class Constants
     {
         public const int BatchSize = 200;
-        public const int RevisionNumber = 200;
+        public const int PageSize = 200;
 
         public const string Hyperlink = "Hyperlink";
         public const string AttachedFile = "AttachedFile";

Common/Migration/MigrationHelpers.cs

@@ -131,18 +131,20 @@ public static JsonPatchOperation GetRelationAddOperation(WorkItemRelation relati
             return jsonPatchOperation;
         }
 
-        public static JsonPatchOperation GetRevisionHistoryAttachmentAddOperation(AttachmentReference attachmentReference, int workItemId)
+        public static JsonPatchOperation GetRevisionHistoryAttachmentAddOperation(AttachmentLink attachmentLink, int workItemId)
         {
             JsonPatchOperation jsonPatchOperation = new JsonPatchOperation();
             jsonPatchOperation.Operation = Operation.Add;
             jsonPatchOperation.Path = $"/{Constants.Relations}/-";
-            jsonPatchOperation.Value = new
+            jsonPatchOperation.Value = new WorkItemRelation
             {
-                rel = Constants.AttachedFile,
-                url = attachmentReference.Url,
-                attributes = new
+                Rel = Constants.AttachedFile,
+                Url = attachmentLink.AttachmentReference.Url,
+                Attributes = new Dictionary<string, object>
                 {
-                    name = $"{Constants.WorkItemHistory}{workItemId}.json"
+                    {  Constants.RelationAttributeName, attachmentLink.FileName },
+                    {  Constants.RelationAttributeResourceSize,  attachmentLink.ResourceSize },
+                    {  Constants.RelationAttributeComment,  attachmentLink.Comment }
                 }
             };
 

Common/Migration/Phase2/Processors/AttachmentsProcessor.cs

@@ -2,13 +2,12 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using Common.Config;
+using Logging;
 using Microsoft.Extensions.Logging;
 using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
 using Microsoft.VisualStudio.Services.WebApi.Patch.Json;
-using Logging;
-using Common.Config;
 
 namespace Common.Migration
 {

Common/Migration/Phase2/Processors/RevisionHistoryAttachmentsProcessor.cs

@@ -1,12 +1,12 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
+using Common.Config;
+using Logging;
 using Microsoft.Extensions.Logging;
 using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
-using Newtonsoft.Json;
 using Microsoft.VisualStudio.Services.WebApi.Patch.Json;
-using Logging;
-using Common.Config;
+using Newtonsoft.Json;
 
 namespace Common.Migration
 {
@@ -28,37 +28,59 @@ public async Task Preprocess(IMigrationContext migrationContext, IBatchMigration
 
         public async Task<IEnumerable<JsonPatchOperation>> Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem)
         {
-            IList<JsonPatchOperation> jsonPatchOperations = new List<JsonPatchOperation>();
-            AttachmentReference aRef = await UploadAttachmentsToTarget(migrationContext, sourceWorkItem);
-            JsonPatchOperation revisionHistoryAttachmentAddOperation = MigrationHelpers.GetRevisionHistoryAttachmentAddOperation(aRef, sourceWorkItem.Id.Value);
-            jsonPatchOperations.Add(revisionHistoryAttachmentAddOperation);
+            var jsonPatchOperations = new List<JsonPatchOperation>();
+            var attachments = await UploadAttachmentsToTarget(migrationContext, sourceWorkItem);
+            foreach (var attachment in attachments)
+            {
+                JsonPatchOperation revisionHistoryAttachmentAddOperation = MigrationHelpers.GetRevisionHistoryAttachmentAddOperation(attachment, sourceWorkItem.Id.Value);
+                jsonPatchOperations.Add(revisionHistoryAttachmentAddOperation);
+            }
 
-            return jsonPatchOperations; // We could just return one item, but we make an IList to be consistent
+            return jsonPatchOperations;
         }
 
-        private async Task<AttachmentReference> UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem)
+        private async Task<IList<AttachmentLink>> UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem)
         {
-            RevisionHistoryAttachments revisionHistoryAttachmentsItem = await GetWorkItemUpdates(migrationContext, sourceWorkItem);
+            var attachmentLinks = new List<AttachmentLink>();
+            int updateLimit = migrationContext.Config.MoveHistoryLimit;
+            int updateCount = 0;
 
-            string attachment = JsonConvert.SerializeObject(revisionHistoryAttachmentsItem.Updates);
-            AttachmentReference aRef;
-            using (MemoryStream stream = new MemoryStream())
+            while (updateCount < updateLimit)
             {
-                var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachment);
-                await stream.WriteAsync(stringBytes, 0, stringBytes.Length);
-                stream.Position = 0;
-                //upload the attachment to the target for each workitem
-                aRef = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream);
+                var updates = await GetWorkItemUpdates(migrationContext, sourceWorkItem, skip: updateCount);
+                string attachmentContent = JsonConvert.SerializeObject(updates);
+                AttachmentReference attachmentReference;
+                using (MemoryStream stream = new MemoryStream())
+                {
+                    var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachmentContent);
+                    await stream.WriteAsync(stringBytes, 0, stringBytes.Length);
+                    stream.Position = 0;
+                    //upload the attachment to the target for each workitem
+                    attachmentReference = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream);
+                    attachmentLinks.Add(
+                        new AttachmentLink(
+                            $"{Constants.WorkItemHistory}-{sourceWorkItem.Id}-{updateCount}.json", 
+                            attachmentReference, 
+                            stringBytes.Length,
+                            comment: $"Update range from {updateCount} to {updateCount + updates.Count}"));
+                }
+
+                updateCount += updates.Count;
+
+                // if we got less than a page size, that means we're on the last
+                // page and shouldn't try and read another page.
+                if (updates.Count < Constants.PageSize)
+                {
+                    break;
+                }
             }
 
-            return aRef;
+            return attachmentLinks;
         }
 
-        private async Task<RevisionHistoryAttachments> GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem)
+        private async Task<IList<WorkItemUpdate>> GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem, int skip = 0)
         {
-            IList<RevisionHistoryAttachments> revisionHistoryAttachments = new List<RevisionHistoryAttachments>();
-            var wiUpdates = await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value);
-            return new RevisionHistoryAttachments { Workitem = sourceWorkItem, Updates = wiUpdates };
+            return await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value, skip);
         }
     }
 }

Common/WorkItemTrackingHelpers.cs

@@ -77,11 +77,11 @@ public async static Task<IList<WorkItem>> GetWorkItemsAsync(WorkItemTrackingHttp
             }, 5);
         }
 
-        public async static Task<List<WorkItemUpdate>> GetWorkItemUpdatesAsync(WorkItemTrackingHttpClient client, int id)
+        public async static Task<List<WorkItemUpdate>> GetWorkItemUpdatesAsync(WorkItemTrackingHttpClient client, int id, int skip = 0)
         {
             return await RetryHelper.RetryAsync(async () =>
             {
-                return await client.GetUpdatesAsync(id, Constants.RevisionNumber);
+                return await client.GetUpdatesAsync(id, Constants.PageSize, skip: skip);
             }, 5);
         }
 

WiMigrator/WiMigrator.csproj

@@ -23,7 +23,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="sample-configuration.json">
+    <None Update="configuration.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>

WiMigrator/sample-configuration.json

@@ -56,9 +56,12 @@
   // when false, it will update any previously migrated work items
   // that have changed on the source since the migration was completed.
   "skip-existing": true,
-  // create a json file containing the last 200 updates of the source
+  // create a json file containing the updates of the source
   // work item and attach it to the migrated work item.
   "move-history": false,
+  // the limit to the number of updates of the source work item
+  // to attach to the migrated work item.
+  "move-history-limit": 200,
   // migrate git commit links as hyperlinks that point to the
   // web view of the commit on the source account.
   "move-git-links": false,