+ Implement 'incremental' provision support #553

+ Persistence storage base service
+ Persistence storage 'file system' implementation, tests

SPMeta2/SPMeta2.Regression.Impl.Tests/Impl/Services/ProvisionServiceTests.cs

@@ -1,28 +1,25 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
+using System.Collections.Generic;
 using System.Linq;
-using System.Reflection;
-using System.Runtime.CompilerServices;
-using Microsoft.SharePoint;
-using Microsoft.SharePoint.Taxonomy;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using SPMeta2.CSOM.ModelHandlers;
+using SPMeta2.Common;
 using SPMeta2.CSOM.Services;
-using SPMeta2.CSOM.Standard.ModelHandlers.Fields;
 using SPMeta2.CSOM.Standard.Services;
+using SPMeta2.Interfaces;
 using SPMeta2.ModelHandlers;
-using SPMeta2.Models;
 using SPMeta2.Services;
 using SPMeta2.SSOM.Services;
 using SPMeta2.SSOM.Standard.Services;
-using SPMeta2.Standard.Definitions.Taxonomy;
-using SPMeta2.Standard.Syntax;
-using SPMeta2.Syntax.Default;
 using SPMeta2.Utils;
-using SPMeta2.Interfaces;
+using SPMeta2.Containers.CSOM;
+using SPMeta2.Services.Impl;
+using SPMeta2.Syntax.Default;
+using SPMeta2.Containers.O365;
+using SPMeta2.CSOM.ModelHosts;
+using SPMeta2.Definitions;
+using SPMeta2.Exceptions;
+using System;
 
-namespace SPMeta2.Regression.Impl.Tests
+namespace SPMeta2.Regression.Impl.Tests.Impl.Services
 {
     [TestClass]
     public class ModelHandlersTest
@@ -149,7 +146,7 @@ public void EnsureStandardSSOMModelHandlers()
         [TestMethod]
         [TestCategory("Regression.Impl.IncrementalProvisionService")]
         [TestCategory("CI.Core")]
-        public void Can_Create_IncrementaProvisionServices()
+        public void Can_Create_IncrementalProvisionServices()
         {
             var services = new List<ProvisionServiceBase>();
 
@@ -171,7 +168,7 @@ public void Can_Create_IncrementaProvisionServices()
         [TestMethod]
         [TestCategory("Regression.Impl.IncrementalProvisionService")]
         [TestCategory("CI.Core")]
-        public void Can_Create_IncrementaProvisionServices_With_FluentAPI()
+        public void Can_Create_IncrementalProvisionServices_With_FluentAPI()
         {
             var services = new List<ProvisionServiceBase>();
 
@@ -191,6 +188,81 @@ public void Can_Create_IncrementaProvisionServices_With_FluentAPI()
             }
         }
 
+
+
+        #endregion
+
+        #region persistence storage
+
+
+        [TestMethod]
+        [TestCategory("Regression.Impl.IncrementalProvisionService.PersistenceStorage")]
+        [TestCategory("CI.Core")]
+        [ExpectedException(typeof(SPMeta2Exception))]
+        public void Can_Provision_Incrementally_With_NoIncrementalModelId()
+        {
+            var provisionRunner = new O365ProvisionRunner();
+            var provisionService = provisionRunner.ProvisionService;
+
+            var incrementalProvisionConfig = new IncrementalProvisionConfig();
+            incrementalProvisionConfig.PersistenceStorages.Add(new DefaultFileSystemPersistenceStorage());
+
+            provisionService.SetIncrementalProvisionMode(incrementalProvisionConfig);
+
+            provisionRunner.SiteUrls.ForEach(siteUrl =>
+            {
+                var model = SPMeta2Model.NewSiteModel(site => { });
+
+                provisionRunner.WithO365Context(siteUrl, context =>
+                {
+                    provisionService.DeployModel(SiteModelHost.FromClientContext(context), model);
+                });
+            });
+        }
+
+        [TestMethod]
+        [TestCategory("Regression.Impl.IncrementalProvisionService.PersistenceStorage")]
+        [TestCategory("CI.Core")]
+        public void Can_Provision_Incrementally_With_FileSystemStorage()
+        {
+            var provisionRunner = new O365ProvisionRunner();
+            var provisionService = provisionRunner.ProvisionService;
+
+            var incrementalProvisionConfig = new IncrementalProvisionConfig();
+            incrementalProvisionConfig.PersistenceStorages.Add(new DefaultFileSystemPersistenceStorage());
+
+            provisionService.SetIncrementalProvisionMode(incrementalProvisionConfig);
+
+            provisionRunner.SiteUrls.ForEach(siteUrl =>
+            {
+                var incrementalModelId = "m2.regression." + Guid.NewGuid().ToString("N");
+
+                var model = SPMeta2Model.NewSiteModel(site =>
+                {
+
+                });
+
+                var incrementalRequireSelfProcessingValue = model.NonPersistentPropertyBag
+                .FirstOrDefault(p => p.Name == "_sys.IncrementalRequireSelfProcessingValue");
+
+                if (incrementalRequireSelfProcessingValue == null)
+                {
+                    incrementalRequireSelfProcessingValue = new PropertyBagValue
+                    {
+                        Name = "_sys.IncrementalProvision.PersistenceStorageModelId",
+                        Value = incrementalModelId
+                    };
+
+                    model.PropertyBag.Add(incrementalRequireSelfProcessingValue);
+                }
+
+                provisionRunner.WithO365Context(siteUrl, context =>
+                {
+                    provisionService.DeployModel(SiteModelHost.FromClientContext(context), model);
+                });
+            });
+        }
+
         #endregion
     }
 }

SPMeta2/SPMeta2.Regression.Impl.Tests/Impl/Services/SSOM/SSOMTokenReplacementServiceTests.cs

@@ -1,23 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text;
-
-using Microsoft.VisualStudio.TestTools.UnitTesting;
-using SPMeta2.CSOM.Services;
-using SPMeta2.Services;
-using System.Reflection;
-using Microsoft.SharePoint;
-using Microsoft.SharePoint.Client;
-using SPMeta2.Containers.CSOM;
-using SPMeta2.Containers.Services;
-using SPMeta2.CSOM.Extensions;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 using SPMeta2.Containers.SSOM;
 using SPMeta2.Regression.Impl.Tests.Impl.Services.Base;
+using SPMeta2.Services;
 using SPMeta2.SSOM.Services;
 
-namespace SPMeta2.Regression.Impl.Tests.Impl.Services
+namespace SPMeta2.Regression.Impl.Tests.Impl.Services.SSOM
 {
 
     [TestClass]

SPMeta2/SPMeta2.Regression.Tests/Impl/Scenarios/Base/SPMeta2RegresionScenarioTestBase.cs

@@ -32,7 +32,8 @@ public SPMeta2RegresionScenarioTestBase()
             {
                 RegressionService.BeforeProvisionRunnerExcecution += (runner) =>
                 {
-                    runner.ProvisionService.SetIncrementalProvisionMode();
+                    var config = new IncrementalProvisionConfig();
+                    runner.ProvisionService.SetIncrementalProvisionMode(config);
 
                     runner.OnBeforeDeployModel += (provisionService, model) =>
                     {

SPMeta2/SPMeta2.Regression.Tests/Impl/Services/IncrementalModelTreeTraverseServiceTests.cs

@@ -10,6 +10,7 @@
 using SPMeta2.Services.Impl;
 using SPMeta2.Syntax.Default;
 using System.Collections.Generic;
+using System.Text;
 using SPMeta2.Extensions;
 using SPMeta2.Regression.Utils;
 using SPMeta2.Containers.Services.Rnd;
@@ -21,7 +22,7 @@
 
 namespace SPMeta2.Regression.Tests.Impl.Services
 {
-   
+
 
     [TestClass]
     public class IncrementalModelTreeTraverseServiceTests
@@ -206,6 +207,24 @@ public void Incremental_Update_AllDefinitions_As_RandomModels()
                 secondProvisionService.PreviousModelHash = firstProvisionService.CurrentModelHash;
                 secondProvisionService.Traverse(null, currentModel);
 
+                // trace size of the model hash  + amount if the aritfacts
+                var modelNodesCount = 0;
+                model.WithNodesOfType<DefinitionBase>(n => { modelNodesCount++; });
+
+                var serializer = ServiceContainer.Instance.GetService<DefaultXMLSerializationService>();
+                serializer.RegisterKnownTypes(new[]
+                {
+                    typeof(ModelHash),
+                    typeof(ModelNodeHash)
+                });
+
+                var data = Encoding.UTF8.GetBytes(serializer.Serialize(firstProvisionService.CurrentModelHash));
+
+                var persistanceFileService = new DefaultFileSystemPersistenceStorage();
+                persistanceFileService.SaveObject(
+                                        string.Format("incremental_state_m2.regression-artifact-{1}-{0}", definitionType.Name, modelNodesCount),
+                                        data);
+
                 RegressionUtils.WriteLine(string.Empty);
                 RegressionUtils.WriteLine("Provisioned model:");
                 RegressionUtils.WriteLine(ModelPrintService.PrintModel(currentModel));
@@ -278,6 +297,8 @@ public void Incremental_Update_NonSingleton_ModelNodes_PlusRandomModelNodes()
 
         #endregion
 
+
+
         #region utils
 
         protected ModelNode GetVeryRandomModel(Type definitionType, SPObjectModelType spObjectModelType)

SPMeta2/SPMeta2/Common/IncrementalProvisionConfig.cs

@@ -3,16 +3,21 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using SPMeta2.Services;
 
 namespace SPMeta2.Common
 {
+
+
+
     public class IncrementalProvisionConfig
     {
         #region constructors
 
         public IncrementalProvisionConfig()
         {
             PreviousModelHash = new ModelHash();
+            PersistenceStorages = new List<PersistenceStorageServiceBase>();
         }
 
         #endregion
@@ -22,6 +27,8 @@ public IncrementalProvisionConfig()
 
         public Type CustomModelTreeTraverseServiceType { get; set; }
 
+        public List<PersistenceStorageServiceBase> PersistenceStorages { get; set; }
+
         #endregion
 
         public static IncrementalProvisionConfig Default

SPMeta2/SPMeta2/Common/ModelHash.cs

@@ -31,6 +31,15 @@ public ModelHash()
         public List<ModelNodeHash> ModelNodes { get; set; }
 
         #endregion
+
+        #region override
+
+        public override string ToString()
+        {
+            return string.Format("Model hash:[{0}] nodes count:[{1}]", Hash, ModelNodes.Count);
+        }
+
+        #endregion
     }
 
 
@@ -56,6 +65,15 @@ public class ModelNodeHash
         public string DefinitionIdentityKeyHash { get; set; }
 
         #endregion
+
+        #region override
+
+        public override string ToString()
+        {
+            return string.Format("Identity key:[{0}] Definition hash:[{1}]", DefinitionIdentityKey, DefinitionHash);
+        }
+
+        #endregion
     }
 
     //[DataContract]

SPMeta2/SPMeta2/SPMeta2.csproj

@@ -362,6 +362,7 @@
     <Compile Include="Services\DeploymentServiceBase.cs" />
     <Compile Include="Services\FluentModelValidationServiceBase.cs" />
     <Compile Include="Services\HashCodeServiceBase.cs" />
+    <Compile Include="Services\Impl\DefaultFileSystemPersistenceStorage.cs" />
     <Compile Include="Services\Impl\DefaultIncrementalModelPrettyPrintService.cs" />
     <Compile Include="Services\Impl\DefaultModelStatInfoService.cs" />
     <Compile Include="Services\Impl\DefaultDiagnosticInfoService.cs" />
@@ -399,6 +400,7 @@
     <Compile Include="Services\ModelTreeTraverseServiceBase.cs" />
     <Compile Include="Services\ModelWeighServiceBase.cs" />
     <Compile Include="Services\Impl\TraceSourceService.cs" />
+    <Compile Include="Services\PersistenceStorageServiceBase.cs" />
     <Compile Include="Services\PostDeploymentServiceBase.cs" />
     <Compile Include="Services\PreDeploymentServiceBase.cs" />
     <Compile Include="Services\ModelDotPrintServiceBase.cs" />

SPMeta2/SPMeta2/Services/Impl/DefaultFileSystemPersistenceStorage.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SPMeta2.Services.Impl
+{
+    public class DefaultFileSystemPersistenceStorage : PersistenceStorageServiceBase
+    {
+        #region constructors
+
+        public DefaultFileSystemPersistenceStorage()
+            : this(DefaultFolderPath)
+        {
+
+        }
+
+        public DefaultFileSystemPersistenceStorage(string folderPath)
+        {
+            CurrentFolderPath = folderPath;
+        }
+
+        #endregion
+
+        #region static
+
+        static DefaultFileSystemPersistenceStorage()
+        {
+            var path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SPMeta2");
+            DefaultFolderPath = path;
+        }
+
+        public static string DefaultFolderPath { get; set; }
+        public string CurrentFolderPath { get; set; }
+
+        #endregion
+
+        #region methods
+
+        public override byte[] LoadObject(string objectId)
+        {
+            var fileName = string.Format("{0}.state", objectId);
+            var filePath = Path.Combine(CurrentFolderPath, fileName);
+
+            if (File.Exists(filePath))
+                return File.ReadAllBytes(filePath);
+
+            return null;
+        }
+
+        public override void SaveObject(string objectId, byte[] data)
+        {
+            Directory.CreateDirectory(CurrentFolderPath);
+
+            var fileName = string.Format("{0}.state", objectId);
+            var filePath = Path.Combine(CurrentFolderPath, fileName);
+
+            File.WriteAllBytes(filePath, data);
+        }
+        #endregion
+
+
+    }
+}