Merge pull request #1465 from HicServices/persist-raw

Adds `PersistentRaw` extended property to `LoadMetadata`

Rdmp.Core.Tests/DataLoad/Engine/Integration/ToMemoryDataLoadJob.cs

@@ -52,6 +52,7 @@ public void CloseLogging()
         public HICDatabaseConfiguration Configuration { get; private set; }
 
         public object Payload { get; set; }
+        public bool PersistentRaw { get; set; }
 
         public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
         {

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -459,6 +459,14 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
                 yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced };
                 yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd, null) { OverrideCommandName = "Clear Ignored Columns", SuggestedCategory = Advanced };
 
+                yield return new ExecuteCommandSetExtendedProperty(_activator, new[] { lmd }, ExtendedProperty.PersistentRaw, null)
+                {
+                    OverrideCommandName = "Persistent RAW",
+                    PromptForValue = true,
+                    PromptForValueTaskDescription = ExtendedProperty.PersistentRawDescription,
+                    SuggestedCategory = Advanced
+                };
+
                 yield return new ExecuteCommandSet(_activator, lmd, typeof(LoadMetadata).GetProperty(nameof(LoadMetadata.IgnoreTrigger)))
                 {
                     OverrideCommandName = $"Ignore Trigger (Current value:{lmd.IgnoreTrigger})",

Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

@@ -389,5 +389,11 @@ public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoad
         {
             return runs;
         }
+
+        public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
+        {
+            return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
+                loadMetadata).Any(p => p.Value == "true");
+        }
     }
 }

Rdmp.Core/Curation/Data/ExtendedProperty.cs

@@ -42,6 +42,13 @@ public class ExtendedProperty  : Argument,IReferenceOtherObjectWithPersist, IInj
         public const string CustomJoinSql = "CustomJoinSql";
         public const string CustomJoinSqlDescription = "Enter the column comparison(s) SQL for the JOIN line.  Your string should include only the boolean comparison logic that follows the ON keyword.  E.g. col1=col2.  You can optionally use substitution tokens {0} and {1} for table name/alias (e.g. for lookup)";
 
+        /// <summary>
+        /// Key used in <see cref="ExtendedProperty"/> to indicate that a <see cref="LoadMetadata"/> should not 
+        /// attempt to DROP/CREATE its RAW database each time it is run
+        /// </summary>
+        public const string PersistentRaw = "PersistentRaw";
+        public const string PersistentRawDescription = "Should the load leave old RAW databases in the RAW server and only cleanup/reload tables at runtime? Value must be 'true' or 'false'";
+
         /// <summary>
         /// Collection of all known property names.  Plugins are free to add to these if desired but must do so pre startup
         /// </summary>
@@ -87,6 +94,7 @@ public string ReferencedObjectRepositoryType
             set { SetField(ref _referencedObjectRepositoryType, value); }
         }
 
+
         #endregion
         public ExtendedProperty()
         {

Rdmp.Core/DataLoad/Engine/Checks/Checkers/PreExecutionChecker.cs

@@ -148,12 +148,27 @@ private void PreExecutionDatabaseCheck()
 
         private void CheckRAWDatabaseIsNotPresent()
         {
+            var persistentRaw = LoadMetadata.UsesPersistentRaw(_loadMetadata);
             var rawDbInfo = _databaseConfiguration.DeployInfo[LoadBubble.Raw];
 
             // Check that the raw database is not present
-            if (!rawDbInfo.Exists()) return;
+            if (!rawDbInfo.Exists())
+            {
+                // RAW db does not exist thats usually good unless...
+                if(persistentRaw)
+                {
+                    var shouldCreate = _notifier.OnCheckPerformed(new CheckEventArgs("RAW database '" + rawDbInfo + "' does not exist but load is persistentRaw", CheckResult.Fail, null, $"Create RAW database?"));
+                    if(shouldCreate)
+                    {
+                        rawDbInfo.Create();
+                    }
+                }
 
-            var shouldDrop = _notifier.OnCheckPerformed(new CheckEventArgs("RAW database '" + rawDbInfo + "' exists", CheckResult.Fail, null, "Drop database " + rawDbInfo));
+                return;
+            }
+
+
+            var shouldDrop = _notifier.OnCheckPerformed(new CheckEventArgs("RAW database '" + rawDbInfo + "' exists", CheckResult.Fail, null, $"Drop {(persistentRaw ? "table(s)": "database")} " + rawDbInfo));
 
             if(!rawDbInfo.GetRuntimeName().EndsWith("_RAW",StringComparison.CurrentCultureIgnoreCase))
                 throw new Exception("rawDbInfo database name did not end with _RAW! It was:" + rawDbInfo.GetRuntimeName()+ " (Why is the system trying to drop this database?)");
@@ -165,8 +180,11 @@ private void CheckRAWDatabaseIsNotPresent()
                     t.Drop();
                 }
 
-                _notifier.OnCheckPerformed(new CheckEventArgs("Finally dropping database" + rawDbInfo + "...", CheckResult.Success));
-                rawDbInfo.Drop();
+                if(!persistentRaw)
+                {
+                    _notifier.OnCheckPerformed(new CheckEventArgs("Finally dropping database" + rawDbInfo + "...", CheckResult.Success));
+                    rawDbInfo.Drop();
+                }
             }
         }
 

Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

@@ -43,6 +43,9 @@ public class DataLoadJob : IDataLoadJob
         public HICDatabaseConfiguration Configuration { get; set; }
         public object Payload { get; set; }
 
+        public bool PersistentRaw { get; set; }
+
+
         private List<NotifyEventArgs> _crashAtEnd = new();
 
         public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();

Rdmp.Core/DataLoad/Engine/Job/IDataLoadJob.cs

@@ -52,6 +52,11 @@ public interface IDataLoadJob : IDataLoadEventListener, IDisposeAfterDataLoad
 
         HICDatabaseConfiguration Configuration { get; }
 
+        /// <summary>
+        /// True to automatically skip creating/dropping the RAW database
+        /// </summary>
+        bool PersistentRaw { get; set; }
+
         /// <summary>
         /// Orders the job to create the tables it requires in the given stage (e.g. RAW/STAGING), the job will also take ownership of the cloner for the purposes
         /// of disposal (DO NOT DISPOSE OF CLONER YOURSELF)

Rdmp.Core/DataLoad/Engine/Job/ThrowImmediatelyDataLoadJob.cs

@@ -69,6 +69,7 @@ public void CloseLogging()
         public HICDatabaseConfiguration Configuration { get; set; }
 
         public object Payload { get; set; }
+        public bool PersistentRaw { get; set; }
 
         private List<NotifyEventArgs> _crashAtEnd = new();
 

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/PopulateRAW.cs

@@ -77,7 +77,12 @@ private void CreateRawDatabaseIfRequired(IDataLoadJob job)
             job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Determined that we must create the RAW database tables..."));
 
             var cloner = new DatabaseCloner(_databaseConfiguration);
-            cloner.CreateDatabaseForStage(LoadBubble.Raw);
+
+            if(!job.PersistentRaw)
+            {
+                cloner.CreateDatabaseForStage(LoadBubble.Raw);
+            }
+
 
             job.CreateTablesInStage(cloner,LoadBubble.Raw);
         }

Rdmp.Core/DataLoad/Engine/LoadProcess/DataLoadProcess.cs

@@ -5,6 +5,9 @@
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
+using System.Linq;
+using NPOI;
+using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
@@ -70,6 +73,7 @@ public virtual ExitCodeType Run(GracefulCancellationToken loadCancellationToken,
                 return ExitCodeType.OperationNotRequired;
 
             job.Payload = payload;
+            job.PersistentRaw = Rdmp.Core.Curation.Data.DataLoad.LoadMetadata.UsesPersistentRaw(LoadMetadata);
 
             return LoadExecution.Run(job, loadCancellationToken);
         }