Task/rdmp 151 Custom Load Metadata Locations

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add Key-Value store for instance settings
 - Allow for Re-extractions of projects to a database, see [ExecuteFullExtractionToDatabaseMSSql](Documentation\DataExtractions\ExecuteFullExtractionToDatabaseMSSql.md)
+- Add ability to customise LoadMetdata Folder Location. See [LoadMetadata](Documentation\DataLoadEngine\LoadMetadata.md)
 - Add ability to point a catalogue to a new data source [Documentation](./Documentation/Catalogues/UpdateCatalogueDataLocation.md)
 - Allow DQE graphs to be scrollable and scalable
 - Allow for partial refreshes of time-based DQE charts

Documentation/CodeTutorials/FAQ.md

@@ -412,7 +412,7 @@ Yes, [determining database types from untyped data (e.g. CSV)](./DataTableUpload
 
 <a name="dle-folders"></a>
 ### What is the purpose of the the data load folders (ForLoading, ForArchiving etc)?
-RDMP data load jobs are configured by creating a [LoadMetadata].  When you create a new [LoadMetadata] you will be prompted to choose/create a set of load folders.  The current value is stored in the `LocationOfFlatFiles` field of [LoadMetadata].  The directories created have the following layout:
+RDMP data load jobs are configured by creating a [LoadMetadata].  When you create a new [LoadMetadata] you will be prompted to choose/create a set of load folders.  The default directories created have the following layout:
 
 ```
 someFolder

Documentation/DataLoadEngine/LoadMetadata.md

@@ -0,0 +1,33 @@
+# Load Metadata
+The Load Metadata stores a reference to the directory you want your data load pipeline to use.
+This directory will be populated with a number of folders
+```
+- MyFolder	
+|_ Data
+|  |_ Cache
+|  |_ ForArchiving
+|  |_ ForLoading
+|_ Executables
+```
+The ForLoading Directory is used as a holding pen for data that you are trying to load into the system
+
+The ForArchiving Directory is used to store a compressed varion of the data involved in the data load for archiving purposes
+
+The Executables directory is provided for storing executable files and/or sql scripts that you might want to run during the load.
+
+The Cache folder is used for the RDMP caching engine (long running fetching tasks e.g. pulling images from a PACS server or reports from a webservice).
+
+
+## Using Different Locations for each Load Metadata Directory
+N.B. This is an advanced feature and should be used with caution.
+
+![Load Metadata Config](./Images/lmdConfig.PNG)
+
+
+Some users may with to store their archives, for example, in a nother location or disk.
+RDMP allows this by having the user specify the directory they wish each type of data to be written to.
+
+This functionality will write files directly into this folder, not into a "Data" subfolder.
+It is not recommended to set each folder option to the same location as RDMP will clean up the "ForLoading" directory and delete all files, including the archive, during this step.
+
+

Rdmp.Core.Tests/Caching/Integration/CachingHostTests.cs

@@ -14,6 +14,7 @@
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Cache;
+using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.ReusableLibraryCode.Progress;
 using Tests.Common;
@@ -39,7 +40,10 @@ public void CacheHostOutwithPermissionWindow()
 
         var cp = WhenIHaveA<CacheProgress>();
         var loadMetadata = cp.LoadProgress.LoadMetadata;
-        loadMetadata.LocationOfFlatFiles = loadDirectory.RootPath.FullName;
+        loadMetadata.LocationOfForLoadingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForLoadingPath);
+        loadMetadata.LocationOfForArchivingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForArchivingPath);
+        loadMetadata.LocationOfExecutablesDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultExecutablesPath);
+        loadMetadata.LocationOfCacheDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultCachePath);
 
         // This feels a bit nasty, but quick and much better than having the test wait for an arbitrary time period.
         var listener = new ExpectedNotificationListener("Download not permitted at this time, sleeping for 60 seconds");

Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs

@@ -61,7 +61,10 @@ public void FetchMultipleDays_Success(bool singleDay)
                 true);
 
         var lmd = Substitute.For<ILoadMetadata>();
-        lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
+        lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForLoading"));
+        lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForArchiving"));
+        lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName ,"Executables");
+        lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "Cache"));
 
         var loadProgress = Substitute.For<ILoadProgress>();
         loadProgress.OriginDate.Returns(new DateTime(2001, 01, 01));

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateNewDataLoadDirectoryTests.cs

@@ -31,14 +31,41 @@ public void TestCreateNewDataLoadDirectory_WithLoadMetadata()
         if (Directory.Exists(root)) Directory.Delete(root, true);
         var lmd = WhenIHaveA<LoadMetadata>();
 
-        Assert.That(lmd.LocationOfFlatFiles, Is.Null);
+        Assert.That(lmd.LocationOfForLoadingDirectory, Is.Null);
 
         Run("CreateNewDataLoadDirectory", $"LoadMetadata:{lmd.ID}", root);
 
         Assert.Multiple(() =>
         {
             Assert.That(Directory.Exists(root));
-            Assert.That(lmd.LocationOfFlatFiles, Is.EqualTo(root));
+            Assert.That(lmd.LocationOfForLoadingDirectory, Is.EqualTo($"{root}\\Data\\ForLoading"));
+            Assert.That(lmd.LocationOfForArchivingDirectory, Is.EqualTo($"{root}\\Data\\ForArchiving"));
+            Assert.That(lmd.LocationOfExecutablesDirectory, Is.EqualTo($"{root}\\Executables"));
+            Assert.That(lmd.LocationOfCacheDirectory, Is.EqualTo($"{root}\\Data\\Cache"));
+        });
+    }
+
+    [Test]
+    public void TestCreateNewDataLoadDirectory_WithSplitLoadMetadata()
+    {
+        var root = Path.Combine(TestContext.CurrentContext.WorkDirectory, "def");
+        if (Directory.Exists(root)) Directory.Delete(root, true);
+        var lmd = WhenIHaveA<LoadMetadata>();
+
+        Assert.That(lmd.LocationOfForLoadingDirectory, Is.Null);
+        Assert.That(lmd.LocationOfForArchivingDirectory, Is.Null);
+        Assert.That(lmd.LocationOfExecutablesDirectory, Is.Null);
+        Assert.That(lmd.LocationOfCacheDirectory, Is.Null);
+
+        Run("CreateNewSplitDataLoadDirectory", $"LoadMetadata:{lmd.ID}", root, root, root, root);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(lmd.LocationOfForLoadingDirectory, Is.EqualTo(root));
+            Assert.That(lmd.LocationOfForArchivingDirectory, Is.EqualTo(root));
+            Assert.That(lmd.LocationOfExecutablesDirectory, Is.EqualTo(root));
+            Assert.That(lmd.LocationOfCacheDirectory, Is.EqualTo(root));
+
         });
     }
 }

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndCacheTest.cs

@@ -49,7 +49,10 @@ protected override void SetUp()
         _LoadDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
                 @"EndToEndCacheTest", true);
-        _lmd.LocationOfFlatFiles = _LoadDirectory.RootPath.FullName;
+        _lmd.LocationOfForLoadingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForLoadingPath);
+        _lmd.LocationOfForArchivingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForArchivingPath);
+        _lmd.LocationOfExecutablesDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultExecutablesPath);
+        _lmd.LocationOfCacheDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultCachePath);
         _lmd.SaveToDatabase();
 
         Clear(_LoadDirectory);

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndDLETest.cs

@@ -13,6 +13,7 @@
 using Tests.Common;
 using Tests.Common.Scenarios;
 using TypeGuesser;
+using System.IO;
 
 namespace Rdmp.Core.Tests.CommandLine.AutomationLoopTests;
 
@@ -44,10 +45,11 @@ public void TestDle_DodgyColumnNames(DatabaseType dbType)
         });
 
         var cata = Import(tbl);
-        var lmd = new LoadMetadata(CatalogueRepository, nameof(TestDle_DodgyColumnNames))
-        {
-            LocationOfFlatFiles = LoadDirectory.RootPath.FullName
-        };
+        var lmd = new LoadMetadata(CatalogueRepository, nameof(TestDle_DodgyColumnNames));
+        lmd.LocationOfForLoadingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForLoadingPath);
+        lmd.LocationOfForArchivingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForArchivingPath);
+        lmd.LocationOfExecutablesDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultExecutablesPath);
+        lmd.LocationOfCacheDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultCachePath);
         lmd.SaveToDatabase();
 
         CreateFlatFileAttacher(lmd, "Troll.csv", cata.GetTableInfoList(false).Single());

Rdmp.Core.Tests/Curation/Integration/LoadMetadataTests.cs

@@ -5,7 +5,9 @@
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
+using System.IO;
 using NUnit.Framework;
+using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.DataLoad.Engine.Checks.Checkers;
 using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
@@ -24,11 +26,17 @@ public void CreateNewAndGetBackFromDatabase()
 
         try
         {
-            loadMetadata.LocationOfFlatFiles = TestContext.CurrentContext.TestDirectory;
+            loadMetadata.LocationOfForLoadingDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultForLoadingPath);
+            loadMetadata.LocationOfForArchivingDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultForArchivingPath);
+            loadMetadata.LocationOfExecutablesDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultExecutablesPath);
+            loadMetadata.LocationOfCacheDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultCachePath);
             loadMetadata.SaveToDatabase();
 
             var loadMetadataWithIdAfterwards = CatalogueRepository.GetObjectByID<LoadMetadata>(loadMetadata.ID);
-            Assert.That(TestContext.CurrentContext.TestDirectory, Is.EqualTo(loadMetadataWithIdAfterwards.LocationOfFlatFiles));
+            Assert.That(Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultForLoadingPath), Is.EqualTo(loadMetadataWithIdAfterwards.LocationOfForLoadingDirectory));
+            Assert.That(Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultForArchivingPath), Is.EqualTo(loadMetadataWithIdAfterwards.LocationOfForArchivingDirectory));
+            Assert.That(Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultExecutablesPath), Is.EqualTo(loadMetadataWithIdAfterwards.LocationOfExecutablesDirectory));
+            Assert.That(Path.Combine(TestContext.CurrentContext.TestDirectory, loadMetadata.DefaultCachePath), Is.EqualTo(loadMetadataWithIdAfterwards.LocationOfCacheDirectory));
         }
         finally
         {

Rdmp.Core.Tests/Curation/Integration/LoadProgressUnitTests.cs

@@ -14,6 +14,7 @@
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
 using Tests.Common;
+using Rdmp.Core.Curation.Data.DataLoad;
 
 namespace Rdmp.Core.Tests.Curation.Integration;
 
@@ -69,7 +70,10 @@ public void LoadProgress_JobFactory_NoDates()
             "LoadProgress_JobFactory_NoDates", true);
 
         var lmd = lp.LoadMetadata;
-        lmd.LocationOfFlatFiles = dir.RootPath.FullName;
+        lmd.LocationOfForLoadingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForLoadingPath);
+        lmd.LocationOfForArchivingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForArchivingPath);
+        lmd.LocationOfExecutablesDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultExecutablesPath);
+        lmd.LocationOfCacheDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultCachePath);
 
         foreach (var cata in lmd.GetAllCatalogues())
         {

Rdmp.Core.Tests/DataLoad/Engine/Integration/CheckingTests/ProcessTaskCheckingTests.cs

@@ -36,7 +36,10 @@ public void CreateTask()
         _dir.Create();
 
         var hicdir = LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true);
-        _lmd.LocationOfFlatFiles = hicdir.RootPath.FullName;
+        _lmd.LocationOfForLoadingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForLoadingPath);
+        _lmd.LocationOfForArchivingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForArchivingPath);
+        _lmd.LocationOfExecutablesDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultExecutablesPath);
+        _lmd.LocationOfCacheDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultCachePath);
         _lmd.SaveToDatabase();
 
         var c = new Catalogue(CatalogueRepository, "c");
@@ -104,7 +107,10 @@ public void MEFIncompatibleType()
     [Test]
     public void MEFCompatibleType_NoProjectDirectory()
     {
-        _lmd.LocationOfFlatFiles = null;
+        _lmd.LocationOfForLoadingDirectory = null;
+        _lmd.LocationOfForArchivingDirectory = null;
+        _lmd.LocationOfExecutablesDirectory = null;
+        _lmd.LocationOfCacheDirectory = null;
         _lmd.SaveToDatabase();
 
         _task.ProcessTaskType = ProcessTaskType.Attacher;
@@ -114,7 +120,7 @@ public void MEFCompatibleType_NoProjectDirectory()
         _task.CreateArgumentsForClassIfNotExists<AnySeparatorFileAttacher>();
 
         var ex = Assert.Throws<Exception>(() => _checker.Check(ThrowImmediatelyCheckNotifier.QuietPicky));
-        Assert.That(ex.InnerException.Message, Is.EqualTo($@"No Project Directory (LocationOfFlatFiles) has been configured on LoadMetadata {_lmd.Name}"));
+        Assert.That(ex.InnerException.Message, Is.EqualTo($@"No Project Directory has been configured on LoadMetadata {_lmd.Name}"));
     }
 
     [Test]
@@ -125,7 +131,10 @@ public void MEFCompatibleType_NoArgs()
                 "DelMeProjDir", true);
         try
         {
-            _lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
+            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
+            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
+            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
+            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;
@@ -152,7 +161,10 @@ public void MEFCompatibleType_Passes()
                 "DelMeProjDir", true);
         try
         {
-            _lmd.LocationOfFlatFiles = projDir.RootPath.FullName;
+            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
+            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
+            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
+            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;

Rdmp.Core.Tests/DataLoad/Engine/Integration/CrossDatabaseTypeTests/CrossDatabaseDataLoadTests.cs

@@ -292,7 +292,10 @@ public void Load(DatabaseType databaseType, TestCase testCase)
         }
         finally
         {
-            Directory.Delete(lmd.LocationOfFlatFiles, true);
+            Directory.Delete(lmd.LocationOfForLoadingDirectory, true);
+            Directory.Delete(lmd.LocationOfForArchivingDirectory, true);
+            Directory.Delete(lmd.LocationOfExecutablesDirectory, true);
+            Directory.Delete(lmd.LocationOfCacheDirectory, true);
 
             foreach (var c in RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>())
                 c.DeleteInDatabase();
@@ -461,7 +464,10 @@ public void DLELoadTwoTables(DatabaseType databaseType)
         }
         finally
         {
-            Directory.Delete(lmd.LocationOfFlatFiles, true);
+            Directory.Delete(lmd.LocationOfForLoadingDirectory, true);
+            Directory.Delete(lmd.LocationOfForArchivingDirectory, true);
+            Directory.Delete(lmd.LocationOfExecutablesDirectory, true);
+            Directory.Delete(lmd.LocationOfCacheDirectory, true);
 
             foreach (var c in RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>())
                 c.DeleteInDatabase();

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadEngineTestsBase.cs

@@ -72,7 +72,10 @@ protected static LoadDirectory SetupLoadDirectory(LoadMetadata lmd)
         var projectDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
                 "MyLoadDir", true);
-        lmd.LocationOfFlatFiles = projectDirectory.RootPath.FullName;
+        lmd.LocationOfForLoadingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForLoadingPath);
+        lmd.LocationOfForArchivingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForArchivingPath);
+        lmd.LocationOfExecutablesDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultExecutablesPath);
+        lmd.LocationOfCacheDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultCachePath);
         lmd.SaveToDatabase();
 
         return projectDirectory;

Rdmp.Core.Tests/DataLoad/Engine/Integration/HICPipelineTests.cs

@@ -72,10 +72,11 @@ public void Create(CatalogueRepository repository, DiscoveredDatabase database,
             };
             ColumnInfo.SaveToDatabase();
 
-            LoadMetadata = new LoadMetadata(repository, "HICLoadPipelineTests")
-            {
-                LocationOfFlatFiles = directory.RootPath.FullName
-            };
+            LoadMetadata = new LoadMetadata(repository, "HICLoadPipelineTests");
+            LoadMetadata.LocationOfForLoadingDirectory = Path.Combine(directory.RootPath.FullName, LoadMetadata.DefaultForLoadingPath);
+            LoadMetadata.LocationOfForArchivingDirectory = Path.Combine(directory.RootPath.FullName, LoadMetadata.DefaultForArchivingPath);
+            LoadMetadata.LocationOfExecutablesDirectory = Path.Combine(directory.RootPath.FullName, LoadMetadata.DefaultExecutablesPath);
+            LoadMetadata.LocationOfCacheDirectory = Path.Combine(directory.RootPath.FullName, LoadMetadata.DefaultCachePath);
             LoadMetadata.SaveToDatabase();
 
             Catalogue = new Catalogue(repository, "HICLoadPipelineTests")