Skip to content

Commit

Permalink
Feature/rdmp 117 multiple loads per catalogue (#1754)
Browse files Browse the repository at this point in the history
* basic db objects

* fix typo

* working db linkage

* working build

* add sql update

* update sql

* update tests

* update sql

* update create sql

* update sql

* update sql

* update class

* fix extra space

* attempt to fix tests

* remove linkage

* fix typo

* add test case

* attempt image fix

* fix up image

* add missing file

* add changelog

* interim

* interim

* tidy up code

* fix build

* icons

* rename icon file (#1755)

* working icon

* update create sql

* update patch

* add patch

* working patches

* add basic tests

* tidy up code

* add go

* attempt to fix patch

* tidy up code

* remove true equal true check

* tidy up from review

* update migration

* remove dupe file (#1774)

* fix build

* interim sql

* update patch

* fix sql patch

* update patch id

* update patch id

* fix bad merge

* tidy up code

* tidy up code

---------

Co-authored-by: James A Sutherland <j.a.sutherland@dundee.ac.uk>
  • Loading branch information
JFriel and jas88 committed Mar 27, 2024
1 parent 1d4b98a commit a8b2fb0
Show file tree
Hide file tree
Showing 39 changed files with 404 additions and 161 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Add timeout override to remote table selectors
- Extractions now remember the last used pipeline
- Allow for custom .bak file physical locations during data loads
- Add ability to have multiple data loads for a single catalogue


## [8.1.4] - 2024-02-19

Expand Down
24 changes: 9 additions & 15 deletions Rdmp.Core.Tests/CommandExecution/ExecuteCommandConfirmLogsTests.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
Expand All @@ -24,11 +24,10 @@ public void ConfirmLogs_NoEntries_Throws()
var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd.ID,
LoggingDataTask = "GGG"
};
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
lm.CreateNewLoggingTaskIfNotExists("GGG");

Expand All @@ -45,11 +44,10 @@ public void ConfirmLogs_HappyEntries_Passes(bool withinTime)
var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd.ID,
LoggingDataTask = "FFF"
};
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
lm.CreateNewLoggingTaskIfNotExists("FFF");
var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
Expand All @@ -70,11 +68,10 @@ public void ConfirmLogs_SadEntry_BecauseNeverEnded_Throws()
var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd.ID,
LoggingDataTask = "FFF"
};
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
lm.CreateNewLoggingTaskIfNotExists("FFF");

Expand All @@ -93,11 +90,10 @@ public void ConfirmLogs_SadEntryWithEx_Throws()
var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd.ID,
LoggingDataTask = "FFF"
};
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
lm.CreateNewLoggingTaskIfNotExists("FFF");
var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
Expand All @@ -116,11 +112,10 @@ public void ConfirmLogs_NotWithinTime_Throws()
var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd.ID,
LoggingDataTask = "FFF"
};
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
lm.CreateNewLoggingTaskIfNotExists("FFF");
var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
Expand All @@ -144,19 +139,18 @@ public void ConfirmLogs_With2CacheProgress_Throws()
var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd1.ID,
LoggingDataTask = "B"
};
cata.SaveToDatabase();

lmd1.LinkToCatalogue(cata);
var lmd2 = new LoadMetadata(CatalogueRepository, "MyLmd");
var cata2 = new Catalogue(CatalogueRepository, "myCata")
{
LoadMetadata_ID = lmd2.ID,
LoggingDataTask = "A"
};
cata2.SaveToDatabase();

var linkage2 = new LoadMetadataCatalogueLinkage(CatalogueRepository, lmd2, cata2);
linkage2.SaveToDatabase();
var lp1 = new LoadProgress(CatalogueRepository, lmd1);
var lp2 = new LoadProgress(CatalogueRepository, lmd2);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ public void TestExecuteCommandAssociateCatalogueWithLoadMetadata_Simple()

Assert.Multiple(() =>
{
Assert.That(cata1.LoadMetadata, Is.Null);
Assert.That(cata2.LoadMetadata, Is.Null);
Assert.That(cata1.LoadMetadatas(), Is.Empty);
Assert.That(cata2.LoadMetadatas(), Is.Empty);
});

var lmd = new LoadMetadata(RepositoryLocator.CatalogueRepository, "mylmd");
Expand All @@ -36,8 +36,8 @@ public void TestExecuteCommandAssociateCatalogueWithLoadMetadata_Simple()

Assert.Multiple(() =>
{
Assert.That(cata1.LoadMetadata_ID, Is.EqualTo(lmd.ID));
Assert.That(cata2.LoadMetadata, Is.Null);
Assert.That(cata1.LoadMetadatas()[0].ID, Is.EqualTo(lmd.ID));
Assert.That(cata2.LoadMetadatas(), Is.Empty);
});
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
Expand Down Expand Up @@ -56,10 +56,12 @@ protected override void SetUp()

_cata = new Catalogue(CatalogueRepository, "EndToEndCacheTest")
{
LoadMetadata_ID = _lmd.ID
};
_cata.SaveToDatabase();

var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, _lmd, _cata);
linkage.SaveToDatabase();

_lp = new LoadProgress(CatalogueRepository, _lmd);
_cp = new CacheProgress(CatalogueRepository, _lp);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
Expand Down Expand Up @@ -52,9 +52,8 @@ public void TestDle_DodgyColumnNames(DatabaseType dbType)

CreateFlatFileAttacher(lmd, "Troll.csv", cata.GetTableInfoList(false).Single());

cata.LoadMetadata_ID = lmd.ID;
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
Assert.That(tbl.GetRowCount(), Is.EqualTo(0));

RunDLE(lmd, 30000, true);
Expand Down
11 changes: 5 additions & 6 deletions Rdmp.Core.Tests/Curation/ImportTests/GatherAndShareTests.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
Expand Down Expand Up @@ -175,9 +175,9 @@ public void GatherAndShare_Catalogue_Test(bool goViaJson)
Assert.That(gObj.Children, Has.Count.EqualTo(2)); //both cata items

var lmd = new LoadMetadata(CatalogueRepository);
cata.LoadMetadata_ID = lmd.ID;
cata.SaveToDatabase();

var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, lmd, cata);
linkage.SaveToDatabase();
//get the share definition
var shareManager = new ShareManager(RepositoryLocator);
var shareDefinition = gObj.ToShareDefinitionWithChildren(shareManager);
Expand All @@ -197,10 +197,9 @@ public void GatherAndShare_Catalogue_Test(bool goViaJson)
//make a local change
cata.Name = "fishfish";
cata.SubjectNumbers = "123";
cata.LoadMetadata_ID = null;
cata.Periodicity = Catalogue.CataloguePeriodicity.Unknown;
cata.SaveToDatabase();

lmd.UnlinkFromCatalogue(cata);
lmd.DeleteInDatabase();

//import the saved copy
Expand Down Expand Up @@ -229,7 +228,7 @@ public void GatherAndShare_Catalogue_Test(bool goViaJson)
//import the defined properties but not name
Assert.That(cata.Name, Is.EqualTo("fishfish"));
Assert.That(cata.Periodicity, Is.EqualTo(Catalogue.CataloguePeriodicity.BiMonthly)); //reset this though
Assert.That(cata.LoadMetadata_ID, Is.Null);
Assert.That(cata.LoadMetadatas(), Is.Empty);
});
cata.SaveToDatabase();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ public void GatherAndShare_LoadMetadata_EmptyLoadMetadata()
public void GatherAndShare_LoadMetadata_WithCatalogue()
{
//create an object
LoadMetadata lmd1;
var lmd2 = ShareToNewRepository(lmd1 = WhenIHaveA<LoadMetadata>());
LoadMetadata lmd1 = WhenIHaveA<LoadMetadata>();
var lmd2 = ShareToNewRepository(lmd1);

var cata1 = lmd1.GetAllCatalogues().Single();
var cata2 = lmd2.GetAllCatalogues().Single();
Expand Down
70 changes: 70 additions & 0 deletions Rdmp.Core.Tests/Curation/Integration/LoadMetadataLinkageTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// Copyright (c) The University of Dundee 2024-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
using NUnit.Framework;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using System.Linq;
using Tests.Common;

namespace Rdmp.Core.Tests.Curation.Integration;

public class LoadMetadataLinkageTests: DatabaseTests
{
[Test]
public void BasicLoadMetadataLinkage()
{
var loadMetadata = new LoadMetadata(CatalogueRepository);
var catalogue = new Catalogue(CatalogueRepository, "catalogue");
loadMetadata.LinkToCatalogue(catalogue);
Assert.That(catalogue.LoadMetadatas().Length, Is.EqualTo(1));
Assert.That(catalogue.LoadMetadatas().First(), Is.EqualTo(loadMetadata));
Assert.That(loadMetadata.GetAllCatalogues().ToArray().Length, Is.EqualTo(1));
}

[Test]
public void BasicLoadMetadataLinkageUnlink()
{
var loadMetadata = new LoadMetadata(CatalogueRepository);
var catalogue = new Catalogue(CatalogueRepository, "catalogue");
loadMetadata.LinkToCatalogue(catalogue);
Assert.That(catalogue.LoadMetadatas().Length, Is.EqualTo(1));
Assert.That(catalogue.LoadMetadatas().First(), Is.EqualTo(loadMetadata));
loadMetadata.UnlinkFromCatalogue(catalogue);
Assert.That(catalogue.LoadMetadatas().Length, Is.EqualTo(0));
Assert.That(loadMetadata.GetAllCatalogues().ToArray().Length , Is.EqualTo(0));
}

[Test]
public void BasicLoadMetadataLinkage_Multiple()
{
var loadMetadata = new LoadMetadata(CatalogueRepository);
var catalogue = new Catalogue(CatalogueRepository, "catalogue");
var catalogue2 = new Catalogue(CatalogueRepository, "catalogue2");
loadMetadata.LinkToCatalogue(catalogue);
loadMetadata.LinkToCatalogue(catalogue2);
Assert.That(catalogue.LoadMetadatas().Length, Is.EqualTo(1));
Assert.That(catalogue.LoadMetadatas().First(), Is.EqualTo(loadMetadata));
Assert.That(catalogue2.LoadMetadatas().Length, Is.EqualTo(1));
Assert.That(catalogue2.LoadMetadatas().First(), Is.EqualTo(loadMetadata));
Assert.That(loadMetadata.GetAllCatalogues().ToArray().Length, Is.EqualTo(2));
}

[Test]
public void BasicLoadMetadataLinkage_MultipleUnlink()
{
var loadMetadata = new LoadMetadata(CatalogueRepository);
var catalogue = new Catalogue(CatalogueRepository, "catalogue");
var catalogue2 = new Catalogue(CatalogueRepository, "catalogue2");
loadMetadata.LinkToCatalogue(catalogue);
loadMetadata.LinkToCatalogue(catalogue2);
loadMetadata.UnlinkFromCatalogue(catalogue2);
Assert.That(catalogue.LoadMetadatas().Length, Is.EqualTo(1));
Assert.That(catalogue.LoadMetadatas().First(), Is.EqualTo(loadMetadata));
Assert.That(catalogue2.LoadMetadatas().Length, Is.EqualTo(0));
Assert.That(catalogue2.LoadMetadatas(), Is.Empty);
Assert.That(loadMetadata.GetAllCatalogues().ToArray().Length, Is.EqualTo(1));
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) The University of Dundee 2018-2019
// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
Expand All @@ -23,8 +23,7 @@ public void Test_NoLoggingTask()
var lmd = WhenIHaveA<LoadMetadata>();
var cata1 = lmd.GetAllCatalogues().Single();
var cata2 = WhenIHaveA<Catalogue>();
cata2.LoadMetadata_ID = lmd.ID;

lmd.LinkToCatalogue(cata2);
Assert.That(lmd.GetAllCatalogues().Count(), Is.EqualTo(2));

var checks = new MetadataLoggingConfigurationChecks(lmd);
Expand All @@ -41,8 +40,7 @@ public void Test_MismatchedLoggingTask()
var lmd = WhenIHaveA<LoadMetadata>();
var cata1 = lmd.GetAllCatalogues().Single();
var cata2 = WhenIHaveA<Catalogue>();
cata2.LoadMetadata_ID = lmd.ID;

lmd.LinkToCatalogue(cata2);
cata1.LoggingDataTask = "OMG YEAGH";

Assert.That(lmd.GetAllCatalogues().Count(), Is.EqualTo(2));
Expand All @@ -60,8 +58,7 @@ public void Test_MissingLoggingServer()
var lmd = WhenIHaveA<LoadMetadata>();
var cata1 = lmd.GetAllCatalogues().Single();
var cata2 = WhenIHaveA<Catalogue>();
cata2.LoadMetadata_ID = lmd.ID;

lmd.LinkToCatalogue(cata2);
cata1.LoggingDataTask = "OMG YEAGH";
cata1.LiveLoggingServer_ID = 2;
cata2.LoggingDataTask = "OMG YEAGH";
Expand All @@ -87,8 +84,7 @@ public void Test_MissingLoggingServer_UseDefault()
eds.Name = "My Logging Server";
eds.SaveToDatabase();

cata2.LoadMetadata_ID = lmd.ID;

lmd.LinkToCatalogue(cata2);
cata1.LoggingDataTask = "OMG YEAGH";
cata1.LiveLoggingServer_ID = null;
cata2.LoggingDataTask = "OMG YEAGH";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@ public void CreateTask()
t.SaveToDatabase();
var col = new ColumnInfo(CatalogueRepository, "col", "bit", t);
ci.SetColumnInfo(col);
c.LoadMetadata_ID = _lmd.ID;
c.SaveToDatabase();

_lmd.LinkToCatalogue(c);
_task = new ProcessTask(CatalogueRepository, _lmd, LoadStage.GetFiles);
_checker = new ProcessTaskChecks(_lmd);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,8 @@ public void TestMerge(DatabaseType databaseType)
new FixedStagingDatabaseNamer(toTbl.Database.GetRuntimeName(), fromTbl.Database.GetRuntimeName()));

var lmd = new LoadMetadata(CatalogueRepository);
cata.LoadMetadata_ID = lmd.ID;
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
var migrationHost = new MigrationHost(From, To, configuration, new HICDatabaseConfiguration(lmd));

//set SetUp a logging task
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,10 @@ protected ITableInfo Import(DiscoveredTable tbl, LoadMetadata lmd, LogManager lo
forwardEngineer.ExecuteForwardEngineering(out var cata, out var cataItems, out var eis);

//make the catalogue use the load configuration
cata.LoadMetadata_ID = lmd.ID;
cata.LoggingDataTask = lmd.Name;
Assert.That(cata.LiveLoggingServer_ID, Is.Not.Null); //catalogue should have one of these because of system defaults
cata.SaveToDatabase();

lmd.LinkToCatalogue(cata);
return ti;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,9 @@ public CatalogueEntities()
Catalogue = new Catalogue(repository, "HICLoadPipelineTests")
{
LoggingDataTask = "Test",
LoadMetadata_ID = LoadMetadata.ID
};
Catalogue.SaveToDatabase();

LoadMetadata.LinkToCatalogue(Catalogue);
var catalogueItem = new CatalogueItem(repository, Catalogue, "Test");
catalogueItem.SetColumnInfo(ColumnInfo);

Expand Down
3 changes: 1 addition & 2 deletions Rdmp.Core.Tests/DataLoad/Engine/Integration/PayloadTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,9 @@ public void TestPayloadInjection()

MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));

b.catalogue.LoadMetadata_ID = lmd.ID;
b.catalogue.LoggingDataTask = "TestPayloadInjection";
b.catalogue.SaveToDatabase();

lmd.LinkToCatalogue(b.catalogue);
var lm = new LogManager(CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");

Expand Down
Loading

0 comments on commit a8b2fb0

Please sign in to comment.