Merge pull request #1042 from SMI/is-ident-package-use

Is ident package use

.gitignore

@@ -319,5 +319,5 @@ package-lock.json
 
 coverage/
 htmlcov/
-
+src/plugin/SmiPlugin/nuget.exe
 rdmp-cli/

PACKAGES.md

@@ -36,3 +36,4 @@
 | fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data |
 | coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io |
 | OpenCover | [GitHub](https://github.com/OpenCover/opencover) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE)  | Calculates code coverage for tests|
+| IsIdentifiable | [GitHub](https://github.com/SMI/IsIdentifiable) | [GNU](https://github.com/SMI/IsIdentifiable/blob/main/LICENSE) | Verifies that anonymisation has been successful in data extraction pipeline|

README.md

@@ -238,6 +238,7 @@ Each SmiServices dependency contains its own documentation.  Follow the link fro
 | [RDMP](https://github.com/HicServices/RDMP) |   [![Build status](https://github.com/HicServices/RDMP/workflows/Build/badge.svg)](https://github.com/HicServices/RDMP/actions?query=workflow%3ABuild) [![Coverage Status](https://coveralls.io/repos/github/HicServices/RDMP/badge.svg?branch=develop)](https://coveralls.io/github/HicServices/RDMP?branch=develop) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HicServices/RDMP.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HicServices/RDMP/alerts/) [![NuGet Badge](https://buildstats.info/nuget/HIC.RDMP.Plugin)](https://buildstats.info/nuget/HIC.RDMP.Plugin) |
 | [RDMP.Dicom](https://github.com/HicServices/RdmpDicom) | [![Build status](https://github.com/HicServices/RdmpDicom/actions/workflows/dotnet-core.yml/badge.svg)](https://github.com/HicServices/RdmpDicom/actions/workflows/dotnet-core.yml) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HicServices/RdmpDicom.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HicServices/RdmpDicom/alerts/) [![NuGet Badge](https://buildstats.info/nuget/HIC.RDMP.Dicom)](https://buildstats.info/nuget/HIC.RDMP.Dicom) |
 | [Dicom Template Builder] | [![Build Status](https://travis-ci.org/HicServices/DicomTemplateBuilder.svg?branch=master)](https://travis-ci.org/HicServices/DicomTemplateBuilder)  [![Total alerts](https://img.shields.io/lgtm/alerts/g/HicServices/DicomTemplateBuilder.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HicServices/DicomTemplateBuilder/alerts/)  |
+| [IsIdentifiable](https://github.com/SMI/IsIdentifiable) | [![.NET Core](https://github.com/SMI/IsIdentifiable/actions/workflows/dotnet-core.yml/badge.svg)](https://github.com/SMI/IsIdentifiable/actions/workflows/dotnet-core.yml) [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/IsIdentifiable.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/IsIdentifiable/alerts/) [![NuGet Badge](https://buildstats.info/nuget/IsIdentifiable)](https://www.nuget.org/packages/IsIdentifiable/)|
 
 ## 3.0 Further Considerations
 
@@ -505,7 +506,7 @@ Scalability is handled through parallel process execution (using [RabbitMQ]).  T
 [Dicom]: ./Glossary.md#dicom
 [Dicom tags]: ./Glossary.md#dicom-tags
 [IsIdentifiable]: ./src/microservices/Microservices.IsIdentifiable/README.md
-[IsIdentifiableReviewer]: ./src/applications/Applications.IsIdentifiableReviewer/README.md
+[IsIdentifiableReviewer]: https://github.com/SMI/IsIdentifiable/tree/main/Reviewer
 [DicomFileMessage]: ./src/common/Smi.Common/Messages/DicomFileMessage.cs
 [SeriesMessage]: ./src/common/Smi.Common/Messages/SeriesMessage.cs
 [ExtractionRequestMessage]: ./src/common/Smi.Common/Messages/Extraction/ExtractionRequestMessage.cs

SmiServices.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.29411.108
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.32112.339
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Applications", "Applications", "{8B943F2C-835B-484A-86D2-3F1462970605}"
 EndProject
@@ -71,8 +71,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiabl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiable.Tests", "tests\microservices\Microservices.IsIdentifiable.Tests\Microservices.IsIdentifiable.Tests.csproj", "{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Applications.IsIdentifiableReviewer", "src\applications\Applications.IsIdentifiableReviewer\Applications.IsIdentifiableReviewer.csproj", "{C2031E86-81B4-405A-A923-9B82E0CE196F}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier.Tests", "tests\microservices\Microservices.FileCopier.Tests\Microservices.FileCopier.Tests.csproj", "{D61F6BF9-E857-457C-B745-40489A8CFE65}"
@@ -97,7 +95,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Applications.ExtractImages.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.DicomAnonymiser", "src\microservices\Microservices.DicomAnonymiser\Microservices.DicomAnonymiser.csproj", "{789280D8-0998-42E6-8E81-3019C681C9FE}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microservices.DicomAnonymiser.Tests", "tests\microservices\Microservices.DicomAnonymiser.Tests\Microservices.DicomAnonymiser.Tests.csproj", "{48000929-AB84-4764-9A9B-A827CAC7BD79}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.DicomAnonymiser.Tests", "tests\microservices\Microservices.DicomAnonymiser.Tests\Microservices.DicomAnonymiser.Tests.csproj", "{48000929-AB84-4764-9A9B-A827CAC7BD79}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{2D0D9CEF-7AF1-4779-92BA-5E124233C160}"
+	ProjectSection(SolutionItems) = preProject
+		bin\smi\buildTestPackage.py = bin\smi\buildTestPackage.py
+		.github\workflows\envs.json = .github\workflows\envs.json
+		.github\workflows\main.yml = .github\workflows\main.yml
+		bin\smi\writeDatabaseStrings.py = bin\smi\writeDatabaseStrings.py
+	EndProjectSection
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -201,10 +207,6 @@ Global
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Debug|x64.Build.0 = Debug|x64
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Release|x64.ActiveCfg = Release|x64
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Release|x64.Build.0 = Release|x64
-		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.ActiveCfg = Debug|x64
-		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.Build.0 = Debug|x64
-		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.ActiveCfg = Release|x64
-		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.Build.0 = Release|x64
 		{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.ActiveCfg = Debug|x64
 		{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.Build.0 = Debug|x64
 		{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.ActiveCfg = Release|x64
@@ -282,7 +284,6 @@ Global
 		{9CC69AAC-2F7C-4F36-B35E-5007FC075155} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{1A27E9E8-F16E-43F9-927E-5FE92E2F97D8} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2} = {421CCD37-3817-4748-B184-A134E19DD75C}
-		{C2031E86-81B4-405A-A923-9B82E0CE196F} = {8B943F2C-835B-484A-86D2-3F1462970605}
 		{D4E52707-FFF7-41E6-8057-C6DB344B8CD7} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{D61F6BF9-E857-457C-B745-40489A8CFE65} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{758C7C8C-683A-42E8-A8C5-C32AED1741BE} = {421CCD37-3817-4748-B184-A134E19DD75C}
@@ -296,6 +297,7 @@ Global
 		{75E52FD9-46B2-4FC0-AE4D-72547B5DE90D} = {8B943F2C-835B-484A-86D2-3F1462970605}
 		{789280D8-0998-42E6-8E81-3019C681C9FE} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{48000929-AB84-4764-9A9B-A827CAC7BD79} = {421CCD37-3817-4748-B184-A134E19DD75C}
+		{2D0D9CEF-7AF1-4779-92BA-5E124233C160} = {A47D6857-A0E0-4604-9E11-3088F32BD378}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {11CDEA53-71E8-4A9B-BC0D-74F4EB54F740}

data/microserviceConfigs/default.yaml

@@ -222,7 +222,7 @@ ExtractImagesOptions:
         ExchangeName: 'TEST.RequestInfoExchange'
         MaxConfirmAttempts: 1
 
-IsIdentifiableOptions:
+IsIdentifiableServiceOptions:
     QueueName: 'TEST.ExtractedFileToVerifyQueue'
     QoSPrefetchCount: 1
     AutoAck: false
@@ -232,17 +232,18 @@ IsIdentifiableOptions:
     ClassifierType: 'Microservices.IsIdentifiable.Service.TesseractStanfordDicomFileClassifier'
     DataDirectory: ''
 
-    #Optional. Full connection string to the database storing the whitelist of valid entries
-    WhitelistConnectionString:
-    #Optional. The DBMS provider of the whitelist table e.g. MySql
-    WhitelistDatabaseType:
-    #Optional. The unqualified name of the whitelist table
-    WhitelistTableName:
-    #Optional. The column in WhitelistTableName which contains the whitelist elements
-    WhitelistColumn:
+IsIdentifiableBaseOptions: 
+    #Optional. Full connection string to the database storing the Allowlist of valid entries
+    AllowlistConnectionString:
+    #Optional. The DBMS provider of the Allowlist table e.g. MySql
+    AllowlistDatabaseType:
+    #Optional. The unqualified name of the Allowlist table
+    AllowlistTableName:
+    #Optional. The column in AllowlistTableName which contains the Allowlist elements
+    AllowlistColumn:
 
-    #Optional. Path to a CSV file containing a single untitled column of whitelist values
-    WhitelistCsv:
+    #Optional. Path to a CSV file containing a single untitled column of Allowlist values
+    AllowlistCsv:
 
     #Optional. Generate a report on the proportion of values failing validation (for each column)
     #ColumnReport: true

src/applications/Applications.SmiRunner/Applications.SmiRunner.csproj

@@ -28,7 +28,6 @@
     <ProjectReference Include="..\Applications.DicomDirectoryProcessor\Applications.DicomDirectoryProcessor.csproj" />
     <ProjectReference Include="..\Applications.ExtractImages\Applications.ExtractImages.csproj" />
     <ProjectReference Include="..\Applications.TriggerUpdates\Applications.TriggerUpdates.csproj" />
-    <ProjectReference Include="..\Applications.IsIdentifiableReviewer\Applications.IsIdentifiableReviewer.csproj" />
   </ItemGroup>
 
 </Project>

src/applications/Applications.SmiRunner/Program.cs

@@ -55,7 +55,6 @@ internal static int Main(string[] args)
                             // Applications
                             TriggerUpdatesVerb _ => Applications.TriggerUpdates.Program.Main(rest),
                             DicomDirectoryProcessorVerb _ => Applications.DicomDirectoryProcessor.Program.Main(rest),
-                            IsIdentifiableReviewerVerb _ => Applications.IsIdentifiableReviewer.Program.Main(rest),
                             ExtractImagesVerb _ => ExtractImages.Program.Main(rest),
                             // Microservices
                             CohortExtractorVerb _ => Microservices.CohortExtractor.Program.Main(rest),

src/common/Smi.Common/Options/GlobalOptions.cs

@@ -14,7 +14,7 @@
 using System.Reflection;
 using System.Text;
 using DatabaseType = FAnsi.DatabaseType;
-
+using IsIdentifiable.Options;
 
 namespace Smi.Common.Options
 {
@@ -64,8 +64,9 @@ public string HostProcessName
 
         public TriggerUpdatesOptions TriggerUpdatesOptions { get; set; } = new TriggerUpdatesOptions();
 
-        public IsIdentifiableOptions IsIdentifiableOptions { get; set; } = new IsIdentifiableOptions();
-        public IsIdentifiableReviewerGlobalOptions IsIdentifiableReviewerOptions { get; set; } = new IsIdentifiableReviewerGlobalOptions();
+        public IsIdentifiableServiceOptions IsIdentifiableServiceOptions { get; set; } = new IsIdentifiableServiceOptions();
+        public IsIdentifiableDicomFileOptions IsIdentifiableBaseOptions { get; set; } = new IsIdentifiableDicomFileOptions();
+
         public ExtractImagesOptions ExtractImagesOptions { get; set; } = new ExtractImagesOptions();
         public DicomAnonymiserOptions DicomAnonymiserOptions { get; set; } = new DicomAnonymiserOptions();
 
@@ -101,146 +102,16 @@ public class LoggingOptions
     }
 
     [UsedImplicitly]
-    public class IsIdentifiableOptions : ConsumerOptions
+    public class IsIdentifiableServiceOptions : ConsumerOptions
     {
         /// <summary>
         /// The full name of the classifier you want to run
         /// </summary>
         public string ClassifierType { get; set; }
 
-        /// <summary>
-        /// The root location in which subfolders must exist containing all data files required by any classifiers
-        /// (typically 1 sub-directory per classifier)
-        /// </summary>
-        public string DataDirectory { get; set; }
-
-        /// <summary>
-        /// "Optional. Full connection string to the database storing the whitelist of valid entries"
-        /// </summary>
-        public string WhitelistConnectionString { get; set; }
-
-        /// <summary>
-        /// "Optional. The DBMS provider of the whitelist table e.g. MySql"
-        /// </summary>
-        public DatabaseType? WhitelistDatabaseType { get; set; }
-
-        /// <summary>
-        /// "Optional. The unqualified name of the whitelist table"
-        /// </summary>
-        public string WhitelistTableName { get; set; }
-
-        /// <summary>
-        /// "Optional. The column in WhitelistTableName which contains the whitelist elements"
-        /// </summary>
-        public string WhitelistColumn { get; set; }
-
-        /// <summary>
-        /// "Optional. Path to a CSV file containing a single untitled column of whitelist values"
-        /// </summary>
-        public string WhitelistCsv { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a report on the proportion of values failing validation (for each column)")]
-        /// </summary>
-        public bool? ColumnReport { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a report listing every unique value failing validation (and the column the value failed in)
-        /// </summary>
-        public bool? ValuesReport { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a full failure storage report that persists Failure objects in a manner that they can be retrieved.
-        /// </summary>
-        public bool? StoreReport { get; set; }
-
-        /// <summary>
-        /// Optional - If specified reports will be generated in the given folder.  If not specified, current directory is used (unless an alternate destination option is picked)
-        /// </summary>
-        public string DestinationCsvFolder { get; set; }
-
-        /// <summary>
-        /// @"Optional - If specified, the given separator will be used instead of ,.  Includes support for \t for tab and \r\n."
-        /// </summary>
-        public string DestinationCsvSeparator { get; set; }
-
-        /// <summary>
-        /// @"Optional - If specified all tabs, newlines (\r and \n) and 2+ spaces will be stripped from the values written as output (applies to all output formats)"
-        /// </summary>
-        public bool? DestinationNoWhitespace { get; set; }
-
-        /// <summary>
-        /// "Optional. Full connection string to the database in which to store the report results"
-        /// </summary>
-        public string DestinationConnectionString { get; set; }
-
-        /// <summary>
-        /// "Optional. The DBMS provider of DestinationConnectionString e.g. MySql"
-        /// </summary>
-        public DatabaseType? DestinationDatabaseType { get; set; }
-
-        /// <summary>
-        /// "Optional. If specified postcodes will not be reported as failures"
-        /// </summary>
-        public bool? IgnorePostcodes { get; set; }
-
-        /// <summary>
-        /// "Optional. Comma separated list of columns/tags which should be ignored and not processed"
-        /// </summary>
-        public string SkipColumns { get; set; }
-
-        /// <summary>
-        /// "Optional. If set and using a 7 class NER model then DATE and TIME objects will not be considered failures."
-        /// </summary>
-        public bool? IgnoreDatesInText { get; set; }
-
-        /// <summary>
-        /// "Optional. Set to control the max size of the in-memory store of processed before the get written out to any destinations. Only makes sense for reports that don't perform any aggregation across the data"
-        /// </summary>
-        public int? MaxCacheSize { get; set; }
-
-        /// <summary>
-        /// "Optional. Filename of additional rules in yaml format."
-        /// </summary>
-        public string RulesFile { get; set; }
-
-        /// <summary>
-        ///  "Optional. Directory of additional rules in yaml format."
-        /// </summary>
-        public string RulesDirectory { get; set; }
-
-        /// <summary>
-        /// "Optional.  Maximum number of answers to cache per column."
-        /// </summary>
-        public int? MaxValidationCacheSize { get; set; }
-
-
         public ProducerOptions IsIdentifiableProducerOptions {get; set;}
-    }
-
-    [UsedImplicitly]
-    public class IsIdentifiableReviewerGlobalOptions
-    {
-        /// <summary>
-        /// Location of database connection strings file (for issuing UPDATE statements)
-        /// </summary>
-        public string TargetsFile { get; set; }
-
-        /// <summary>
-        /// File containing rules for ignoring validation errors
-        /// </summary>
-        public string IgnoreList { get; set; }
-
-        /// <summary>
-        /// File containing rules for when to issue UPDATE statements
-        /// </summary>
-        public string RedList { get; set; }
-
-        /// <summary>
-        /// Sets the user interface to use a specific color palette yaml file
-        /// </summary>
-        public string Theme { get; set; }
 
+        public string DataDirectory { get; set; }
     }
 
     [UsedImplicitly]

src/common/Smi.Common/Smi.Common.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <IsPublishable>false</IsPublishable>
   </PropertyGroup>
@@ -26,6 +26,7 @@
     <PackageReference Include="coveralls.io" Version="1.4.2" />
     <PackageReference Include="HIC.DicomTypeTranslation" Version="3.0.0" />
     <PackageReference Include="HIC.FAnsiSql" Version="2.0.3" />
+  	<PackageReference Include="IsIdentifiable" Version="0.0.3" />
     <PackageReference Include="HIC.RDMP.Plugin" Version="7.0.9" />
     <PackageReference Include="JetBrains.Annotations" Version="2021.3.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs

@@ -1,9 +1,9 @@
 using CsvHelper;
 using CsvHelper.Configuration;
+using IsIdentifiable.Reporting;
 using JetBrains.Annotations;
 using Microservices.CohortPackager.Execution.ExtractJobStorage;
 using Microservices.CohortPackager.Execution.JobProcessing.Reporting.CsvRecords;
-using Microservices.IsIdentifiable.Reporting;
 using Newtonsoft.Json;
 using NLog;
 using System;

src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs

@@ -1,6 +1,6 @@
 
+using IsIdentifiable.Reporting;
 using Microservices.CohortPackager.Execution.ExtractJobStorage;
-using Microservices.IsIdentifiable.Reporting;
 using Newtonsoft.Json;
 using Smi.Common.Messages;
 using Smi.Common.Messages.Extraction;

src/microservices/Microservices.DicomRelationalMapper/Execution/DicomRelationalMapperHost.cs

@@ -53,6 +53,11 @@ public override void Start()
 
             Type databaseNamerType = repositoryLocator.CatalogueRepository.MEF.GetType(Globals.DicomRelationalMapperOptions.DatabaseNamerType);
 
+            if(databaseNamerType == null)
+            {
+                throw new Exception($"Could not find Type '{Globals.DicomRelationalMapperOptions.DatabaseNamerType}'");
+            }
+
             string liveDatabaseName = lmd.GetDistinctLiveDatabaseServer().GetCurrentDatabase().GetRuntimeName();
 
             var instance = ObjectFactory.CreateInstance<INameDatabasesAndTablesDuringLoads>(databaseNamerType, liveDatabaseName, Globals.DicomRelationalMapperOptions.Guid);