Refactored IsIdentifiable to draw from nuget package and deleted loca…

…l code

Building but not passing tests, still need to rework default.yaml and cli options for the service

src/common/Smi.Common/Options/GlobalOptions.cs

@@ -14,7 +14,7 @@
 using System.Reflection;
 using System.Text;
 using DatabaseType = FAnsi.DatabaseType;
-
+using IsIdentifiable.Options;
 
 namespace Smi.Common.Options
 {
@@ -64,8 +64,9 @@ public string HostProcessName
 
         public TriggerUpdatesOptions TriggerUpdatesOptions { get; set; } = new TriggerUpdatesOptions();
 
-        public IsIdentifiableOptions IsIdentifiableOptions { get; set; } = new IsIdentifiableOptions();
-        public IsIdentifiableReviewerGlobalOptions IsIdentifiableReviewerOptions { get; set; } = new IsIdentifiableReviewerGlobalOptions();
+        public IsIdentifiableServiceOptions IsIdentifiableServiceOptions { get; set; } = new IsIdentifiableServiceOptions();
+        public IsIdentifiableDicomFileOptions IsIdentifiableBaseOptions { get; set; } = new IsIdentifiableDicomFileOptions();
+
         public ExtractImagesOptions ExtractImagesOptions { get; set; } = new ExtractImagesOptions();
         public DicomAnonymiserOptions DicomAnonymiserOptions { get; set; } = new DicomAnonymiserOptions();
 
@@ -101,146 +102,16 @@ public class LoggingOptions
     }
 
     [UsedImplicitly]
-    public class IsIdentifiableOptions : ConsumerOptions
+    public class IsIdentifiableServiceOptions : ConsumerOptions
     {
         /// <summary>
         /// The full name of the classifier you want to run
         /// </summary>
         public string ClassifierType { get; set; }
 
-        /// <summary>
-        /// The root location in which subfolders must exist containing all data files required by any classifiers
-        /// (typically 1 sub-directory per classifier)
-        /// </summary>
-        public string DataDirectory { get; set; }
-
-        /// <summary>
-        /// "Optional. Full connection string to the database storing the whitelist of valid entries"
-        /// </summary>
-        public string WhitelistConnectionString { get; set; }
-
-        /// <summary>
-        /// "Optional. The DBMS provider of the whitelist table e.g. MySql"
-        /// </summary>
-        public DatabaseType? WhitelistDatabaseType { get; set; }
-
-        /// <summary>
-        /// "Optional. The unqualified name of the whitelist table"
-        /// </summary>
-        public string WhitelistTableName { get; set; }
-
-        /// <summary>
-        /// "Optional. The column in WhitelistTableName which contains the whitelist elements"
-        /// </summary>
-        public string WhitelistColumn { get; set; }
-
-        /// <summary>
-        /// "Optional. Path to a CSV file containing a single untitled column of whitelist values"
-        /// </summary>
-        public string WhitelistCsv { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a report on the proportion of values failing validation (for each column)")]
-        /// </summary>
-        public bool? ColumnReport { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a report listing every unique value failing validation (and the column the value failed in)
-        /// </summary>
-        public bool? ValuesReport { get; set; }
-
-        /// <summary>
-        /// Optional. Generate a full failure storage report that persists Failure objects in a manner that they can be retrieved.
-        /// </summary>
-        public bool? StoreReport { get; set; }
-
-        /// <summary>
-        /// Optional - If specified reports will be generated in the given folder.  If not specified, current directory is used (unless an alternate destination option is picked)
-        /// </summary>
-        public string DestinationCsvFolder { get; set; }
-
-        /// <summary>
-        /// @"Optional - If specified, the given separator will be used instead of ,.  Includes support for \t for tab and \r\n."
-        /// </summary>
-        public string DestinationCsvSeparator { get; set; }
-
-        /// <summary>
-        /// @"Optional - If specified all tabs, newlines (\r and \n) and 2+ spaces will be stripped from the values written as output (applies to all output formats)"
-        /// </summary>
-        public bool? DestinationNoWhitespace { get; set; }
-
-        /// <summary>
-        /// "Optional. Full connection string to the database in which to store the report results"
-        /// </summary>
-        public string DestinationConnectionString { get; set; }
-
-        /// <summary>
-        /// "Optional. The DBMS provider of DestinationConnectionString e.g. MySql"
-        /// </summary>
-        public DatabaseType? DestinationDatabaseType { get; set; }
-
-        /// <summary>
-        /// "Optional. If specified postcodes will not be reported as failures"
-        /// </summary>
-        public bool? IgnorePostcodes { get; set; }
-
-        /// <summary>
-        /// "Optional. Comma separated list of columns/tags which should be ignored and not processed"
-        /// </summary>
-        public string SkipColumns { get; set; }
-
-        /// <summary>
-        /// "Optional. If set and using a 7 class NER model then DATE and TIME objects will not be considered failures."
-        /// </summary>
-        public bool? IgnoreDatesInText { get; set; }
-
-        /// <summary>
-        /// "Optional. Set to control the max size of the in-memory store of processed before the get written out to any destinations. Only makes sense for reports that don't perform any aggregation across the data"
-        /// </summary>
-        public int? MaxCacheSize { get; set; }
-
-        /// <summary>
-        /// "Optional. Filename of additional rules in yaml format."
-        /// </summary>
-        public string RulesFile { get; set; }
-
-        /// <summary>
-        ///  "Optional. Directory of additional rules in yaml format."
-        /// </summary>
-        public string RulesDirectory { get; set; }
-
-        /// <summary>
-        /// "Optional.  Maximum number of answers to cache per column."
-        /// </summary>
-        public int? MaxValidationCacheSize { get; set; }
-
-
         public ProducerOptions IsIdentifiableProducerOptions {get; set;}
-    }
-
-    [UsedImplicitly]
-    public class IsIdentifiableReviewerGlobalOptions
-    {
-        /// <summary>
-        /// Location of database connection strings file (for issuing UPDATE statements)
-        /// </summary>
-        public string TargetsFile { get; set; }
-
-        /// <summary>
-        /// File containing rules for ignoring validation errors
-        /// </summary>
-        public string IgnoreList { get; set; }
-
-        /// <summary>
-        /// File containing rules for when to issue UPDATE statements
-        /// </summary>
-        public string RedList { get; set; }
-
-        /// <summary>
-        /// Sets the user interface to use a specific color palette yaml file
-        /// </summary>
-        public string Theme { get; set; }
 
+        public string DataDirectory { get; set; }
     }
 
     [UsedImplicitly]

src/common/Smi.Common/Smi.Common.csproj

@@ -22,11 +22,12 @@
     <PackageReference Include="CsvHelper" Version="27.2.1" />
     <PackageReference Include="CommandLineParser" Version="2.8.0" />
     <PackageReference Include="Equ" Version="2.3.0" />
-    <PackageReference Include="fo-dicom.NetCore" Version="[4.0.7]" />
+    <PackageReference Include="fo-dicom.NetCore" Version="[4.0.8]" />
     <PackageReference Include="coveralls.io" Version="1.4.2" />
     <PackageReference Include="HIC.DicomTypeTranslation" Version="3.0.0" />
     <PackageReference Include="HIC.FAnsiSql" Version="2.0.2" />
     <PackageReference Include="HIC.RDMP.Plugin" Version="7.0.3" />
+    <PackageReference Include="IsIdentifiable" Version="0.0.1" />
     <PackageReference Include="JetBrains.Annotations" Version="2021.3.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="NLog" Version="4.7.13" />
@@ -35,7 +36,6 @@
         <PrivateAssets>all</PrivateAssets>
         <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="System.IO.Abstractions" Version="16.1.9" />
     <PackageReference Include="YamlDotNet" Version="11.2.1" />
   </ItemGroup>
 </Project>

src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs

@@ -1,9 +1,9 @@
 using CsvHelper;
 using CsvHelper.Configuration;
+using IsIdentifiable.Reporting;
 using JetBrains.Annotations;
 using Microservices.CohortPackager.Execution.ExtractJobStorage;
 using Microservices.CohortPackager.Execution.JobProcessing.Reporting.CsvRecords;
-using Microservices.IsIdentifiable.Reporting;
 using Newtonsoft.Json;
 using NLog;
 using System;

src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs

@@ -1,6 +1,6 @@
 
+using IsIdentifiable.Reporting;
 using Microservices.CohortPackager.Execution.ExtractJobStorage;
-using Microservices.IsIdentifiable.Reporting;
 using Newtonsoft.Json;
 using Smi.Common.Messages;
 using Smi.Common.Messages.Extraction;

src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj

@@ -8,7 +8,6 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="StackExchange.Redis" Version="2.2.88" />
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\common\Smi.Common\Smi.Common.csproj" />