Added -y option to ii

Fixes  #161

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 # [Unreleased]
 
+## Added
+
+- Added new command line flag `-y somefile.yaml` in `ii` CLI tool to specify a custom config file
+
 # [0.0.5] - 2022-07-20
 
 ## Dependencies

Tests/IsIdentifiableTests/IsIdentifiable.Tests.csproj

@@ -30,6 +30,9 @@
     <ProjectReference Include="..\..\IsIdentifiable\IsIdentifiable.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <None Update="default.yaml">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="pixeldatareport.csv">
       <Generator>TextTemplatingFileGenerator</Generator>
       <LastGenOutput>pixeldatareport.cs</LastGenOutput>

Tests/IsIdentifiableTests/ReviewerTests/SettingsFileTests.cs

@@ -0,0 +1,87 @@
+using NUnit.Framework;
+using System.IO;
+
+namespace IsIdentifiable.Tests.ReviewerTests
+{
+    public class SettingsFileTests 
+    {
+        [Test]
+        public void TestCutSettingsFileArgs_NoArgs()
+        {
+            Assert.IsNull(Program.CutSettingsFileArgs(new string[0],out var result));
+            Assert.IsEmpty(result);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_NoYamlFile()
+        {
+            Assert.IsNull(Program.CutSettingsFileArgs(new [] { "review" ,"somefish"}, out var result));
+            Assert.AreEqual(2, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("somefish", result[1]);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_DashYOnly()
+        {
+            // missing argument, let CommandLineParser sort them out
+            Assert.IsNull(Program.CutSettingsFileArgs(new[] { "-y" }, out var result));
+            Assert.AreEqual(1, result.Length);
+            Assert.AreEqual("-y", result[0]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlOnly()
+        {
+            Assert.AreEqual("myfile.yaml",Program.CutSettingsFileArgs(new[] { "-y" ,"myfile.yaml"}, out var result));
+            Assert.IsEmpty(result);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlAfter()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] {"review", "db","someconstr" , "-y", "myfile.yaml" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlBefore()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] { "-y", "myfile.yaml", "review", "db", "someconstr" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlInMiddle()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] { "review", "-y", "myfile.yaml", "db", "someconstr" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestDeserialize_SmiServices_DefaultYaml()
+        {
+            var f = Path.Combine(TestContext.CurrentContext.TestDirectory,"default.yaml");
+            FileAssert.Exists(f);
+
+            var opts = Program.Deserialize(f);
+            Assert.IsNotNull(opts.IsIdentifiableReviewerOptions);
+            Assert.IsNotNull(opts.IsIdentifiableOptions);
+        }
+    }
+}

Tests/IsIdentifiableTests/default.yaml

@@ -0,0 +1,281 @@
+
+RabbitOptions:
+    RabbitMqHostName: 'localhost'
+    RabbitMqHostPort: 5672
+    RabbitMqVirtualHost: '/'
+    RabbitMqUserName: 'guest'
+    RabbitMqPassword: 'guest'
+    RabbitMqControlExchangeName: 'TEST.ControlExchange'
+    FatalLoggingExchange: 'TEST.FatalLoggingExchange'
+
+LoggingOptions:  
+    LogConfigFile: ''
+    LogsRoot: ''
+    TraceLogging: true
+
+FileSystemOptions:
+    FileSystemRoot: 'C:\temp'
+    ExtractRoot: 'C:\temp'
+    # Set this to "*" if your dicom files do not have file extensions
+    DicomSearchPattern: "*.dcm"
+
+RDMPOptions:
+    CatalogueConnectionString: 'server=localhost\sqlexpress;integrated security=true;database=RDMP_Catalogue'
+    DataExportConnectionString: 'server=localhost\sqlexpress;integrated security=true;database=RDMP_DataExport'
+
+    # Alternative to connection strings for if you have RDMP running with a YAML file system backend.  If specified then this will override the connection strings        
+    YamlDir: 
+MongoDatabases:
+    DicomStoreOptions:
+        HostName: 'localhost'
+        Port: 27017
+        UserName: ''
+        Password: ''
+        DatabaseName: 'dicom'
+    ExtractionStoreOptions:
+        HostName: 'localhost'
+        Port: 27017
+        UserName: ''
+        Password: ''
+        DatabaseName: 'extraction'
+
+DicomRelationalMapperOptions:
+    Guid: '6ff062af-5538-473f-801c-ed2b751c7897'
+    QueueName: 'TEST.AnonymousImageQueue'
+    QoSPrefetchCount: 10000
+    AutoAck: false
+    LoadMetadataId: 1
+    DatabaseNamerType: 'GuidDatabaseNamer'
+    MinimumBatchSize: 10000
+    UseInsertIntoForRAWMigration: true
+    RetryOnFailureCount: 1
+    RetryDelayInSeconds: 60
+    RunChecks: true
+
+UpdateValuesOptions:
+    QueueName: 'TEST.UpdateValuesQueue'
+    QoSPrefetchCount: 10000
+    AutoAck: false
+    #TableInfosToUpdate: 1,2,3 <-set this to limit which tables get updated when no explicit table is listed in the message
+    UpdateTimeout: 500 # number of seconds to wait for each UPDATE statement
+
+TriggerUpdatesOptions:
+    ExchangeName: 'TEST.TriggerUpdatesExchange'
+    MaxConfirmAttempts: 1
+
+CohortExtractorOptions:
+    QueueName: 'TEST.RequestQueue'
+    QoSPrefetchCount: 10000
+    AutoAck: false
+    AllCatalogues: true
+
+    # List of IDs of Catalogues to extract from (in ascending order).
+    # Ignored if "AllCatalogues == true"
+    OnlyCatalogues: [1,2,3]
+
+    # ID(s) of ColumnInfo that contains a list of values which should not have data extracted for them.  e.g. opt out.  The name of the column referenced must match a column in the extraction table
+    #RejectColumnInfos: [105,110]
+
+    AuditorType: 'Microservices.CohortExtractor.Audit.NullAuditExtractions'
+    RequestFulfillerType: 'Microservices.CohortExtractor.Execution.RequestFulfillers.FromCataloguesExtractionRequestFulfiller'
+    ProjectPathResolverType: 'Microservices.CohortExtractor.Execution.ProjectPathResolvers.DefaultProjectPathResolver'
+    ExtractAnonRoutingKey: anon
+    ExtractIdentRoutingKey: ident
+    # Writes (Producer) to this exchange
+    ExtractFilesProducerOptions: 
+        ExchangeName: 'TEST.ExtractFileExchange'
+        MaxConfirmAttempts: 1
+    # And audits this too
+    ExtractFilesInfoProducerOptions: 
+        ExchangeName: 'TEST.FileCollectionInfoExchange'
+        MaxConfirmAttempts: 1
+
+CohortPackagerOptions:
+    JobWatcherTimeoutInSeconds: 30
+    ReporterType: 'LoggingReporter'
+    NotifierType: 'LoggingNotifier'
+    ReportFormat: 'Combined'
+    ReportNewLine: ''
+    ExtractRequestInfoOptions:
+        QueueName: 'TEST.RequestInfoQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+    FileCollectionInfoOptions:
+        QueueName: 'TEST.FileCollectionInfoQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+    NoVerifyStatusOptions:
+        QueueName: 'TEST.ExtractedFileNoVerifyQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+    VerificationStatusOptions:
+        QueueName: 'TEST.ExtractedFileVerifiedQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+
+DicomReprocessorOptions:
+    ProcessingMode: 'ImageReprocessing'
+    ReprocessingProducerOptions: 
+        ExchangeName: 'TEST.IdentifiableImageExchange'
+        MaxConfirmAttempts: 1
+
+DicomTagReaderOptions:
+    QueueName: 'TEST.AccessionDirectoryQueue'
+    QoSPrefetchCount: 1
+    AutoAck: false
+    NackIfAnyFileErrors: true
+    ImageProducerOptions: 
+        ExchangeName: 'TEST.IdentifiableImageExchange'
+        MaxConfirmAttempts: 1
+    SeriesProducerOptions: 
+        ExchangeName: 'TEST.IdentifiableSeriesExchange'
+        MaxConfirmAttempts: 1
+    FileReadOption: 'ReadLargeOnDemand'
+    TagProcessorMode: 'Serial'
+    MaxIoThreads: 1
+
+IdentifierMapperOptions:
+    QueueName: 'TEST.IdentifiableImageQueue'
+    QoSPrefetchCount: 1000
+    AutoAck: false
+    AnonImagesProducerOptions: 
+        ExchangeName: 'TEST.AnonymousImageExchange'
+        MaxConfirmAttempts: 1
+    MappingConnectionString: 'Server=localhost\sqlexpress;Integrated Security=true;Initial Catalog=MappingDatabase;'
+    MappingDatabaseType: 'MicrosoftSQLServer'
+    MappingTableName: 'MappingTable'
+    TimeoutInSeconds: 600
+    SwapColumnName: 'CHI'
+    ReplacementColumnName: 'ECHI'
+    SwapperType: 'Microservices.IdentifierMapper.Execution.Swappers.ForGuidIdentifierSwapper'
+    AllowRegexMatching: false
+    RedisConnectionString: ''
+
+MongoDbPopulatorOptions:
+    SeriesQueueConsumerOptions:
+        QueueName: 'TEST.MongoSeriesQueue'
+        QoSPrefetchCount: 1000
+        AutoAck: false
+    ImageQueueConsumerOptions:
+        QueueName: 'TEST.MongoImageQueue'
+        QoSPrefetchCount: 10000
+        AutoAck: false
+    MongoDbFlushTime: 30 # Seconds
+    FailedWriteLimit: 5
+
+ProcessDirectoryOptions:
+    AccessionDirectoryProducerOptions:
+        ExchangeName: 'TEST.AccessionDirectoryExchange'
+        MaxConfirmAttempts: 1
+
+CTPAnonymiserOptions:
+    VerifyRoutingKey: verify
+    NoVerifyRoutingKey: noverify
+    AnonFileConsumerOptions:
+        QueueName: 'TEST.ExtractFileAnonQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+    ExtractFileStatusProducerOptions:
+        ExchangeName: 'TEST.ExtractedFileStatusExchange'
+        MaxConfirmAttempts: 1
+
+DicomAnonymiserOptions:
+    AnonymiserType:
+    AnonFileConsumerOptions:
+        QueueName: 'TEST.ExtractFileAnonQueue'
+        QoSPrefetchCount: 1
+        AutoAck: false
+    ExtractFileStatusProducerOptions:
+        ExchangeName: 'TEST.ExtractedFileStatusExchange'
+        MaxConfirmAttempts: 1
+    RoutingKeySuccess: verify
+    RoutingKeyFailure: noverify
+    FailIfSourceWriteable: true
+
+FileCopierOptions:
+    NoVerifyRoutingKey: noverify
+    QueueName: 'TEST.ExtractFileIdentQueue'
+    QoSPrefetchCount: 1
+    AutoAck: false
+    CopyStatusProducerOptions:
+        ExchangeName: 'TEST.ExtractedFileStatusExchange'
+        MaxConfirmAttempts: 1
+
+ExtractImagesOptions:
+    MaxIdentifiersPerMessage: 1000
+    ExtractionRequestProducerOptions:
+        ExchangeName: 'TEST.RequestExchange'
+        MaxConfirmAttempts: 1
+    ExtractionRequestInfoProducerOptions:
+        ExchangeName: 'TEST.RequestInfoExchange'
+        MaxConfirmAttempts: 1
+
+IsIdentifiableServiceOptions:
+    QueueName: 'TEST.ExtractedFileToVerifyQueue'
+    QoSPrefetchCount: 1
+    AutoAck: false
+    IsIdentifiableProducerOptions:
+        ExchangeName: 'TEST.ExtractedFileVerifiedExchange'
+        MaxConfirmAttempts: 1
+    ClassifierType: 'Microservices.IsIdentifiable.Service.TesseractStanfordDicomFileClassifier'
+    DataDirectory: ''
+
+IsIdentifiableOptions: 
+    #Optional. Full connection string to the database storing the Allowlist of valid entries
+    AllowlistConnectionString:
+    #Optional. The DBMS provider of the Allowlist table e.g. MySql
+    AllowlistDatabaseType:
+    #Optional. The unqualified name of the Allowlist table
+    AllowlistTableName:
+    #Optional. The column in AllowlistTableName which contains the Allowlist elements
+    AllowlistColumn:
+
+    #Optional. Path to a CSV file containing a single untitled column of Allowlist values
+    AllowlistCsv:
+
+    #Optional. Generate a report on the proportion of values failing validation (for each column)
+    #ColumnReport: true
+
+    #Optional. Generate a report listing every unique value failing validation (and the column the value failed in)
+    #ValuesReport: true
+
+    #Optional. Generate a full failure storage report that persists Failure objects in a manner that they can be retrieved.
+    #StoreReport: true
+
+    #Optional - If specified reports will be generated in the given folder.  If not specified, current directory is used (unless an alternate destination option is picked)
+    DestinationCsvFolder:
+    #Optional - If specified, the given separator will be used instead of ,.  Includes support for \t for tab and \r\n
+    DestinationCsvSeparator:
+    #Optional - If specified all tabs, newlines (\r and \n) and 2+ spaces will be stripped from the values written as output (applies to all output formats)
+    DestinationNoWhitespace:
+
+    #Optional. Full connection string to the database in which to store the report results
+    DestinationConnectionString:
+    #Optional. The DBMS provider of DestinationConnectionString e.g. MySql
+    DestinationDatabaseType:
+
+    #Optional. If specified postcodes will not be reported as failures
+    IgnorePostcodes: false
+    #Optional. Comma separated list of columns/tags which should be ignored and not processed
+    SkipColumns:
+    #Optional. If set and using a 7 class NER model then DATE and TIME objects will not be considered failures.
+    IgnoreDatesInText:
+    #Optional. Set to control the max size of the in-memory store of processed before the get written out to any destinations. Only makes sense for reports that don't perform any aggregation across the data
+    MaxCacheSize:
+
+    #Optional. Filename of additional rules in yaml format.
+    RulesFile:
+    #Optional. Directory of additional rules in yaml format.
+    RulesDirectory:
+    #Optional.  Maximum number of answers to cache per column.
+    MaxValidationCacheSize:
+
+IsIdentifiableReviewerOptions:
+    #Location of database connection strings file (for issuing UPDATE statements)    
+    TargetsFile:
+    #File containing rules for ignoring validation errors
+    IgnoreList:
+    #File containing rules for when to issue UPDATE statements
+    RedList:
+    #Sets the user interface to use a specific color palette yaml file
+    Theme:

ii/GlobalOptions.cs

@@ -2,7 +2,7 @@
 
 namespace IsIdentifiable;
 
-internal class GlobalOptions
+public class GlobalOptions
 {
     public IsIdentifiableBaseOptions? IsIdentifiableOptions { get; set; }