Merge pull request #162 from SMI/flexible-yaml

Added -y option to ii

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 # [Unreleased]
 
+## Added
+
+- Added new command line flag `-y somefile.yaml` in `ii` CLI tool to specify a custom config file
+
+## Changed
+
+- `ii` startup errors are written to stderr instead of stdout
+
 # [0.0.5] - 2022-07-20
 
 ## Dependencies

Tests/IsIdentifiableTests/IsIdentifiable.Tests.csproj

@@ -30,6 +30,9 @@
     <ProjectReference Include="..\..\IsIdentifiable\IsIdentifiable.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <None Update="default.yaml">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="pixeldatareport.csv">
       <Generator>TextTemplatingFileGenerator</Generator>
       <LastGenOutput>pixeldatareport.cs</LastGenOutput>

Tests/IsIdentifiableTests/ReviewerTests/SettingsFileTests.cs

@@ -0,0 +1,87 @@
+using NUnit.Framework;
+using System.IO;
+
+namespace IsIdentifiable.Tests.ReviewerTests
+{
+    public class SettingsFileTests 
+    {
+        [Test]
+        public void TestCutSettingsFileArgs_NoArgs()
+        {
+            Assert.IsNull(Program.CutSettingsFileArgs(new string[0],out var result));
+            Assert.IsEmpty(result);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_NoYamlFile()
+        {
+            Assert.IsNull(Program.CutSettingsFileArgs(new [] { "review" ,"somefish"}, out var result));
+            Assert.AreEqual(2, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("somefish", result[1]);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_DashYOnly()
+        {
+            // missing argument, let CommandLineParser sort them out
+            Assert.IsNull(Program.CutSettingsFileArgs(new[] { "-y" }, out var result));
+            Assert.AreEqual(1, result.Length);
+            Assert.AreEqual("-y", result[0]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlOnly()
+        {
+            Assert.AreEqual("myfile.yaml",Program.CutSettingsFileArgs(new[] { "-y" ,"myfile.yaml"}, out var result));
+            Assert.IsEmpty(result);
+        }
+
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlAfter()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] {"review", "db","someconstr" , "-y", "myfile.yaml" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlBefore()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] { "-y", "myfile.yaml", "review", "db", "someconstr" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestCutSettingsFileArgs_YamlInMiddle()
+        {
+            Assert.AreEqual("myfile.yaml", Program.CutSettingsFileArgs(new[] { "review", "-y", "myfile.yaml", "db", "someconstr" }, out var result));
+
+            Assert.AreEqual(3, result.Length);
+            Assert.AreEqual("review", result[0]);
+            Assert.AreEqual("db", result[1]);
+            Assert.AreEqual("someconstr", result[2]);
+        }
+
+        [Test]
+        public void TestDeserialize_SmiServices_DefaultYaml()
+        {
+            var f = Path.Combine(TestContext.CurrentContext.TestDirectory,"default.yaml");
+            FileAssert.Exists(f);
+
+            var opts = Program.Deserialize(f);
+            Assert.IsNotNull(opts.IsIdentifiableReviewerOptions);
+            Assert.IsNotNull(opts.IsIdentifiableOptions);
+        }
+    }
+}

Tests/IsIdentifiableTests/default.yaml

@@ -0,0 +1,82 @@
+# This file demonstrates that you can load `ii review -y default.yaml` with a larger config file and it will ignore settings unrelated
+# to it.  This allows you to have a config file that is consumed by your own process (via IsIdentifiable Nuget package) but still use
+# the same config file with the reviewer CLI tool
+
+RabbitOptions:
+    RabbitMqHostName: 'localhost'
+    RabbitMqHostPort: 5672
+    RabbitMqVirtualHost: '/'
+    RabbitMqUserName: 'guest'
+    RabbitMqPassword: 'guest'
+    RabbitMqControlExchangeName: 'TEST.ControlExchange'
+    FatalLoggingExchange: 'TEST.FatalLoggingExchange'
+
+IsIdentifiableServiceOptions:
+    QueueName: 'TEST.ExtractedFileToVerifyQueue'
+    QoSPrefetchCount: 1
+    AutoAck: false
+    IsIdentifiableProducerOptions:
+        ExchangeName: 'TEST.ExtractedFileVerifiedExchange'
+        MaxConfirmAttempts: 1
+    ClassifierType: 'Microservices.IsIdentifiable.Service.TesseractStanfordDicomFileClassifier'
+    DataDirectory: ''
+
+IsIdentifiableOptions: 
+    #Optional. Full connection string to the database storing the Allowlist of valid entries
+    AllowlistConnectionString:
+    #Optional. The DBMS provider of the Allowlist table e.g. MySql
+    AllowlistDatabaseType:
+    #Optional. The unqualified name of the Allowlist table
+    AllowlistTableName:
+    #Optional. The column in AllowlistTableName which contains the Allowlist elements
+    AllowlistColumn:
+
+    #Optional. Path to a CSV file containing a single untitled column of Allowlist values
+    AllowlistCsv:
+
+    #Optional. Generate a report on the proportion of values failing validation (for each column)
+    #ColumnReport: true
+
+    #Optional. Generate a report listing every unique value failing validation (and the column the value failed in)
+    #ValuesReport: true
+
+    #Optional. Generate a full failure storage report that persists Failure objects in a manner that they can be retrieved.
+    #StoreReport: true
+
+    #Optional - If specified reports will be generated in the given folder.  If not specified, current directory is used (unless an alternate destination option is picked)
+    DestinationCsvFolder:
+    #Optional - If specified, the given separator will be used instead of ,.  Includes support for \t for tab and \r\n
+    DestinationCsvSeparator:
+    #Optional - If specified all tabs, newlines (\r and \n) and 2+ spaces will be stripped from the values written as output (applies to all output formats)
+    DestinationNoWhitespace:
+
+    #Optional. Full connection string to the database in which to store the report results
+    DestinationConnectionString:
+    #Optional. The DBMS provider of DestinationConnectionString e.g. MySql
+    DestinationDatabaseType:
+
+    #Optional. If specified postcodes will not be reported as failures
+    IgnorePostcodes: false
+    #Optional. Comma separated list of columns/tags which should be ignored and not processed
+    SkipColumns:
+    #Optional. If set and using a 7 class NER model then DATE and TIME objects will not be considered failures.
+    IgnoreDatesInText:
+    #Optional. Set to control the max size of the in-memory store of processed before the get written out to any destinations. Only makes sense for reports that don't perform any aggregation across the data
+    MaxCacheSize:
+
+    #Optional. Filename of additional rules in yaml format.
+    RulesFile:
+    #Optional. Directory of additional rules in yaml format.
+    RulesDirectory:
+    #Optional.  Maximum number of answers to cache per column.
+    MaxValidationCacheSize:
+
+IsIdentifiableReviewerOptions:
+    #Location of database connection strings file (for issuing UPDATE statements)    
+    TargetsFile:
+    #File containing rules for ignoring validation errors
+    IgnoreList:
+    #File containing rules for when to issue UPDATE statements
+    RedList:
+    #Sets the user interface to use a specific color palette yaml file
+    Theme:

ii/GlobalOptions.cs

@@ -2,7 +2,7 @@
 
 namespace IsIdentifiable;
 
-internal class GlobalOptions
+public class GlobalOptions
 {
     public IsIdentifiableBaseOptions? IsIdentifiableOptions { get; set; }
 

ii/Program.cs

@@ -8,6 +8,7 @@
 using IsIdentifiable.Options;
 using IsIdentifiable.Redacting;
 using IsIdentifiable.Runners;
+using System.Linq;
 using YamlDotNet.Serialization;
 
 namespace IsIdentifiable;
@@ -17,19 +18,64 @@ public static class Program
     const string SettingsFile = "ii-settings.yaml";
     static GlobalOptions? GlobalOptions;
 
+
+    public static string? CutSettingsFileArgs(string []args, out string[] newArgs )
+    {
+        var idx = Array.IndexOf(args, "-y");
+
+        // do we have 2 elements "-y" then "somepath"?
+        if (idx != -1 && idx < args.Length - 1)
+        {
+            var settingsFileLocation = args[idx + 1];
+
+            // remove -y myfile
+            newArgs = args.Take(idx).ToArray();
+
+            foreach (var after in args.Skip(idx + 2))
+            {
+                newArgs = newArgs.Append(after).ToArray();
+            }
+
+            return settingsFileLocation;
+        }
+
+        // no change
+        newArgs = args;
+        return null;
+    }
+
     public static int Main(string[] args)
     {
-
-        if(File.Exists(SettingsFile))
+
+        string? explicitLocation = CutSettingsFileArgs(args, out var newArgs);
+        args = newArgs;
+
+        var settingsFileLocation = SettingsFile;
+
+        // if user specified -y on command line
+        if (!string.IsNullOrWhiteSpace(explicitLocation))
+        {
+            //verify the file they told us about is real
+            if (!File.Exists(explicitLocation))
+            {
+                Console.Error.WriteLine($"Could not find file: {explicitLocation}");
+                return 1;
+            }
+
+            // if it is then use it instead of ii-settings.yaml
+            settingsFileLocation = explicitLocation;
+        }
+
+        // load GlobalOptions
+        if(File.Exists(settingsFileLocation))
         {
             try
             {
-                var d = new Deserializer();
-                GlobalOptions = d.Deserialize<GlobalOptions>(File.ReadAllText(SettingsFile));
+                GlobalOptions = Deserialize(settingsFileLocation);
             }
             catch (Exception ex)
             {
-                Console.WriteLine($"Could not deserialize {SettingsFile}:{ex}");
+                Console.Error.WriteLine($"Could not deserialize {SettingsFile}:{ex}");
                 return 1;
             }
         }
@@ -67,6 +113,15 @@ public static int Main(string[] args)
         return res;
     }
 
+    public static GlobalOptions? Deserialize(string settingsFileLocation)
+    {
+        IDeserializer deserializer = new DeserializerBuilder()
+                        .IgnoreUnmatchedProperties()
+                        .Build();
+
+        return deserializer.Deserialize<GlobalOptions>(File.ReadAllText(settingsFileLocation));
+    }
+
     private static int Run(IsIdentifiableReviewerOptions opts)
     {
         Inherit(opts);