feature: #1294: Added ability to process MV4 homework

--- docs: #1294: Added homework instructions and updated debug client models changelog. chore: #1294: Updated SQL queries. refactor: #1294: Yield Genotype matcher response as enumerable prevent memory exceptions. refactor: #1294: Use IEnumerable when bulk inserting to reduce memory usage. chore: #1294: Add SQL to select out chore: #1294: Store matching genotypes refactor: #1294: Move match prediction debug models to debug client models namespace. chore: #1294: Add HLA version to homework set. chore: #1294: SQL for selecting out homework subjects with missing HLA. chore: #1294: Process all sets imported found in a directory. chore: #1294: Import pdps directly from file. chore: #1294: Detect subjects with missing HLA. chore: #1294: Add functions to submit MV4 homework request. chore: #1294: Add genotype count to debug model, genotype matcher response.
Anthony-Nolan · May 6, 2024 · eb64991 · eb64991
1 parent e816fc8
commit eb64991
Show file tree

Hide file tree

Showing 98 changed files with 8,314 additions and 153 deletions.
diff --git a/...otypeFrequencySet/FrequencySetMetadata.cs → ...s/MatchPrediction/FrequencySetMetadata.cs b/...otypeFrequencySet/FrequencySetMetadata.cs → ...s/MatchPrediction/FrequencySetMetadata.cs
@@ -1,6 +1,4 @@
-using System;
-
-namespace Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet
+namespace Atlas.Common.Public.Models.MatchPrediction
 {
     /// <summary>
     /// This is the data used to determine which frequency set to use, for both Donors and Patients.

diff --git a/...ction/Models/MatchPredictionParameters.cs → ...chPrediction/MatchPredictionParameters.cs b/...ction/Models/MatchPredictionParameters.cs → ...chPrediction/MatchPredictionParameters.cs
@@ -1,8 +1,7 @@
 using Atlas.Common.Public.Models.GeneticData;
-using System.Collections.Generic;
 using Newtonsoft.Json;
 
-namespace Atlas.MatchPrediction.Models
+namespace Atlas.Common.Public.Models.MatchPrediction
 {
     /// <summary>
     /// Parameters controlling Match Prediction behaviour (in addition to <see cref="SubjectData"/>)
@@ -33,4 +32,4 @@ public MatchPredictionParameters(ISet<Locus> allowedLoci, string matchingAlgorit
             MatchingAlgorithmHlaNomenclatureVersion = matchingAlgorithmHlaNomenclatureVersion;
         }
     }
-}
+}
diff --git a/Atlas.Common/Sql/BulkInsert/BulkInsertRepository.cs b/Atlas.Common/Sql/BulkInsert/BulkInsertRepository.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
+using Atlas.Common.Utils.Extensions;
 using Microsoft.Data.SqlClient;
 
 namespace Atlas.Common.Sql.BulkInsert
@@ -13,7 +14,7 @@ public interface IBulkInsertRepository<in TEntity> where TEntity : IBulkInsertMo
         /// <summary>
         /// Note, `Id` and properties annotated with <see cref="BulkInsertIgnoreAttribute"/> will not be included in the bulk insert. 
         /// </summary>
-        Task BulkInsert(IReadOnlyCollection<TEntity> entities);
+        Task BulkInsert(IEnumerable<TEntity> entities);
     }
 
     public abstract class BulkInsertRepository<TEntity> : IBulkInsertRepository<TEntity> where TEntity : IBulkInsertModel
@@ -29,14 +30,17 @@ protected BulkInsertRepository(string connectionString, string bulkInsertTableNa
             this.bulkInsertTableName = bulkInsertTableName;
         }
 
-        public async Task BulkInsert(IReadOnlyCollection<TEntity> entities)
+        public async Task BulkInsert(IEnumerable<TEntity> entities)
         {
-            if (!entities.Any())
+            // ReSharper disable once PossibleMultipleEnumeration - `IsNullOrEmpty` does not enumerate the collection
+            if (entities.IsNullOrEmpty())
             {
                 return;
             }
 
             var columnNames = GetColumnNames();
+
+            // ReSharper disable once PossibleMultipleEnumeration
             var dataTable = BuildDataTable(entities, columnNames);
 
             using (var sqlBulk = BuildSqlBulkCopy(columnNames))

diff --git a/Atlas.Debug.Client.Models/CHANGELOG_DebugClientModels.md b/Atlas.Debug.Client.Models/CHANGELOG_DebugClientModels.md
@@ -6,6 +6,12 @@ This package contains client models utilised by the Atlas debug endpoints.
 ## Changelog
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### 2.2.0
+* Created new namespace `Atlas.Debug.Client.Models.MatchPrediction` and moved various models related to debugging match prediction here.
+  * Extended existing model, `GenotypeImputationResponse`, with new prop, `GenotypeCount`.
+  * Changed type of prop `MatchedGenotypePairs` on existing model `GenotypeMatcherResponse` from `string` to `IEnumerable<string>`.
+      * Instead of one, potentially very long, formatted string, the matched genotype pairs are now returned as a collection of formatted strings, one for each matching patient-donor genotype pair.
+
 ### 2.1.0
 * Creation of new library, `Atlas.Debug.Client.Models`, that contains models used in debug endpoints.
 * Moved following models to new project:

diff --git a/...Models/Debug/GenotypeImputationRequest.cs → ...chPrediction/GenotypeImputationRequest.cs b/...Models/Debug/GenotypeImputationRequest.cs → ...chPrediction/GenotypeImputationRequest.cs
@@ -1,13 +1,13 @@
-using Atlas.MatchPrediction.Models;
+using Atlas.Common.Public.Models.MatchPrediction;
 
-namespace Atlas.MatchPrediction.Functions.Models.Debug
+namespace Atlas.Debug.Client.Models.MatchPrediction
 {
     public class GenotypeImputationRequest
     {
         public SubjectInfo SubjectInfo { get; set; }
 
         /// <summary>
-        /// <inheritdoc cref="MatchPrediction.Models.MatchPredictionParameters"/>
+        /// <inheritdoc cref="Common.Public.Models.MatchPrediction.MatchPredictionParameters"/>
         /// </summary>
         public MatchPredictionParameters MatchPredictionParameters { get; set; }
     }

diff --git a/...odels/Debug/GenotypeImputationResponse.cs → ...hPrediction/GenotypeImputationResponse.cs b/...odels/Debug/GenotypeImputationResponse.cs → ...hPrediction/GenotypeImputationResponse.cs
@@ -1,7 +1,7 @@
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
-using Atlas.MatchPrediction.Models;
+using Atlas.Client.Models.Search.Results.MatchPrediction;
+using Atlas.Common.Public.Models.MatchPrediction;
 
-namespace Atlas.MatchPrediction.Functions.Models.Debug
+namespace Atlas.Debug.Client.Models.MatchPrediction
 {
     public class GenotypeImputationResponse
     {

diff --git a/...ns/Models/Debug/GenotypeMatcherRequest.cs → ...MatchPrediction/GenotypeMatcherRequest.cs b/...ns/Models/Debug/GenotypeMatcherRequest.cs → ...MatchPrediction/GenotypeMatcherRequest.cs
@@ -1,11 +1,11 @@
-using Atlas.MatchPrediction.Models;
+using Atlas.Common.Public.Models.MatchPrediction;
 
-namespace Atlas.MatchPrediction.Functions.Models.Debug
+namespace Atlas.Debug.Client.Models.MatchPrediction
 {
     public class GenotypeMatcherRequest
     {
         /// <summary>
-        /// <inheritdoc cref="MatchPrediction.Models.MatchPredictionParameters"/> 
+        /// <inheritdoc cref="Common.Public.Models.MatchPrediction.MatchPredictionParameters"/> 
         /// </summary>
         public MatchPredictionParameters MatchPredictionParameters { get; set; }
 

diff --git a/...s/Models/Debug/GenotypeMatcherResponse.cs → ...atchPrediction/GenotypeMatcherResponse.cs b/...s/Models/Debug/GenotypeMatcherResponse.cs → ...atchPrediction/GenotypeMatcherResponse.cs
@@ -1,7 +1,8 @@
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
-using Atlas.MatchPrediction.Models;
+using System.Collections.Generic;
+using Atlas.Client.Models.Search.Results.MatchPrediction;
+using Atlas.Common.Public.Models.MatchPrediction;
 
-namespace Atlas.MatchPrediction.Functions.Models.Debug
+namespace Atlas.Debug.Client.Models.MatchPrediction
 {
     public class GenotypeMatcherResponse
     {
@@ -10,25 +11,28 @@ public class GenotypeMatcherResponse
         public SubjectResult DonorInfo { get; set; }
 
         /// <summary>
-        /// Patient-donor genotype pairs (represented as a single, formatted string) and their match counts.
+        /// Patient-donor genotype pairs and their match counts.
         /// </summary>
-        public string MatchedGenotypePairs { get; set; }
+        public IEnumerable<string> MatchedGenotypePairs { get; set; }
     }
 
     public class SubjectResult
     {
         public bool IsUnrepresented { get; set; }
+        public int GenotypeCount { get; set; }
         public decimal SumOfLikelihoods { get; set; }
         public HaplotypeFrequencySet HaplotypeFrequencySet { get; set; }
         public string HlaTyping { get; set; }
 
         public SubjectResult(
             bool isUnrepresented,
+            int genotypeCount,
             decimal sumOfLikelihoods,
             HaplotypeFrequencySet haplotypeFrequencySet,
             string hlaTyping)
         {
             IsUnrepresented = isUnrepresented;
+            GenotypeCount = genotypeCount;
             SumOfLikelihoods = sumOfLikelihoods;
             HaplotypeFrequencySet = haplotypeFrequencySet;
             HlaTyping = hlaTyping;

diff --git a/...ion.Functions/Models/Debug/SubjectInfo.cs → ...ent.Models/MatchPrediction/SubjectInfo.cs b/...ion.Functions/Models/Debug/SubjectInfo.cs → ...ent.Models/MatchPrediction/SubjectInfo.cs
@@ -1,7 +1,7 @@
 using Atlas.Common.Public.Models.GeneticData.PhenotypeInfo.TransferModels;
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
+using Atlas.Common.Public.Models.MatchPrediction;
 
-namespace Atlas.MatchPrediction.Functions.Models.Debug
+namespace Atlas.Debug.Client.Models.MatchPrediction
 {
     public class SubjectInfo
     {

diff --git a/Atlas.Functions/Services/MatchPredictionInputBuilder.cs b/Atlas.Functions/Services/MatchPredictionInputBuilder.cs
@@ -7,9 +7,9 @@
 using Atlas.Common.ApplicationInsights.Timing;
 using Atlas.Common.Public.Models.GeneticData;
 using Atlas.Common.Public.Models.GeneticData.PhenotypeInfo.TransferModels;
+using Atlas.Common.Public.Models.MatchPrediction;
 using Atlas.Functions.Settings;
 using Atlas.MatchPrediction.ExternalInterface;
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
 using Atlas.MatchPrediction.ExternalInterface.Models.MatchProbability;
 using EnumStringValues;
 using Microsoft.Extensions.Options;

diff --git a/Atlas.ManualTesting.Common/Repositories/IProcessedResultsRepository.cs b/Atlas.ManualTesting.Common/Repositories/IProcessedResultsRepository.cs
@@ -3,6 +3,6 @@
     public interface IProcessedResultsRepository<in TDbModel>
     {
         Task DeleteResults(int searchRequestRecordId);
-        Task BulkInsert(IReadOnlyCollection<TDbModel> results);
+        Task BulkInsert(IEnumerable<TDbModel> results);
     }
 }
diff --git a/Atlas.ManualTesting.Common/Services/FileReader.cs b/Atlas.ManualTesting.Common/Services/FileReader.cs
@@ -4,13 +4,16 @@ namespace Atlas.ManualTesting.Common.Services
 {
     public interface IFileReader<T>
     {
-        Task<IReadOnlyCollection<T>> ReadAllLines(string delimiter, string filePath);
+        Task<IReadOnlyCollection<T>> ReadAllLines(string delimiter, string filePath, bool hasHeaderRecord = true);
         IAsyncEnumerable<T> ReadAsync(string delimiter, string filePath);
     }
 
     public class FileReader<T> : IFileReader<T>
     {
-        public async Task<IReadOnlyCollection<T>> ReadAllLines(string delimiter, string filePath)
+        public async Task<IReadOnlyCollection<T>> ReadAllLines(
+            string delimiter,
+            string filePath,
+            bool hasHeaderRecord = true)
         {
             FileChecks(filePath);
 
@@ -19,6 +22,7 @@ public async Task<IReadOnlyCollection<T>> ReadAllLines(string delimiter, string
             using var csv = new CsvReader(reader);
 
             csv.Configuration.Delimiter = delimiter;
+            csv.Configuration.HasHeaderRecord = hasHeaderRecord;
             csv.Configuration.HeaderValidated = null;
             csv.Configuration.MissingFieldFound = null;
             csv.Configuration.TypeConverterOptionsCache.GetOptions<string>().NullValues.Add("");

diff --git a/Atlas.ManualTesting.Common/Services/HttpRequester.cs b/Atlas.ManualTesting.Common/Services/HttpRequester.cs
@@ -0,0 +1,56 @@
+using Newtonsoft.Json;
+using Polly;
+
+namespace Atlas.ManualTesting.Common.Services
+{
+    public class AtlasHttpResult<TResult>
+    {
+        public bool WasSuccess => Result != null;
+        public TResult? Result { get; set; }
+
+        public AtlasHttpResult(TResult? result)
+        {
+            Result = result;
+        }
+    }
+
+    public abstract class AtlasHttpRequester
+    {
+        private readonly HttpClient httpRequestClient;
+        private readonly string requestUrl;
+
+        protected AtlasHttpRequester(HttpClient httpRequestClient, string requestUrl)
+        {
+            this.httpRequestClient = httpRequestClient;
+            this.requestUrl = requestUrl;
+        }
+
+        protected async Task<AtlasHttpResult<TResult>> PostRequest<TRequest, TResult>(TRequest request)
+        {
+            var retryPolicy = Policy.Handle<Exception>().RetryAsync(10);
+
+            var requestResponse = await retryPolicy.ExecuteAndCaptureAsync(
+                async () => await SendRequest<TRequest, TResult>(request));
+
+            return new AtlasHttpResult<TResult>(requestResponse.Result);
+        }
+
+        private async Task<TResult?> SendRequest<TRequest, TResult>(TRequest request)
+        {
+            try
+            {
+                var response = await httpRequestClient.PostAsync(
+                    requestUrl, new StringContent(JsonConvert.SerializeObject(request)));
+                response.EnsureSuccessStatusCode();
+
+                return JsonConvert.DeserializeObject<TResult>(await response.Content.ReadAsStringAsync());
+            }
+            catch (Exception ex)
+            {
+                System.Diagnostics.Debug.WriteLine($"Failed to send request {request}. Details: {ex.Message} " +
+                                "Re-attempting until success or re-attempt count reached.");
+                throw;
+            }
+        }
+    }
+}
diff --git a/Atlas.MatchPrediction.Functions/Functions/Debug/GenotypeImputationFunctions.cs b/Atlas.MatchPrediction.Functions/Functions/Debug/GenotypeImputationFunctions.cs
@@ -1,9 +1,9 @@
 using System.IO;
 using System.Threading.Tasks;
 using Atlas.Common.Public.Models.GeneticData.PhenotypeInfo.TransferModels;
+using Atlas.Common.Public.Models.MatchPrediction;
 using Atlas.Common.Utils.Http;
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
-using Atlas.MatchPrediction.Functions.Models.Debug;
+using Atlas.Debug.Client.Models.MatchPrediction;
 using Atlas.MatchPrediction.Functions.Services.Debug;
 using Atlas.MatchPrediction.Models;
 using Atlas.MatchPrediction.Services.HaplotypeFrequencies;
@@ -13,7 +13,6 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Azure.WebJobs;
 using Microsoft.Azure.WebJobs.Extensions.Http;
-using MoreLinq;
 using Newtonsoft.Json;
 
 namespace Atlas.MatchPrediction.Functions.Functions.Debug
@@ -52,7 +51,7 @@ public class GenotypeImputationFunctions
             {
                 HlaTyping = input.SubjectInfo.HlaTyping.ToPhenotypeInfo().PrettyPrint(),
                 MatchPredictionParameters = input.MatchPredictionParameters,
-                HaplotypeFrequencySet = frequencySet,
+                HaplotypeFrequencySet = frequencySet.ToClientHaplotypeFrequencySet(),
                 GenotypeCount = imputedGenotypes.GenotypeLikelihoods.Count,
                 SumOfLikelihoods = imputedGenotypes.SumOfLikelihoods,
                 GenotypeLikelihoods = imputedGenotypes.GenotypeLikelihoods.ToSingleDelimitedString()

diff --git a/Atlas.MatchPrediction.Functions/Functions/Debug/GenotypeLikelihoodFunctions.cs b/Atlas.MatchPrediction.Functions/Functions/Debug/GenotypeLikelihoodFunctions.cs
@@ -1,7 +1,7 @@
 using System.IO;
 using System.Threading.Tasks;
+using Atlas.Common.Public.Models.MatchPrediction;
 using Atlas.Common.Utils.Http;
-using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
 using Atlas.MatchPrediction.ExternalInterface.Models.MatchPredictionSteps.GenotypeLikelihood;
 using Atlas.MatchPrediction.Services.GenotypeLikelihood;
 using Atlas.MatchPrediction.Services.HaplotypeFrequencies;

diff --git a/Atlas.MatchPrediction.Functions/Functions/Debug/MatchCalculationFunctions.cs b/Atlas.MatchPrediction.Functions/Functions/Debug/MatchCalculationFunctions.cs
@@ -2,8 +2,8 @@
 using System.Threading.Tasks;
 using Atlas.Common.Public.Models.GeneticData.PhenotypeInfo.TransferModels;
 using Atlas.Common.Utils.Http;
+using Atlas.Debug.Client.Models.MatchPrediction;
 using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
-using Atlas.MatchPrediction.Functions.Models.Debug;
 using Atlas.MatchPrediction.Functions.Services.Debug;
 using Atlas.MatchPrediction.Models;
 using Atlas.MatchPrediction.Services.HaplotypeFrequencies;
@@ -49,15 +49,15 @@ public class MatchCalculationFunctions
                 MatchPredictionParameters = input.MatchPredictionParameters
             });
 
-            var response = new GenotypeMatcherResponse
+            var matcherResponse = new GenotypeMatcherResponse
             {
                 MatchPredictionParameters = input.MatchPredictionParameters,
                 PatientInfo = BuildSubjectResult(result.PatientResult, frequencySet.PatientSet, input.Patient),
                 DonorInfo = BuildSubjectResult(result.DonorResult, frequencySet.DonorSet, input.Donor),
-                MatchedGenotypePairs = result.GenotypeMatchDetails.ToSingleDelimitedString()
+                MatchedGenotypePairs = result.GenotypeMatchDetails.ToFormattedStrings(),
             };
 
-            return new JsonResult(response);
+            return new JsonResult(matcherResponse);
         }
 
         private static SubjectResult BuildSubjectResult(
@@ -67,8 +67,9 @@ public class MatchCalculationFunctions
         {
             return new SubjectResult(
                 subjectResult.IsUnrepresented,
+                subjectResult.GenotypeCount,
                 subjectResult.SumOfLikelihoods,
-                set, 
+                set.ToClientHaplotypeFrequencySet(), 
                 subjectInfo.HlaTyping.ToPhenotypeInfo().PrettyPrint());
         }
     }

diff --git a/Atlas.MatchPrediction.Functions/Functions/HaplotypeFrequencySetFunctions.cs b/Atlas.MatchPrediction.Functions/Functions/HaplotypeFrequencySetFunctions.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Atlas.Common.AzureEventGrid;
+using Atlas.Common.Public.Models.MatchPrediction;
 using Atlas.MatchPrediction.ExternalInterface;
 using Atlas.MatchPrediction.ExternalInterface.Models.HaplotypeFrequencySet;
 using Atlas.MatchPrediction.Models;