Skip to content

Commit

Permalink
chore: #1294: Process all sets imported found in a directory.
Browse files Browse the repository at this point in the history
  • Loading branch information
zabeen committed Apr 29, 2024
1 parent 89cec57 commit b53b0df
Show file tree
Hide file tree
Showing 7 changed files with 220 additions and 112 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using Dapper;
using Microsoft.Data.SqlClient;

namespace Atlas.MatchPrediction.Test.Validation.Data.Repositories.Homework
{
public interface IHomeworkDeletionRepository
{
Task DeleteAll();
}

public class HomeworkDeletionRepository : IHomeworkDeletionRepository
{
private readonly string connectionString;

public HomeworkDeletionRepository(string connectionString)
{
this.connectionString = connectionString;
}

/// <inheritdoc />
public async Task DeleteAll()
{
const string deleteAllPairs = "DELETE FROM PatientDonorPairs";

await using (var connection = new SqlConnection(connectionString))
{
await connection.ExecuteAsync(deleteAllPairs);
}

const string deleteAllSets = "DELETE FROM HomeworkSets";

await using (var connection = new SqlConnection(connectionString))
{
await connection.ExecuteAsync(deleteAllSets);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using Atlas.Common.ServiceBus;
using Atlas.Common.Utils.Extensions;
using Atlas.DonorImport.ExternalInterface.DependencyInjection;
using Atlas.ManualTesting.Common.Models;
using Atlas.ManualTesting.Common.Models.Entities;
using Atlas.ManualTesting.Common.Repositories;
using Atlas.ManualTesting.Common.Services;
Expand Down Expand Up @@ -152,12 +151,15 @@ private static void RegisterDatabaseServices(this IServiceCollection services, F
this IServiceCollection services,
Func<IServiceProvider, string> fetchSqlConnectionString)
{
services.AddScoped<IHomeworkDeletionRepository, HomeworkDeletionRepository>(sp =>
new HomeworkDeletionRepository(fetchSqlConnectionString(sp)));
services.AddScoped<IHomeworkSetRepository, HomeworkSetRepository>(sp =>
new HomeworkSetRepository(fetchSqlConnectionString(sp)));
services.AddScoped<IPatientDonorPairRepository, PatientDonorPairRepository>(sp =>
new PatientDonorPairRepository(fetchSqlConnectionString(sp)));

services.AddScoped<IHomeworkRequestProcessor, HomeworkRequestProcessor>();
services.AddScoped<IHomeworkCreator, HomeworkCreator>();
services.AddScoped<IHomeworkProcessor, HomeworkProcessor>();
services.AddScoped<IPatientDonorPairProcessor, PatientDonorPairProcessor>();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Atlas.Common.Utils.Http;
using Atlas.ManualTesting.Common.Services;
using Atlas.MatchingAlgorithm.Client.Models.DataRefresh;
using Atlas.MatchPrediction.Test.Validation.Data.Repositories.Homework;
using Atlas.MatchPrediction.Test.Validation.Models;
using Atlas.MatchPrediction.Test.Validation.Services.Exercise4;
using Atlas.MatchPrediction.Test.Validation.Services.Exercise4.Homework;
Expand All @@ -13,6 +14,7 @@
using Microsoft.Azure.WebJobs.Extensions.Http;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Net;
Expand All @@ -31,20 +33,26 @@ public class Exercise4Functions
private readonly ISearchRequester searchRequester;
private readonly IResultSetProcessor<SearchResultsNotification> searchResultSetProcessor;
private readonly ISearchResultNotificationSender messageSender;
private readonly IHomeworkRequestProcessor homeworkRequestProcessor;
private readonly IHomeworkDeletionRepository deletionRepository;
private readonly IHomeworkCreator homeworkCreator;
private readonly IHomeworkProcessor homeworkProcessor;

public Exercise4Functions(
IValidationAtlasPreparer atlasPreparer,
ISearchRequester searchRequester,
IResultSetProcessor<SearchResultsNotification> searchResultSetProcessor,
ISearchResultNotificationSender messageSender,
IHomeworkRequestProcessor homeworkRequestProcessor)
IHomeworkDeletionRepository deletionRepository,
IHomeworkCreator homeworkCreator,
IHomeworkProcessor homeworkProcessor)
{
this.atlasPreparer = atlasPreparer;
this.searchRequester = searchRequester;
this.searchResultSetProcessor = searchResultSetProcessor;
this.messageSender = messageSender;
this.homeworkRequestProcessor = homeworkRequestProcessor;
this.deletionRepository = deletionRepository;
this.homeworkCreator = homeworkCreator;
this.homeworkProcessor = homeworkProcessor;
}

[SuppressMessage(null, SuppressMessage.UnusedParameter, Justification = SuppressMessage.UsedByAzureTrigger)]
Expand Down Expand Up @@ -121,23 +129,24 @@ public class Exercise4Functions
}
}

[FunctionName($"{FunctionNamePrefix}3_{nameof(SubmitHomeworkRequest)}")]
public async Task<IActionResult> SubmitHomeworkRequest(
[FunctionName($"{FunctionNamePrefix}3_{nameof(CreateNewHomeworkSets)}")]
public async Task<IActionResult> CreateNewHomeworkSets(
[HttpTrigger(AuthorizationLevel.Function, "post")]
[RequestBodyType(typeof(HomeworkRequest), nameof(HomeworkRequest))]
HttpRequest request)
{
var homeworkRequest = await request.DeserialiseRequestBody<HomeworkRequest>();
return new OkObjectResult(await homeworkRequestProcessor.StoreHomeworkRequest(homeworkRequest));
if(homeworkRequest.DeletePreviousHomeworkSets) await deletionRepository.DeleteAll();
return new OkObjectResult(await homeworkCreator.CreateHomeworkSets(homeworkRequest));
}

[FunctionName($"{FunctionNamePrefix}4_{nameof(StartOrContinueHomeworkSet)}")]
public async Task StartOrContinueHomeworkSet(
[FunctionName($"{FunctionNamePrefix}4_{nameof(StartOrContinueHomeworkSets)}")]
public async Task StartOrContinueHomeworkSets(
[HttpTrigger(AuthorizationLevel.Function, "post")]
[RequestBodyType(typeof(int), "homeworkSetId")]
[RequestBodyType(typeof(IEnumerable<int>), "homeworkSetIds")]
HttpRequest request)
{
await homeworkRequestProcessor.StartOrContinueHomeworkRequest(await request.DeserialiseRequestBody<int>());
await homeworkProcessor.StartOrContinueHomeworkSets(await request.DeserialiseRequestBody<IEnumerable<int>>());
}
}
}
10 changes: 5 additions & 5 deletions Atlas.MatchPrediction.Test.Validation/Models/HomeworkRequest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ namespace Atlas.MatchPrediction.Test.Validation.Models
public class HomeworkRequest
{
/// <summary>
/// Path where homework set file should be read from.
/// Path where homework set files should be read from.
/// Make sure to escape the backslashes in the path.
/// </summary>
public string InputPath { get; set; }
Expand All @@ -17,13 +17,13 @@ public class HomeworkRequest
public string ResultsPath { get; set; }

/// <summary>
/// Include the file extension, e.g., "A.csv".
/// Loci set to `true` will be included in the analysis.
/// </summary>
public string SetFileName { get; set; }
public LociInfoTransfer<bool> MatchLoci { get; set; }

/// <summary>
/// Loci set to `true` will be included in the analysis.
/// If `true`, all previously stored homework sets will be deleted from the validation database.
/// </summary>
public LociInfoTransfer<bool> MatchLoci { get; set; }
public bool DeletePreviousHomeworkSets { get; set; }
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Atlas.Common.Utils.Extensions;
using Atlas.ManualTesting.Common.Services;
using Atlas.MatchPrediction.Test.Validation.Data.Models.Homework;
using Atlas.MatchPrediction.Test.Validation.Data.Repositories.Homework;
using Atlas.MatchPrediction.Test.Validation.Models;

namespace Atlas.MatchPrediction.Test.Validation.Services.Exercise4.Homework
{
public interface IHomeworkCreator
{
/// <summary>
/// Creates new homework sets from the input files defined in the request.
/// </summary>
/// <returns>Homework Set Ids</returns>
Task<IEnumerable<int>> CreateHomeworkSets(HomeworkRequest request);
}

internal class HomeworkCreator : IHomeworkCreator
{
private const string FileDelimiter = ",";
private readonly IHomeworkSetRepository setRepository;
private readonly IFileReader<SubjectIdPair> fileReader;
private readonly IPatientDonorPairRepository pdpRepository;

public HomeworkCreator(
IHomeworkSetRepository setRepository,
IFileReader<SubjectIdPair> fileReader,
IPatientDonorPairRepository pdpRepository)
{
this.setRepository = setRepository;
this.fileReader = fileReader;
this.pdpRepository = pdpRepository;
}

/// <inheritdoc />
public async Task<IEnumerable<int>> CreateHomeworkSets(HomeworkRequest request)
{
var setIds = new List<int?>();

var fileNames = GetCsvFileNames(request.InputPath);

foreach (var file in fileNames)
{
setIds.Add(await ImportPatientDonorPairs(request, file));
}

return setIds.Where(id => id.HasValue).Select(id => id.Value);
}

private static IEnumerable<string> GetCsvFileNames(string path)
{
return Directory.GetFiles(path)
.Where(file => Path.GetExtension(file) == ".csv")
.Select(Path.GetFileName);
}

private async Task<int?> ImportPatientDonorPairs(HomeworkRequest request, string setFileName)
{
var subjectIdPairs = await fileReader.ReadAllLines(
FileDelimiter,
Path.Combine(request.InputPath, setFileName),
hasHeaderRecord: false);

if (subjectIdPairs.IsNullOrEmpty())
{
System.Diagnostics.Debug.WriteLine($"No patient-donor pairs found in file {setFileName}.");
return null;
}

var setId = await setRepository.Add(setFileName, request.ResultsPath, request.MatchLoci.MatchLociToString());
await pdpRepository.BulkInsert(subjectIdPairs.Select(ids => MapToDatabaseModel(ids, setId)).ToList());
return setId;
}

private static PatientDonorPair MapToDatabaseModel(SubjectIdPair ids, int homeworkSetId)
{
return new PatientDonorPair
{
PatientId = ids.PatientId,
DonorId = ids.DonorId,
HomeworkSet_Id = homeworkSetId
};
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Atlas.Common.Utils.Extensions;
using Atlas.MatchPrediction.Test.Validation.Data.Repositories.Homework;
using Atlas.MatchPrediction.Test.Validation.Models;

namespace Atlas.MatchPrediction.Test.Validation.Services.Exercise4.Homework
{
public interface IHomeworkProcessor
{
Task StartOrContinueHomeworkSets(IEnumerable<int> homeworkSetIds);
}

internal class HomeworkProcessor : IHomeworkProcessor
{
private readonly IHomeworkSetRepository setRepository;
private readonly IPatientDonorPairRepository pdpRepository;
private readonly IPatientDonorPairProcessor pdpProcessor;

public HomeworkProcessor(
IHomeworkSetRepository setRepository,
IPatientDonorPairRepository pdpRepository,
IPatientDonorPairProcessor pdpProcessor)
{
this.setRepository = setRepository;
this.pdpRepository = pdpRepository;
this.pdpProcessor = pdpProcessor;
}

public async Task StartOrContinueHomeworkSets(IEnumerable<int> homeworkSetIds)
{
foreach (var homeworkSetId in homeworkSetIds)
{
await ProcessHomeworkSet(homeworkSetId);
}
}

private async Task ProcessHomeworkSet(int homeworkSetId)
{
var set = await setRepository.Get(homeworkSetId);

if (set == null)
{
throw new ArgumentException($"No homework set found with id {homeworkSetId}.");
}

var pdps = (await pdpRepository.GetUnprocessedPairs(homeworkSetId)).ToList();

if (pdps.IsNullOrEmpty())
{
System.Diagnostics.Debug.WriteLine($"No unprocessed patient-donor pairs found for homework set {homeworkSetId}.");
return;
}

var matchLoci = set.MatchLoci.ToLociInfo();

foreach (var pdp in pdps)
{
await pdpProcessor.Process(pdp, matchLoci);
}
}
}
}
Loading

0 comments on commit b53b0df

Please sign in to comment.