Skip to content

Commit

Permalink
Implements breakpoint cache preheating to improve processing performa…
Browse files Browse the repository at this point in the history 
…nce.
  • Loading branch information
@AClark-WHONET
AClark-WHONET committed Feb 16, 2024
1 parent fcb4a69 commit 2264d9c
Show file tree
Hide file tree
Showing 4 changed files with 266 additions and 141 deletions.
203 changes: 135 additions & 68 deletions Interpretation Engine/AntibioticSpecificInterpretationRules.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;

namespace AMR_Engine
Expand Down Expand Up @@ -34,10 +36,6 @@ public class AntibioticSpecificInterpretationRules
AntibioticComponents thisAntibiotic = new AntibioticComponents(whonetAntimicrobialFullCode);
AntimicrobialTestMethod = thisAntibiotic.TestMethod;

List<int> prioritizedGuidelineYears = null;
if (guidelineYear > 0)
prioritizedGuidelineYears = new List<int> { guidelineYear };

AntimicrobialResult = antimicrobialResult_;

InterpretationLibrary.ParseResult(AntimicrobialTestMethod, AntimicrobialResult, ref NumericResult, ref ResultModifier);
Expand Down Expand Up @@ -99,68 +97,12 @@ public class AntibioticSpecificInterpretationRules
MostApplicableBreakpoint = BreakpointLookup[whonetOrganismCode][thisAntibiotic.Guideline][guidelineYear][whonetAntimicrobialFullCode];
else
{

// We haven't seen this combination before, so we need to evaluate it.
List<Breakpoint> applicableBPs =
Breakpoint.GetApplicableBreakpoints(
whonetOrganismCode,
userDefinedBreakpoints,
prioritizedGuidelines: new List<string>() { thisAntibiotic.Guideline },
prioritizedGuidelineYears: prioritizedGuidelineYears,
prioritizedBreakpointTypes: prioritizedBreakpointTypes,
prioritizedSitesOfInfection: prioritizedSitesOfInfection,
prioritizedWhonetAbxFullDrugCodes: new List<string>() { whonetAntimicrobialFullCode },
returnFirstBreakpointOnly: true);

if (applicableBPs.Count > 0)
MostApplicableBreakpoint = applicableBPs[0];
else
MostApplicableBreakpoint = null;

// Create the missing levels in our lookup for next time, and store this breakpoint set.
if (BreakpointLookup.ContainsKey(whonetOrganismCode))
{
if (BreakpointLookup[whonetOrganismCode].ContainsKey(thisAntibiotic.Guideline))
{
if (BreakpointLookup[whonetOrganismCode][thisAntibiotic.Guideline].ContainsKey(guidelineYear))
{
// Only the antibiotic info missing.
BreakpointLookup[whonetOrganismCode][thisAntibiotic.Guideline][guidelineYear].Add(whonetAntimicrobialFullCode, MostApplicableBreakpoint);
}
else
{
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, MostApplicableBreakpoint);

BreakpointLookup[whonetOrganismCode][thisAntibiotic.Guideline].Add(guidelineYear, abxSet);
}
}
else
{
// Create everything below the organism.
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, MostApplicableBreakpoint);

Dictionary<int, Dictionary<string, Breakpoint>> yearSet = new Dictionary<int, Dictionary<string, Breakpoint>>();
yearSet.Add(guidelineYear, abxSet);

BreakpointLookup[whonetOrganismCode].Add(thisAntibiotic.Guideline, yearSet);
}
}
else
{
// The organism key missing, which means we have to create the whole structure.
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, MostApplicableBreakpoint);

Dictionary<int, Dictionary<string, Breakpoint>> yearSet = new Dictionary<int, Dictionary<string, Breakpoint>>();
yearSet.Add(guidelineYear, abxSet);

Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>> guidelineSet = new Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>();
guidelineSet.Add(thisAntibiotic.Guideline, yearSet);

BreakpointLookup.Add(whonetOrganismCode, guidelineSet);
}
// Determine the most applicable breakpoint, if any.
// This shouldn't be necessary if the breakpoint cache was preheated.
MostApplicableBreakpoint =
DetermineMostApplicableBreakpoint(
userDefinedBreakpoints, guidelineYear, prioritizedBreakpointTypes, prioritizedSitesOfInfection,
whonetOrganismCode, thisAntibiotic.Guideline, whonetAntimicrobialFullCode);
}
}
}
Expand Down Expand Up @@ -188,7 +130,8 @@ public class AntibioticSpecificInterpretationRules

private readonly Breakpoint MostApplicableBreakpoint;

private static readonly Dictionary<string, Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>> BreakpointLookup = new Dictionary<string, Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>>();
private static readonly Dictionary<string, Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>> BreakpointLookup =
new Dictionary<string, Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>>();

private static readonly object BreakpointLookupLock = new object();

Expand All @@ -204,6 +147,59 @@ public static void ClearBreakpoints()
}
}

/// <summary>
/// Prepopulate the breakpoint cache with the "MostApplicableBreakpoint" for each combination provided.
/// This will greatly improve performance rather than looking them up ad hoc during processing because
/// the BreakpointLookup has to be locked (blocking the other threads) during lookup.
/// </summary>
/// <param name="distinctInterpretationKeys"></param>
public static void PreheatBreakpointCache(
List<Breakpoint> userDefinedBreakpoints,
int guidelineYear,
List<string> prioritizedBreakpointTypes,
List<string> prioritizedSitesOfInfection,
List<Tuple<string, string, string>> distinctInterpretationKeys,
BackgroundWorker worker = null)
{
// We don't need to look up keys that already exist in the cache.
distinctInterpretationKeys =
distinctInterpretationKeys.
Where(k => !(BreakpointLookup.ContainsKey(k.Item1)
&& BreakpointLookup[k.Item1].ContainsKey(k.Item2)
&& BreakpointLookup[k.Item1][k.Item2].ContainsKey(guidelineYear)
&& BreakpointLookup[k.Item1][k.Item2][guidelineYear].ContainsKey(k.Item3))).
ToList();

int lastReportedProgress = 0;
int currentProgress = 0;
int totalKeys = distinctInterpretationKeys.Count();

lock (BreakpointLookup)
{
for (int x = 0; x < totalKeys; x++)
{
Tuple<string, string, string> k = distinctInterpretationKeys[x];

DetermineMostApplicableBreakpoint(
userDefinedBreakpoints,
guidelineYear,
prioritizedBreakpointTypes,
prioritizedSitesOfInfection,
k.Item1, k.Item2, k.Item3);

if (worker != null)
{
currentProgress = (x * 100) / totalKeys;
if (currentProgress > lastReportedProgress)
{
lastReportedProgress = currentProgress;
worker.ReportProgress(lastReportedProgress);
}
}
}
}
}

/// <summary>
/// Apply any matching intrinsic resistance rules and breakpoints for this result.
/// </summary>
Expand Down Expand Up @@ -394,6 +390,77 @@ private string ApplyBreakpoints()
return Constants.InterpretationCodes.Uninterpretable;
}

private static Breakpoint DetermineMostApplicableBreakpoint(
List<Breakpoint> userDefinedBreakpoints,
int guidelineYear,
List<string> prioritizedBreakpointTypes,
List<string> prioritizedSitesOfInfection,
string whonetOrganismCode,
string guideline,
string whonetAntimicrobialFullCode)
{
// We haven't seen this combination before, so we need to evaluate it.
// If there is no breakpoint matching these requirements, then we will return Null here
// and save that value for future lookups to indicate that there is no applicable breakpoint.
Breakpoint mostApplicableBreakpoint =
Breakpoint.GetApplicableBreakpoints(
whonetOrganismCode,
userDefinedBreakpoints,
prioritizedGuidelines: new List<string>() { guideline },
prioritizedGuidelineYears: new List<int> { guidelineYear },
prioritizedBreakpointTypes: prioritizedBreakpointTypes,
prioritizedSitesOfInfection: prioritizedSitesOfInfection,
prioritizedWhonetAbxFullDrugCodes: new List<string>() { whonetAntimicrobialFullCode },
returnFirstBreakpointOnly: true).FirstOrDefault();

// Create the missing levels in our lookup for next time, and store this breakpoint set.
if (BreakpointLookup.ContainsKey(whonetOrganismCode))
{
if (BreakpointLookup[whonetOrganismCode].ContainsKey(guideline))
{
if (BreakpointLookup[whonetOrganismCode][guideline].ContainsKey(guidelineYear))
{
// Only the antibiotic info missing.
BreakpointLookup[whonetOrganismCode][guideline][guidelineYear].Add(whonetAntimicrobialFullCode, mostApplicableBreakpoint);
}
else
{
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, mostApplicableBreakpoint);

BreakpointLookup[whonetOrganismCode][guideline].Add(guidelineYear, abxSet);
}
}
else
{
// Create everything below the organism.
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, mostApplicableBreakpoint);

Dictionary<int, Dictionary<string, Breakpoint>> yearSet = new Dictionary<int, Dictionary<string, Breakpoint>>();
yearSet.Add(guidelineYear, abxSet);

BreakpointLookup[whonetOrganismCode].Add(guideline, yearSet);
}
}
else
{
// The organism key missing, which means we have to create the whole structure.
Dictionary<string, Breakpoint> abxSet = new Dictionary<string, Breakpoint>();
abxSet.Add(whonetAntimicrobialFullCode, mostApplicableBreakpoint);

Dictionary<int, Dictionary<string, Breakpoint>> yearSet = new Dictionary<int, Dictionary<string, Breakpoint>>();
yearSet.Add(guidelineYear, abxSet);

Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>> guidelineSet = new Dictionary<string, Dictionary<int, Dictionary<string, Breakpoint>>>();
guidelineSet.Add(guideline, yearSet);

BreakpointLookup.Add(whonetOrganismCode, guidelineSet);
}

return mostApplicableBreakpoint;
}

#endregion

}
Expand Down
53 changes: 34 additions & 19 deletions Interpretation Engine/Breakpoint.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ public static string[] DefaultOrder()
public readonly string BREAKPOINT_TYPE;
public readonly string HOST;
public readonly string SITE_OF_INFECTION;
public readonly string[] SITES_OF_INFECTION;
public readonly string REFERENCE_TABLE;
public readonly string WHONET_ABX_CODE;
public readonly string WHONET_TEST;
Expand Down Expand Up @@ -162,6 +163,20 @@ public static string[] DefaultOrder()
BREAKPOINT_TYPE = BREAKPOINT_TYPE_;
HOST = HOST_;
SITE_OF_INFECTION = SITE_OF_INFECTION_;

SITES_OF_INFECTION = IO_Library.SplitLine(
SITE_OF_INFECTION_,
Constants.Delimiters.CommaChar).
Select(s => {
string nonBlankSite = s.Trim();
// Process (Blank) as the empty string.
if (nonBlankSite == Constants.SitesOfInfection.Blank)
nonBlankSite = string.Empty;
return nonBlankSite;
}).ToArray();

REFERENCE_TABLE = REFERENCE_TABLE_;
WHONET_ABX_CODE = WHONET_ABX_CODE_;
WHONET_TEST = WHONET_TEST_;
Expand Down Expand Up @@ -197,6 +212,9 @@ public static string[] DefaultOrder()
List<string> prioritizedSitesOfInfection = null, List<string> prioritizedWhonetAbxFullDrugCodes = null,
bool returnFirstBreakpointOnly = false)
{
if (prioritizedWhonetAbxFullDrugCodes != null && prioritizedWhonetAbxFullDrugCodes.Count() > 1 && returnFirstBreakpointOnly)
throw new ArgumentException("There must be exactly one drug specified when only the first breakpoint is requested.");

if (!Organism.CurrentOrganisms.ContainsKey(whonetOrganismCode))
{
if (Organism.MergedOrganisms.ContainsKey(whonetOrganismCode) && Organism.CurrentOrganisms.ContainsKey(Organism.MergedOrganisms[whonetOrganismCode]))
Expand Down Expand Up @@ -224,23 +242,14 @@ public static string[] DefaultOrder()
// Returns an ordered list of breakpoints according to a default order (most specific first),
// or one that the caller specified through prioritized parameters.
// Find all matches on ORGANISM_CODE_TYPE, but sort them by specificity.

IEnumerable<Breakpoint> relevantBreakpoints =
from Breakpoint thisBreakpoint in Breakpoints.Concat(userDefinedBreakpoints)
where prioritizedGuidelineYears is null || prioritizedGuidelineYears.Contains(thisBreakpoint.YEAR)
where prioritizedGuidelines is null || prioritizedGuidelines.Contains(thisBreakpoint.GUIDELINES) || thisBreakpoint.GUIDELINES == UserDefinedGuidelineCode
where prioritizedBreakpointTypes is null || prioritizedBreakpointTypes.Contains(thisBreakpoint.BREAKPOINT_TYPE)
where prioritizedSitesOfInfection.Any((requestedSite) =>
IO_Library.SplitLine(thisBreakpoint.SITE_OF_INFECTION, Constants.Delimiters.CommaChar).
Select((sitesFromBP) => sitesFromBP.Trim()).
Any((sitesFromBP) =>
{
// Process (Blank) as the empty string.
if (requestedSite == Constants.SitesOfInfection.Blank)
requestedSite = string.Empty;
return requestedSite.Equals(sitesFromBP, StringComparison.InvariantCultureIgnoreCase);
}))
thisBreakpoint.SITES_OF_INFECTION.
Any(sitesFromBP => requestedSite.Equals(sitesFromBP, StringComparison.InvariantCultureIgnoreCase)))
where recodedDrugCodes is null || recodedDrugCodes.Contains(thisBreakpoint.WHONET_TEST)
where (
// This section restricts to only those breakpoints which match our organism at some level.
Expand Down Expand Up @@ -295,6 +304,19 @@ public static string[] DefaultOrder()
// We have found the relevant breakpoints. We must now filter them down to only the applicable breakpoints.
List<Breakpoint> applicableBreakpoints = new List<Breakpoint>();

if (returnFirstBreakpointOnly)
{
Breakpoint top = relevantBreakpoints.FirstOrDefault();

// No breakpoints available.
if (top == null)
return applicableBreakpoints;

// No need to put the breakpoints into groups prior to selecting the top breakpoint.
applicableBreakpoints.Add(top);
return applicableBreakpoints;
}

// Group the relevant breakpoints by the following fields.
// We want to take the most specific breakpoints for each set.
var groups = relevantBreakpoints.GroupBy(
Expand Down Expand Up @@ -329,14 +351,7 @@ public static string[] DefaultOrder()
}
}

if (returnFirstBreakpointOnly && applicableBreakpoints.Count > 1)
{
// Get the "default" breakpoint only.
applicableBreakpoints.RemoveRange(1, applicableBreakpoints.Count - 1);
return applicableBreakpoints;
}
else
return applicableBreakpoints;
return applicableBreakpoints;
}

/// <summary>
Expand Down

0 comments on commit 2264d9c

Please sign in to comment.