Skip to content

Commit

Permalink
More unit tests for track significance.
Browse files Browse the repository at this point in the history
  • Loading branch information
@janothan
janothan committed Mar 30, 2021
1 parent ae5c7e4 commit 5fc810e
Show file tree
Hide file tree
Showing 2 changed files with 150 additions and 45 deletions.
67 changes: 44 additions & 23 deletions ...nformatik/dws/melt/matching_eval/evaluator/significance/EvaluatorMcNemarSignificance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,17 @@ public class EvaluatorMcNemarSignificance extends Evaluator {
protected double alpha;

/**
* If we calculate the significance for a complete track, the crossTrackSignificanceShare defines the share
* If we calculate the significance for a complete track, the {@code trackSignificanceShare} defines the share
* of test case alignments that need to be significantly different so that the track counts as significantly
* different.
*
* For example: Given a track has 10 test cases and a crossTrackSignificanceShare of 0.5.
* If 4 alignments are significantly different, the track counts as not significant.
* If 5 or more alignments are significantly different, the track counts as significant.
* <p>
* For example: Given a track has 10 test cases and a {@code trackSignificanceShare} of 0.5.
* If 4 alignments are significantly different, the track counts as not significant for the matcher combination.
* If 5 or more alignments are significantly different, the track counts as significant for the matcher combination.
*/
private double crossTrackSignificanceShare = 0.5;
private double trackSignificanceShare = DEFAULT_TRACK_SIGNIFICANCE_SHARE;

public static final double DEFAULT_TRACK_SIGNIFICANCE_SHARE = 0.5;


// Default file names (files will be created in baseDirectory.
Expand Down Expand Up @@ -184,10 +186,11 @@ public void writeResultsToDirectory(File baseDirectory) {

/**
* Write the results file on the granularity of matchers.
* @param pValues The p values.
*
* @param pValues The p values.
* @param fileToWrite The file that shall be written.
*/
private void writeAggregatedTestcasesResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite){
private void writeAggregatedTestcasesResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite) {
try {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileToWrite),
StandardCharsets.UTF_8));
Expand All @@ -207,10 +210,11 @@ private void writeAggregatedTestcasesResultFile(Map<McNemarIndividualResult, Dou

/**
* Given a Map with p values, aggregate the number of significant/not significant alignments.
*
* @param pValues P values.
* @return Map with aggregated significance counts.
*/
private Map<McNemarCrossTrackResult, SignificanceCount> calculateAggregatedTestCaseMap(Map<McNemarIndividualResult, Double> pValues){
private Map<McNemarCrossTrackResult, SignificanceCount> calculateAggregatedTestCaseMap(Map<McNemarIndividualResult, Double> pValues) {
Map<McNemarCrossTrackResult, SignificanceCount> resultMap = new HashMap<>();
for (Map.Entry<McNemarIndividualResult, Double> entry : pValues.entrySet()) {
McNemarCrossTrackResult crossTrackResult = entry.getKey().getCrossTrackResult();
Expand All @@ -227,10 +231,11 @@ private Map<McNemarCrossTrackResult, SignificanceCount> calculateAggregatedTestC

/**
* Write the results file on the granularity of matchers.
* @param pValues The p values.
*
* @param pValues The p values.
* @param fileToWrite The file that shall be written.
*/
private void writeAggregatedTracksResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite){
private void writeAggregatedTracksResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite) {
try {
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileToWrite),
StandardCharsets.UTF_8));
Expand All @@ -240,22 +245,22 @@ private void writeAggregatedTracksResultFile(Map<McNemarIndividualResult, Double
Map<McNemarCrossTrackResult, SignificanceCount> resultMap = new HashMap<>();
Set<String> tracks = McNemarIndividualResult.getDistinctTracks(pValues);

for(String track : tracks){
for (String track : tracks) {
Map<McNemarIndividualResult, Double> trackResults =
McNemarIndividualResult.getEntriesForTrack(pValues, track);
int numberOfTestCases = McNemarIndividualResult.getNumberOfTestCases(trackResults);
Map<McNemarCrossTrackResult, SignificanceCount> testCaseResults = calculateAggregatedTestCaseMap(pValues);

for(Map.Entry<McNemarCrossTrackResult, SignificanceCount> entry : testCaseResults.entrySet()){
for (Map.Entry<McNemarCrossTrackResult, SignificanceCount> entry : testCaseResults.entrySet()) {
// let's determine whether we have a share of significant test case alignments which is large enough
double shareSignificantlyDifferentTCs =
(double) entry.getValue().significantlyDifferent / numberOfTestCases;
boolean isSignificantlyDifferent = shareSignificantlyDifferentTCs >= crossTrackSignificanceShare;
boolean isSignificantlyDifferent = shareSignificantlyDifferentTCs >= trackSignificanceShare;

if(resultMap.containsKey(entry.getKey())){
if (resultMap.containsKey(entry.getKey())) {
// we have an entry
SignificanceCount count = resultMap.get(entry.getKey());
if(isSignificantlyDifferent){
if (isSignificantlyDifferent) {
count.significantlyDifferent++;
} else {
count.notSignificantlyDifferent++;
Expand All @@ -264,7 +269,7 @@ private void writeAggregatedTracksResultFile(Map<McNemarIndividualResult, Double
} else {
// we do not have an entry yet
SignificanceCount count;
if(isSignificantlyDifferent){
if (isSignificantlyDifferent) {
count = new SignificanceCount(Significance.SIGNIFICANT);
} else {
count = new SignificanceCount(Significance.NOT_SIGNIFICANT);
Expand All @@ -289,7 +294,8 @@ private void writeAggregatedTracksResultFile(Map<McNemarIndividualResult, Double

/**
* Write the results file on the granularity of tracks.
* @param pValues The p values.
*
* @param pValues The p values.
* @param fileToWrite The file that shall be written.
*/
private void writeTrackResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite) {
Expand Down Expand Up @@ -323,7 +329,8 @@ private void writeTrackResultFile(Map<McNemarIndividualResult, Double> pValues,

/**
* Write the results file on the granularity of test cases.
* @param pValues The p values.
*
* @param pValues The p values.
* @param fileToWrite The file that shall be written.
*/
private void writeTestCaseResultFile(Map<McNemarIndividualResult, Double> pValues, File fileToWrite) {
Expand All @@ -334,7 +341,7 @@ private void writeTestCaseResultFile(Map<McNemarIndividualResult, Double> pValue
for (Map.Entry<McNemarIndividualResult, Double> entry : pValues.entrySet()) {

String isSignificantlyDifferentString;
if(entry.getValue().isNaN()){
if (entry.getValue().isNaN()) {
isSignificantlyDifferentString = "<undefined>";
} else {
isSignificantlyDifferentString = "" + (entry.getValue() < this.alpha);
Expand All @@ -351,12 +358,12 @@ private void writeTestCaseResultFile(Map<McNemarIndividualResult, Double> pValue
public Map<McNemarIndividualResult, Double> calculatePvalues(double alpha, TestType testType) {
Map<McNemarIndividualResult, Double> result = new HashMap<>();
for (ExecutionResult result1 : results) {
if(result1.getRefinements().size() > 0){
if (result1.getRefinements().size() > 0) {
// for now we only work with raw results
continue;
}
for (ExecutionResult result2 : results) {
if(result2.getRefinements().size() > 0){
if (result2.getRefinements().size() > 0) {
// for now we only work with raw results
continue;
}
Expand Down Expand Up @@ -456,7 +463,7 @@ private double pValueConsideringFalsePositives(ExecutionResult executionResult1,
if (Double.isNaN(resultAsymptotic)) {
return pValueConsideringFalsePositives(executionResult1, executionResult2, TestType.EXACT);
} else return resultAsymptotic;
} else if (testType == ASYMPTOTIC_CONTINUITY_CORRECTION_EXACT_FALLBACK){
} else if (testType == ASYMPTOTIC_CONTINUITY_CORRECTION_EXACT_FALLBACK) {
double resultAsymptoticCCorrection = pValueConsideringFalsePositives(executionResult1, executionResult2,
TestType.ASYMPTOTIC_CONTINUITY_CORRECTION);
if (Double.isNaN(resultAsymptoticCCorrection)) {
Expand Down Expand Up @@ -516,4 +523,18 @@ static long fact(int n) {
}
return res;
}

public double getTrackSignificanceShare() {
return trackSignificanceShare;
}

public void setTrackSignificanceShare(double trackSignificanceShare) {
if (trackSignificanceShare >= 0 && trackSignificanceShare <= 1.0) {
this.trackSignificanceShare = trackSignificanceShare;
} else {
LOGGER.error("The trackSignificanceShare is not in the boundaries [0, 1]. Using the default: " +
DEFAULT_TRACK_SIGNIFICANCE_SHARE);
this.trackSignificanceShare = DEFAULT_TRACK_SIGNIFICANCE_SHARE;
}
}
}
Loading

0 comments on commit 5fc810e

Please sign in to comment.