Merge branch 'master' of github.com:dwslab/melt

dwslab · Aug 12, 2020 · 5bcaeb0 · 5bcaeb0
2 parents f189846 + 4bbb297
commit 5bcaeb0
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 26 deletions.
diff --git a/matching-eval/src/main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/Executor.java b/matching-eval/src/main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/Executor.java
@@ -134,9 +134,9 @@ public static ExecutionResultSet runTracks(List<Track> tracks, Map<String, IOnto
             return new ExecutionResultSet();
         }
         ExecutionResultSet r = new ExecutionResultSet();
-        for(Entry<String, IOntologyMatchingToolBridge> matcher : matchers.entrySet()){
-            for(Track track : tracks){
-                for (TestCase tc : track.getTestCases()) {
+        for(Track track : tracks){
+            for (TestCase tc : track.getTestCases()) {
+                for(Entry<String, IOntologyMatchingToolBridge> matcher : matchers.entrySet()){
                     ExecutionResult er = ExecutionRunner.runMatcher(tc, matcher.getValue(), matcher.getKey());
                     if(er != null)
                         r.add(er);
@@ -163,8 +163,8 @@ public static ExecutionResultSet run(List<TestCase> testCases, Map<String, IOnto
             return new ExecutionResultSet();
         }
         ExecutionResultSet r = new ExecutionResultSet();
-        for (Entry<String, IOntologyMatchingToolBridge> matcher : matchers.entrySet()) {
-            for (TestCase tc : testCases) {
+        for (TestCase tc : testCases) {
+            for (Entry<String, IOntologyMatchingToolBridge> matcher : matchers.entrySet()) {
                 ExecutionResult er = ExecutionRunner.runMatcher(tc, matcher.getValue(), matcher.getKey());
                 if(er != null)
                     r.add(er);

diff --git a/.../main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/evaluator/EvaluatorRank.java b/.../main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/evaluator/EvaluatorRank.java
@@ -24,32 +24,22 @@ public class EvaluatorRank extends Evaluator{
 
     private static final Logger LOGGER = LoggerFactory.getLogger(EvaluatorRank.class);
 
-    private boolean partialGoldStandard;
 
     /**
-     * Constructor which needs the results as well as the info if the gold standard for al testcases is partial or not.
+     * Constructor
      * @param results the execution result set
-     * @param partialGoldStandard true if testcase in this executionresultset is a partial gold standard
      */
-    public EvaluatorRank(ExecutionResultSet results, boolean partialGoldStandard) {
+    public EvaluatorRank(ExecutionResultSet results) {
         super(results);
-        this.partialGoldStandard = partialGoldStandard;
     }
 
-    /**
-     * Initialize the evaluator with a full gold standard (not partial).
-     * @param results the execution result set
-     */
-    public EvaluatorRank(ExecutionResultSet results) {
-        this(results, false);
-    }
 
     @Override
     public void writeResultsToDirectory(File baseDirectory) {
 
-        RankingMetric random = new RankingMetric(partialGoldStandard, SameConfidenceRanking.RANDOM);
-        RankingMetric alphabetically = new RankingMetric(partialGoldStandard, SameConfidenceRanking.ALPHABETICALLY);
-        RankingMetric top = new RankingMetric(partialGoldStandard, SameConfidenceRanking.TOP);
+        RankingMetric random = new RankingMetric(SameConfidenceRanking.RANDOM);
+        RankingMetric alphabetically = new RankingMetric(SameConfidenceRanking.ALPHABETICALLY);
+        RankingMetric top = new RankingMetric(SameConfidenceRanking.TOP);
 
         try {
             if(!baseDirectory.exists()){

diff --git a/...ni_mannheim/informatik/dws/melt/matching_eval/evaluator/metric/ranking/RankingMetric.java b/...ni_mannheim/informatik/dws/melt/matching_eval/evaluator/metric/ranking/RankingMetric.java
@@ -2,6 +2,7 @@
 
 import de.uni_mannheim.informatik.dws.melt.matching_eval.ExecutionResult;
 import de.uni_mannheim.informatik.dws.melt.matching_eval.evaluator.metric.Metric;
+import de.uni_mannheim.informatik.dws.melt.matching_eval.evaluator.metric.cm.GoldStandardCompleteness;
 import de.uni_mannheim.informatik.dws.melt.yet_another_alignment_api.Alignment;
 import de.uni_mannheim.informatik.dws.melt.yet_another_alignment_api.Correspondence;
 import java.util.ArrayList;
@@ -18,20 +19,19 @@ public class RankingMetric extends Metric<RankingResult>{
 
     private static Logger LOGGER = LoggerFactory.getLogger(RankingMetric.class);
 
-    private static double logOf2 = Math.log(2);    
-    protected boolean partialGoldStandard;
+    private static double logOf2 = Math.log(2);
+
     protected SameConfidenceRanking sameConfidenceRanking;
 
-    public RankingMetric(boolean partialGoldStandard, SameConfidenceRanking sameConfidenceRanking){
-        this.partialGoldStandard = partialGoldStandard;
+    public RankingMetric(SameConfidenceRanking sameConfidenceRanking){
         this.sameConfidenceRanking = sameConfidenceRanking;
     }
 
     @Override
     protected RankingResult compute(ExecutionResult executionResult) {
 
         Alignment systemAlignment = executionResult.getSystemAlignment();
-        if(this.partialGoldStandard){
+        if(executionResult.getTestCase().getGoldStandardCompleteness() != GoldStandardCompleteness.COMPLETE){
             systemAlignment = getSystemResultReducedToGoldStandardEntities(executionResult);
         }
         Alignment referenceAlignment = executionResult.getReferenceAlignment();

diff --git a/...c/main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/tracks/TrackRepository.java b/...c/main/java/de/uni_mannheim/informatik/dws/melt/matching_eval/tracks/TrackRepository.java
@@ -1,6 +1,9 @@
 package de.uni_mannheim.informatik.dws.melt.matching_eval.tracks;
 
 import de.uni_mannheim.informatik.dws.melt.matching_eval.evaluator.metric.cm.GoldStandardCompleteness;
+import de.uni_mannheim.informatik.dws.melt.yet_another_alignment_api.Alignment;
+import java.io.File;
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.lang.reflect.ParameterizedType;
@@ -591,4 +594,17 @@ private static Set<Track> retriveDefinedTracks(){
 
         return tracks;
     }
+
+
+    public static TestCase generateTestCaseWithSampledReferenceAlignment(TestCase tc, double fraction){
+        Alignment sample = tc.getParsedReferenceAlignment().sampleByFraction(0.5);
+        try {
+            File f = File.createTempFile("ref_sample", ".rdf");
+            sample.serialize(f);
+            return new TestCase(tc.getName(), tc.getSource(), tc.getTarget(), tc.getReference(), tc.getTrack(), f.toURI(), tc.getGoldStandardCompleteness());
+        } catch (IOException ex) {
+            LOGGER.error("Could not write sample reference alignment to file", ex);
+            return tc;
+        }
+    }
 }
diff --git a/...annheim/informatik/dws/melt/matching_eval/evaluator/metric/ranking/RankingMetricTest.java b/...annheim/informatik/dws/melt/matching_eval/evaluator/metric/ranking/RankingMetricTest.java
@@ -35,7 +35,7 @@ void testAlphabetically() throws Exception {
         reference.add(new Correspondence("s", "t"));        
 
 
-        RankingMetric ranker = new RankingMetric(false, SameConfidenceRanking.ALPHABETICALLY);
+        RankingMetric ranker = new RankingMetric(SameConfidenceRanking.ALPHABETICALLY);
         TestCase tcDummy = TrackRepository.Anatomy.Default.getFirstTestCase();
         RankingResult result = ranker.get(new ExecutionResult(tcDummy, "TestMatcher", system, reference));