Add the f1 scores for each tree to the parser protobuf responses

stanfordnlp · Feb 18, 2024 · 2725b06 · 2725b06
1 parent a80772d
commit 2725b06
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 2 deletions.
diff --git a/src/edu/stanford/nlp/parser/metrics/EvaluateExternalParser.java b/src/edu/stanford/nlp/parser/metrics/EvaluateExternalParser.java
@@ -94,12 +94,17 @@ public List<Pair<ParserQuery, Tree>> convertDataset(List<Tree> goldTrees, List<L
   }
 
 
-  public CoreNLPProtos.EvaluateParserResponse buildResponse(double f1, Double kbestF1)  {
+  public CoreNLPProtos.EvaluateParserResponse buildResponse(double f1, Double kbestF1, List<Double> f1History)  {
     CoreNLPProtos.EvaluateParserResponse.Builder responseBuilder = CoreNLPProtos.EvaluateParserResponse.newBuilder();
     responseBuilder.setF1(f1);
     if (kbestF1 != null) {
       responseBuilder.setKbestF1(kbestF1);
     }
+    if (f1History != null) {
+      for (Double treeF1 : f1History) {
+        responseBuilder.addTreeF1(treeF1);
+      }
+    }
     CoreNLPProtos.EvaluateParserResponse response = responseBuilder.build();
     return response;
   }
@@ -116,7 +121,8 @@ public CoreNLPProtos.EvaluateParserResponse scoreDataset(List<Tree> goldTrees, L
     if (evaluator.hasPCFGTopKF1()) {
       kbestF1 = evaluator.getPCFGTopKF1();
     }
-    return buildResponse(f1, kbestF1);
+    List<Double> f1History = evaluator.getF1History();
+    return buildResponse(f1, kbestF1, f1History);
   }
 
   public CoreNLPProtos.EvaluateParserResponse processRequest(CoreNLPProtos.EvaluateParserRequest parses) throws IOException {

diff --git a/src/edu/stanford/nlp/parser/metrics/EvaluateTreebank.java b/src/edu/stanford/nlp/parser/metrics/EvaluateTreebank.java
@@ -91,6 +91,8 @@ public class EvaluateTreebank  {
   AbstractEval.ScoreEval factLL = null;
   AbstractEval kGoodLB = null;
 
+  List<Double> factLBHistory = null;
+
   BestOfTopKEval pcfgTopK = null;
   private final List<BestOfTopKEval> topKEvals = new ArrayList<>();
 
@@ -182,6 +184,7 @@ public EvaluateTreebank(Options op, Lexicon lex, ParserQueryFactory pqFactory, F
     }
     if (Boolean.parseBoolean(op.testOptions.evals.getProperty("factLB"))) {
       factLB = new Evalb("factor LP/LR", runningAverages);
+      factLBHistory = new ArrayList<>();
     }
     if (op.testOptions.evals.getProperty("factChildSpecific") != null) {
       String filter = op.testOptions.evals.getProperty("factChildSpecific");
@@ -268,6 +271,10 @@ public boolean hasPCFGTopKF1() {
     return pcfgTopK != null;
   }
 
+  public List<Double> getF1History() {
+    return Collections.unmodifiableList(factLBHistory);
+  }
+
   /**
    * Remove tree scores, so they don't print.
    * <br>
@@ -524,6 +531,7 @@ else if(pwFileOut != null) {
         //Factored parser (1best) eval
         if (factLB != null) {
           factLB.evaluate(treeFact, transGoldTree, pwErr);
+          factLBHistory.add(factLB.getLastF1());
         }
         if (factChildSpecific != null) {
           factChildSpecific.evaluate(treeFact, transGoldTree, pwErr);