Unsupervised evaluation. New keys: answerSubset and answer.

Makefile

@@ -1069,3 +1069,51 @@ unsupervised_first_experiment:
     -trainingCorpora "data/unsupervised/training/unsupervised_parser.json.noDeps.gz" \
     -logFile ../working/unsupervised_first_experiment/business_film_people.log.txt \
     > ../working/unsupervised_first_experiment/business_film_people.txt
+
+tacl_unsupervised_declarative_testing_loaded_model:
+	mkdir -p working/tacl_unsupervised_declarative_testing_loaded_model
+	java -Xms2048m -cp lib/*:graph-parser.jar in.sivareddy.graphparser.cli.RunGraphToQueryTrainingMain \
+	-schema data/freebase/schema/business_film_people_schema.txt \
+	-relationTypesFile data/freebase/stats/business_film_people_relation_types.txt \
+	-cachedKB data/freebase/domain_facts/business_facts.txt.gz \
+	-lexicon data/dummy.txt \
+	-domain "http://business.freebase.com;http://film.freebase.com;http://people.freebase.com" \
+	-nthreads 20 \
+	-nBestTrainSyntacticParses 1 \
+	-nBestTestSyntacticParses 1 \
+	-nbestGraphs 100 \
+	-useSchema true \
+	-useKB true \
+	-groundFreeVariables true \
+	-useEmptyTypes false \
+	-ignoreTypes false \
+	-urelGrelFlag true \
+	-urelPartGrelPartFlag false \
+	-utypeGtypeFlag true \
+	-gtypeGrelFlag false \
+	-wordGrelPartFlag false \
+	-wordBigramGrelPartFlag false \
+	-argGrelPartFlag false \
+	-stemMatchingFlag true \
+	-mediatorStemGrelPartMatchingFlag true \
+	-argumentStemMatchingFlag true \
+	-argumentStemGrelPartMatchingFlag true \
+	-graphIsConnectedFlag false \
+	-graphHasEdgeFlag true \
+	-countNodesFlag false \
+	-edgeNodeCountFlag false \
+	-duplicateEdgesFlag true \
+	-grelGrelFlag true \
+	-useLexiconWeightsRel true \
+	-useLexiconWeightsType true \
+	-validQueryFlag true \
+	-initialEdgeWeight 1.0 \
+	-initialTypeWeight -2.0 \
+	-initialWordWeight -0.05 \
+	-stemFeaturesWeight 0.0 \
+	-endpoint kinloch \
+	-loadModelFromFile test_data/tacl_unsupervised.model \
+	-testFile test_data/declarative_evaluation.txt \
+	-logFile working/tacl_unsupervised_declarative_testing_loaded_model/business_film_people.log.txt \
+	> working/tacl_unsupervised_declarative_testing_loaded_model/business_film_people.txt
+

commands.sh

@@ -1,3 +1,6 @@
+make deplambda_mwg_train
+make deplambda_supervised_with_unsupervised_lexicon_loaded_model_dev
+make deplambda_supervised_with_unsupervised_lexicon_loaded_model_train
 make tacl_unsupervised_free917 2> ../working/tacl_unsupervised_free917.errors.txt
 make deplambda_unsupervised_free917 2> ../working/deplambda_unsupervised_free917.errors.txt
 make deplambda_mwg 2> ../working/deplambda_mwg.errors.txt

lib_data/lexicon_specialCases.txt

@@ -41,3 +41,6 @@ that	*	S[em]/S[dcl]	(S[em]{Y}/S[dcl]{Y}){_};_ 1 Y	CLOSED
 # Artificial possessive - Over generation of ungrounded graphs - Comment it for public usage
 's	IPOS	(NP[nb]/N)\NP	((NP[nb]{Y}/N{Y}){_}\NP{Z}){_};_ 1 Y,_ 2 Z	(lambda $f1 $f2 $Y (exists $Z (and (COPULA $Y $Z) (TYPE $Y $Z))))
 's	IPOS	(NP[nb]/N)\NP	((NP[nb]{Y}/N{Y}){_}\NP{Z}){_};_ 1 Y,_ 2 Z	EVENT
+
+# BLANK category
+BLANK	*	N	N{X}	(lambda $X (QUESTION $X))

lib_data/lexicon_specialCases_questions.txt

@@ -70,3 +70,6 @@ be	*	(S[dcl]\NP)/(S[adj]\NP)	((S[dcl]{Z}\NP{Y}){_}/(S[adj]{Z}\NP{Y*}){Z}){_};_ 1
 # Artificial possessive - Over generation of ungrounded graphs - Comment it for public usage
 's	IPOS	(NP[nb]/N)\NP	((NP[nb]{Y}/N{Y}){_}\NP{Z}){_};_ 1 Y,_ 2 Z	(lambda $f1 $f2 $Y (exists $Z (and (COPULA $Y $Z) (TYPE $Y $Z))))
 's	IPOS	(NP[nb]/N)\NP	((NP[nb]{Y}/N{Y}){_}\NP{Z}){_};_ 1 Y,_ 2 Z	EVENT
+
+# BLANK category
+BLANK	*	N	N{X}	(lambda $X (QUESTION $X))

src/in/sivareddy/graphparser/learning/GraphToQueryTraining.java

@@ -1358,9 +1358,11 @@ private void testCurrentModelSentence(JsonObject jsonSentence, Logger logger,
       Map<Integer, Integer> firstBestMap, List<Integer> testingNbestParsesRange) {
     boolean debugEnabled = logger.isDebugEnabled();
     boolean foundAnswer = false;
-    Preconditions.checkArgument(
-        jsonSentence.has("sparqlQuery") || jsonSentence.has("targetValue"),
-        "Test sentence should either have a gold query or targetValue");
+    Preconditions
+        .checkArgument(
+            jsonSentence.has("sparqlQuery") || jsonSentence.has("targetValue")
+                || jsonSentence.has("answerSubset") || jsonSentence.has("answer"),
+            "Test sentence should either have a gold query or targetValue or answer values");
     String sentence = jsonSentence.get("sentence").getAsString();
 
     logger.info("Sentence " + sentCount + ": " + sentence);
@@ -1371,7 +1373,6 @@ private void testCurrentModelSentence(JsonObject jsonSentence, Logger logger,
       logger.info("Gold Query : " + goldQuery);
       goldResults = rdfGraphTools.runQueryHttp(goldQuery);
       logger.info("Gold Results : " + goldResults);
-
     } else if (jsonSentence.has("targetValue")) {
       String goldAnswersString = jsonSentence.get("targetValue").getAsString();
       Pattern goldAnswerPattern =
@@ -1383,6 +1384,18 @@ private void testCurrentModelSentence(JsonObject jsonSentence, Logger logger,
       }
       goldResults = new HashMap<>();
       goldResults.put("targetValue", goldAnswers);
+    } else if (jsonSentence.has("answer")) {
+      JsonArray goldAnswersArray = jsonSentence.get("answer").getAsJsonArray();
+      LinkedHashSet<String> goldAnswers = new LinkedHashSet<>();
+      goldAnswersArray.forEach(answer -> goldAnswers.add(answer.getAsString()));
+      goldResults = new HashMap<>();
+      goldResults.put("answer", goldAnswers);
+    } else if (jsonSentence.has("answerSubset")) {
+      JsonArray goldAnswersArray = jsonSentence.get("answerSubset").getAsJsonArray();
+      LinkedHashSet<String> goldAnswers = new LinkedHashSet<>();
+      goldAnswersArray.forEach(answer -> goldAnswers.add(answer.getAsString()));
+      goldResults = new HashMap<>();
+      goldResults.put("answerSubset", goldAnswers);
     }
 
     // Get ungrounded graphs

src/in/sivareddy/graphparser/util/RdfGraphTools.java

@@ -1,5 +1,6 @@
 package in.sivareddy.graphparser.util;
 
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -283,6 +284,21 @@ public static Pair<Set<String>, Set<String>> getCleanedResults(
         }
         predAnswersCleaned.add(predAnswer);
       }
+    } else if (goldVar.equals("answerSubset") || goldVar.equals("answer")) {
+      LinkedHashSet<String> predAnswers = predResults.get(predVar);
+      for (String predAnswer : predAnswers) {
+        boolean answerIsDate = predAnswer.contains("XMLSchema#datetime");
+        predAnswer = predAnswer.split("\\^\\^")[0];
+        String[] answers = predAnswer.split("/");
+        predAnswer = answers[answers.length - 1];
+        if (answerIsDate) {
+          Matcher matcher = Pattern.compile("([0-9]{3,4})").matcher(predAnswer);
+          if (matcher.find()) {
+            predAnswer = matcher.group(1);
+          }
+        }
+        predAnswersCleaned.add(predAnswer);
+      }
     } else {
       predAnswersCleaned = predResults.get(predVar);
     }
@@ -366,6 +382,27 @@ public static boolean equalResults(
         return false;
       }
       return predAnswersCleaned.equals(goldAnswers);
+    } else if (goldVar.equals("answerSubset") || goldVar.equals("answer")) {
+      // If the gold answers are subset of the predicted answers, return true.
+      HashSet<String> predAnswersCleaned = new HashSet<>();
+      LinkedHashSet<String> predAnswers = predResults.get(predVar);
+      for (String predAnswer : predAnswers) {
+        boolean answerIsDate = predAnswer.contains("XMLSchema#datetime");
+        predAnswer = predAnswer.split("\\^\\^")[0];
+        String[] answers = predAnswer.split("/");
+        predAnswer = answers[answers.length - 1];
+        if (answerIsDate) {
+          Matcher matcher = Pattern.compile("([0-9]{3,4})").matcher(predAnswer);
+          if (matcher.find()) {
+            predAnswer = matcher.group(1);
+          }
+        }
+        predAnswersCleaned.add(predAnswer);
+      }
+      if (goldVar.equals("answerSubset"))
+        return predAnswersCleaned.containsAll(goldResults.get(goldVar));
+      else
+        return predAnswersCleaned.equals(goldResults.get(goldVar));
     } else {
       return goldResults.get(goldVar).equals(predResults.get(predVar));
     }

test_data/declarative_evaluation.txt

@@ -0,0 +1,5 @@
+{"entities":[{"index":0,"entity":"m.08z129"},{"index":2,"entity":"m.0by5r3"}],"words":[{"ner":"O","word":"SBC_Communications","pos":"NNP"},{"ner":"O","word":"acquired","pos":"VBD"},{"ner":"O","word":"Sterling_Commerce","pos":"NNP"},{"ner":"O","word":"in","pos":"IN"},{"ner":"DATE","word":"BLANK","pos":"NN"},{"ner":"O","word":".","pos":"."}],"synPars":[{"synPar":"(\u003cT S[dcl] rp 0 2\u003e (\u003cT S[dcl] ba 1 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N SBC_Communications SBC_Communications NNP I-ORG I-NP N\u003e)) (\u003cT S[dcl]\\NP ba 0 2\u003e (\u003cT S[dcl]\\NP fa 0 2\u003e (\u003cL (S[dcl]\\NP)/NP acquired acquire VBD O I-VP (S[dcl]\\NP)/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N Sterling_Commerce Sterling_Commerce NNP I-ORG I-NP N\u003e))) (\u003cT (S\\NP)\\(S\\NP) fa 0 2\u003e (\u003cL ((S\\NP)\\(S\\NP))/NP in in IN O I-PP ((S[X]\\NP)\\(S[X]\\NP))/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N BLANK BLANK NN I-DAT I-NP N\u003e))))) (\u003cL . . . . O O .\u003e))","score":20.387499999999999}],"sentence":"SBC_Communications acquired Sterling_Commerce in BLANK .", "answer":["2000"]}
+{"entities":[{"index":0,"entity":"m.08z129"}],"words":[{"ner":"O","word":"SBC_Communications","pos":"NNP"},{"ner":"O","word":"acquired","pos":"VBD"},{"ner":"O","word":"BLANK","pos":"NN"},{"ner":"O","word":"in","pos":"IN"},{"ner":"DATE","word":"2000","pos":"NNP"},{"ner":"O","word":".","pos":"."}],"synPars":[{"synPar":"(\u003cT S[dcl] rp 0 2\u003e (\u003cT S[dcl] ba 1 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N SBC_Communications SBC_Communications NNP I-ORG I-NP N\u003e)) (\u003cT S[dcl]\\NP ba 0 2\u003e (\u003cT S[dcl]\\NP fa 0 2\u003e (\u003cL (S[dcl]\\NP)/NP acquired acquire VBD O I-VP (S[dcl]\\NP)/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N BLANK BLANK NN O I-NP N\u003e))) (\u003cT (S\\NP)\\(S\\NP) fa 0 2\u003e (\u003cL ((S\\NP)\\(S\\NP))/NP in in IN O I-PP ((S[X]\\NP)\\(S[X]\\NP))/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N 2000 2000 NNP I-DAT I-NP N\u003e))))) (\u003cL . . . . O O .\u003e))","score":20.387499999999999}],"sentence":"SBC_Communications acquired BLANK in March_2000 .", "answerSubset":["m.0by5r3"]}
+{"entities":[{"index":2,"entity":"m.0by5r3"}],"words":[{"ner":"O","word":"BLANK","pos":"NN"},{"ner":"O","word":"acquired","pos":"VBD"},{"ner":"O","word":"Sterling_Commerce","pos":"NNP"},{"ner":"O","word":"in","pos":"IN"},{"ner":"DATE","word":"2000","pos":"NNP"},{"ner":"O","word":".","pos":"."}],"synPars":[{"synPar":"(\u003cT S[dcl] rp 0 2\u003e (\u003cT S[dcl] ba 1 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N BLANK BLANK NN O I-NP N\u003e)) (\u003cT S[dcl]\\NP ba 0 2\u003e (\u003cT S[dcl]\\NP fa 0 2\u003e (\u003cL (S[dcl]\\NP)/NP acquired acquire VBD O I-VP (S[dcl]\\NP)/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N Sterling_Commerce Sterling_Commerce NNP I-ORG I-NP N\u003e))) (\u003cT (S\\NP)\\(S\\NP) fa 0 2\u003e (\u003cL ((S\\NP)\\(S\\NP))/NP in in IN O I-PP ((S[X]\\NP)\\(S[X]\\NP))/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N 2000 2000 NNP I-DAT I-NP N\u003e))))) (\u003cL . . . . O O .\u003e))","score":20.387499999999999}],"sentence":"BLANK acquired Sterling_Commerce in March_2000 .", "answerSubset":["m.08z129"]}
+{"entities":[{"index":4,"entity":"m.0gjpq"},{"index":6,"score":1.0,"entity":"m.0gjq6"}],"words":[{"ner":"O","word":"BLANK","pos":"NN"},{"ner":"O","word":"was","pos":"VBD"},{"ner":"O","word":"founded","pos":"VBN"},{"ner":"O","word":"by","pos":"IN"},{"ner":"O","word":"Larry_Page","pos":"NNP"},{"ner":"O","word":"and","pos":"CC"},{"ner":"O","word":"Sergey_Brin","pos":"NNP"},{"ner":"O","word":".","pos":"."}],"synPars":[{"synPar":"(\u003cT S[dcl] rp 0 2\u003e (\u003cT S[dcl] ba 1 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N BLANK BLANK NN O I-NP N\u003e)) (\u003cT S[dcl]\\NP fa 0 2\u003e (\u003cL (S[dcl]\\NP)/(S[pss]\\NP) was be VBD O I-VP (S[dcl]\\NP)/(S[pss]\\NP)\u003e) (\u003cT S[pss]\\NP ba 0 2\u003e (\u003cL S[pss]\\NP founded found VBN O I-VP S[pss]\\NP\u003e) (\u003cT (S\\NP)\\(S\\NP) fa 0 2\u003e (\u003cL ((S\\NP)\\(S\\NP))/NP by by IN O I-PP ((S[X]\\NP)\\(S[X]\\NP))/NP\u003e) (\u003cT NP ba 0 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N Larry_Page Larry_Page NNP I-ORG I-NP N\u003e)) (\u003cT NP[conj] conj 0 2\u003e (\u003cL conj and and CC I-ORG I-NP conj\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N Sergey_Brin Sergey_Brin NNP I-ORG I-NP N\u003e)))))))) (\u003cL . . . . O O .\u003e))","score":29.345600000000001}],"sentence":"BLANK was founded by Larry_Page and Sergey_Brin .","answer":["m.045c7b"]}
+{"entities":[{"index":0,"score":0.99757700000000005,"entity":"m.03_x5t"},{"index":4,"score":0.99294700000000002,"entity":"m.0fldjy"}],"words":[{"ner":"O","word":"Eva_Mendes","pos":"NNP"},{"ner":"O","word":"plays","pos":"VBZ"},{"ner":"O","word":"BLANK","pos":"NN"},{"ner":"O","word":"in","pos":"IN"},{"ner":"O","word":"The_Spirit","pos":"NNP"},{"ner":"O","word":".","pos":"."}],"synPars":[{"synPar":"(\u003cT S[dcl] rp 0 2\u003e (\u003cT S[dcl] ba 1 2\u003e (\u003cT NP lex 0 1\u003e (\u003cL N Eva_Mendes Eva_Mendes NNP I-ORG I-NP N\u003e)) (\u003cT S[dcl]\\NP ba 0 2\u003e (\u003cT S[dcl]\\NP fa 0 2\u003e (\u003cL (S[dcl]\\NP)/NP plays play VBZ O I-VP (S[dcl]\\NP)/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N BLANK BLANK NN O I-NP N\u003e))) (\u003cT (S\\NP)\\(S\\NP) fa 0 2\u003e (\u003cL ((S\\NP)\\(S\\NP))/NP in in IN O I-PP ((S[X]\\NP)\\(S[X]\\NP))/NP\u003e) (\u003cT NP lex 0 1\u003e (\u003cL N The_Spirit The_Spirit NNP I-LOC I-NP N\u003e))))) (\u003cL . . . . O O .\u003e))","score":23.3507}],"sentence":"Eva_Mendes plays BLANK in The_Spirit .", "answer":["m.043mtzk"]}