Merge branch 'master' of github.com:moses-smt/mosesdecoder

moses-smt · Jun 6, 2012 · 156b5db · 156b5db
2 parents 86a1886 + 175cd57
commit 156b5db
Show file tree

Hide file tree

Showing 8 changed files with 121 additions and 224 deletions.
diff --git a/cruise-control/config.ems b/cruise-control/config.ems
@@ -8,6 +8,9 @@
 #
 working-dir = WORKDIR/ems_workdir
 
+# Giza and friends
+external-bin-dir = WORKDIR/giza-pp/bin/
+
 # specification of the language pair
 input-extension = fr
 output-extension = en

diff --git a/misc/queryPhraseTable.cpp b/misc/queryPhraseTable.cpp
@@ -18,6 +18,7 @@ int main(int argc, char **argv)
   int nscores = 5;
   std::string ttable = "";
   bool useAlignments = false;
+  bool reportCounts = false;
 
   for(int i = 1; i < argc; i++) {
     if(!strcmp(argv[i], "-n")) {
@@ -28,8 +29,11 @@ int main(int argc, char **argv)
       if(i + 1 == argc)
         usage();
       ttable = argv[++i];
-    } else if(!strcmp(argv[i], "-a"))
+    } else if(!strcmp(argv[i], "-a")) {
       useAlignments = true;
+    } else if (!strcmp(argv[i], "-c")) {
+      reportCounts = true;
+    }
     else
       usage();
   }
@@ -54,22 +58,26 @@ int main(int argc, char **argv)
     else
       ptree.GetTargetCandidates(srcphrase, tgtcands);
 
-    for(uint i = 0; i < tgtcands.size(); i++) {
-      std::cout << line << " |||";
-      for(uint j = 0; j < tgtcands[i].first.size(); j++)
-        std::cout << ' ' << *tgtcands[i].first[j];
-      std::cout << " |||";
-
-      if(useAlignments) {
-        std::cout << " " << wordAlignment[i] << " |||";
+    if (reportCounts) {
+      std::cout << line << " " << tgtcands.size() << "\n";
+    } else {
+      for(uint i = 0; i < tgtcands.size(); i++) {
+        std::cout << line << " |||";
+        for(uint j = 0; j < tgtcands[i].first.size(); j++)
+          std::cout << ' ' << *tgtcands[i].first[j];
+        std::cout << " |||";
+
+        if(useAlignments) {
+          std::cout << " " << wordAlignment[i] << " |||";
+        }
+
+        for(uint j = 0; j < tgtcands[i].second.size(); j++)
+          std::cout << ' ' << tgtcands[i].second[j];
+        std::cout << '\n';
       }
-
-      for(uint j = 0; j < tgtcands[i].second.size(); j++)
-        std::cout << ' ' << tgtcands[i].second[j];
       std::cout << '\n';
     }
 
-    std::cout << '\n';
     std::cout.flush();
   }
 }
@@ -78,6 +86,7 @@ void usage()
 {
   std::cerr << 	"Usage: queryPhraseTable [-n <nscores>] [-a] -t <ttable>\n"
             "-n <nscores>      number of scores in phrase table (default: 5)\n"
+            "-c                only report counts of entries\n"
             "-a                binary phrase table contains alignments\n"
             "-t <ttable>       phrase table\n";
   exit(1);

diff --git a/regression-testing/Jamfile b/regression-testing/Jamfile
@@ -21,12 +21,24 @@ if $(with-regtest) {
   actions reg_test_score {
     $(TOP)/regression-testing/run-test-scorer.perl --scorer=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
   }
-  reg_test score : [ glob $(TESTS)/score.* ] : ../scripts/training/phrase-extract//score : @reg_test_score ;
+  reg_test score : [ glob $(TESTS)/score.* ] : ../phrase-extract//score : @reg_test_score ;
+
+  actions reg_test_extract {
+    $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+  }
+  reg_test extract : [ glob $(TESTS)/extract.* ] : ../phrase-extract//extract : @reg_test_extract ;
+
+
+  actions reg_test_extractrules {
+    $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+  }
+  reg_test extractrules : [ glob $(TESTS)/extract-rules.* : $(TESTS)/extract-rules.hierarchical ] : ../phrase-extract//extract-rules : @reg_test_extractrules ;
+
 
   actions reg_test_mert {
-    $(TOP)/regression-testing/run-test-mert.perl --mert-dir=$(TOP)/mert --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+    $(TOP)/regression-testing/run-test-mert.perl --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
   }
-  reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//legacy : @reg_test_mert ;
+  reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//mert : @reg_test_mert ;
 
-  alias all : phrase chart mert score ;
+   alias all : phrase chart mert score extract extractrules ;
 }
diff --git a/regression-testing/run-test-extract.perl b/regression-testing/run-test-extract.perl
@@ -53,7 +53,7 @@ BEGIN
 
 if (-e $outPath)
 {
-  my $cmd = "diff --exclude=.DS_Store $outPath/ $truthPath/ | wc -l";
+  my $cmd = "diff --exclude=.DS_Store --exclude=._* $outPath/ $truthPath/ | wc -l";
   my $numDiff = `$cmd`;
 
   if ($numDiff == 0)

diff --git a/regression-testing/run-test-suite.perl b/regression-testing/run-test-suite.perl
diff --git a/scripts/generic/extract-parallel.perl b/scripts/generic/extract-parallel.perl
@@ -24,9 +24,11 @@
 my $align = $ARGV[6]; # 3rd arg of extract argument
 my $extract = $ARGV[7]; # 4th arg of extract argument
 
+my $makeTTable = 1; # whether to build the ttable extract files
 my $otherExtractArgs= "";
 for (my $i = 8; $i < $#ARGV + 1; ++$i)
 {
+  $makeTTable = 0 if $ARGV[$i] eq "--NoTTable";
   $otherExtractArgs .= $ARGV[$i] ." ";
 }
 
@@ -123,11 +125,14 @@
 
 
 @children = ();
-$pid = RunFork($catCmd);
-push(@children, $pid);
+if ($makeTTable)
+{
+  $pid = RunFork($catCmd);
+  push(@children, $pid);
 
-$pid = RunFork($catInvCmd);
-push(@children, $pid);
+  $pid = RunFork($catInvCmd);
+  push(@children, $pid);
+}
 
 my $numStr = NumStr(0);
 if (-e "$TMPDIR/extract.$numStr.o.gz")