Merge branch 'master' of github.com:moses-smt/mosesdecoder

BUILD-INSTRUCTIONS.txt

@@ -8,7 +8,7 @@ available at http://boost.org .
 
 There are several optional dependencies:
 
-GIZA++ from http://code.google.com/p/giza-pp/ is used to build phrase tables.
+GIZA++ from http://code.google.com/p/giza-pp/ is used to align words in the parallel corpus during training.
 
 Moses server requires xmlrpc-c with abyss-server.  Source is available from
 http://xmlrpc-c.sourceforge.net/.  
@@ -85,7 +85,6 @@ Building consists of running
 Common options are:
 --with-srilm=/path/to/srilm to compile the decoder with SRILM support
 --with-irstlm=/path/to/irstlm to compile the decoder with IRSTLM support
---with-giza=/path/to/giza to enable training scripts
 -jN where N is the number of CPUs
 
 --with-macports=/path/to/macports use MacPorts on Mac OS X.

Jamroot

@@ -15,10 +15,6 @@
 #Note that, like language models, this is the --prefix where the library was
 #installed, not some executable within the library.  
 #
-#--with-giza=/path/to/giza 
-#Indicates where binaries GIZA++, snt2cooc.out, and mkcls live.  
-#Builds scripts/training/train-model.perl using these paths. 
-#
 #Thread-caching malloc (optional):
 #--with-tcmalloc
 #
@@ -104,7 +100,7 @@ build-project scripts ;
 #Regression tests (only does anything if --with-regtest is passed)
 build-project regression-testing ;
 
-alias programs : lm//query lm//build_binary moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score ;
+alias programs : lm//query lm//build_binary moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor ;
 
 install-bin-libs programs ;
 install-headers headers-base : [ glob-tree *.h *.hh : jam-files dist bin lib include kenlm moses ] : . ;
@@ -115,3 +111,9 @@ alias install : prefix-bin prefix-lib headers-base headers-moses ;
 if ! [ option.get "includedir" : : $(prefix)/include ] {
   explicit install headers-base headers-moses ;
 }
+
+if [ path.exists $(TOP)/dist ] && $(prefix) != dist {
+  echo "You have a $(TOP)/dist directory, but the build system now places files directly in the root i.e. $(TOP)/bin ." ;
+  echo "To disable this message, delete $(TOP)/dist ." ;
+  echo ;
+}

scripts/ems/biconcor/Jamfile → biconcor/Jamfile

@@ -1,3 +1,2 @@
 exe biconcor : Vocabulary.cpp SuffixArray.cpp TargetCorpus.cpp Alignment.cpp Mismatch.cpp PhrasePair.cpp PhrasePairCollection.cpp biconcor.cpp base64.cpp ;
 
-install legacy : biconcor : <location>. ;

cruise-control/config.ems

@@ -8,6 +8,9 @@
 #
 working-dir = WORKDIR/ems_workdir
 
+# Giza and friends
+external-bin-dir = WORKDIR/giza-pp/bin/
+
 # specification of the language pair
 input-extension = fr
 output-extension = en

jam-files/sanity.jam

@@ -158,9 +158,11 @@ if [ option.get "git" : : "yes" ] {
 prefix = [ option.get "prefix" ] ;
 if $(prefix) {
   prefix = [ path.root $(prefix) [ path.pwd ] ] ;
+  prefix = $(prefix)$(GITTAG) ;
 } else {
   prefix = $(TOP)$(GITTAG) ;
 }
+
 rule install-bin-libs ( deps * ) {
   local bindir = [ option.get "bindir" : $(prefix)/bin ] ;
   local libdir = [ option.get "libdir" : $(prefix)/lib ] ;

misc/queryPhraseTable.cpp

@@ -18,6 +18,7 @@ int main(int argc, char **argv)
   int nscores = 5;
   std::string ttable = "";
   bool useAlignments = false;
+  bool reportCounts = false;
 
   for(int i = 1; i < argc; i++) {
     if(!strcmp(argv[i], "-n")) {
@@ -28,8 +29,11 @@ int main(int argc, char **argv)
       if(i + 1 == argc)
         usage();
       ttable = argv[++i];
-    } else if(!strcmp(argv[i], "-a"))
+    } else if(!strcmp(argv[i], "-a")) {
       useAlignments = true;
+    } else if (!strcmp(argv[i], "-c")) {
+      reportCounts = true;
+    }
     else
       usage();
   }
@@ -54,22 +58,26 @@ int main(int argc, char **argv)
     else
       ptree.GetTargetCandidates(srcphrase, tgtcands);
 
-    for(uint i = 0; i < tgtcands.size(); i++) {
-      std::cout << line << " |||";
-      for(uint j = 0; j < tgtcands[i].first.size(); j++)
-        std::cout << ' ' << *tgtcands[i].first[j];
-      std::cout << " |||";
-
-      if(useAlignments) {
-        std::cout << " " << wordAlignment[i] << " |||";
+    if (reportCounts) {
+      std::cout << line << " " << tgtcands.size() << "\n";
+    } else {
+      for(uint i = 0; i < tgtcands.size(); i++) {
+        std::cout << line << " |||";
+        for(uint j = 0; j < tgtcands[i].first.size(); j++)
+          std::cout << ' ' << *tgtcands[i].first[j];
+        std::cout << " |||";
+
+        if(useAlignments) {
+          std::cout << " " << wordAlignment[i] << " |||";
+        }
+
+        for(uint j = 0; j < tgtcands[i].second.size(); j++)
+          std::cout << ' ' << tgtcands[i].second[j];
+        std::cout << '\n';
       }
-
-      for(uint j = 0; j < tgtcands[i].second.size(); j++)
-        std::cout << ' ' << tgtcands[i].second[j];
       std::cout << '\n';
     }
 
-    std::cout << '\n';
     std::cout.flush();
   }
 }
@@ -78,6 +86,7 @@ void usage()
 {
   std::cerr << 	"Usage: queryPhraseTable [-n <nscores>] [-a] -t <ttable>\n"
             "-n <nscores>      number of scores in phrase table (default: 5)\n"
+            "-c                only report counts of entries\n"
             "-a                binary phrase table contains alignments\n"
             "-t <ttable>       phrase table\n";
   exit(1);

regression-testing/Jamfile

@@ -21,12 +21,24 @@ if $(with-regtest) {
   actions reg_test_score {
     $(TOP)/regression-testing/run-test-scorer.perl --scorer=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
   }
-  reg_test score : [ glob $(TESTS)/score.* ] : ../scripts/training/phrase-extract//score : @reg_test_score ;
+  reg_test score : [ glob $(TESTS)/score.* ] : ../phrase-extract//score : @reg_test_score ;
+
+  actions reg_test_extract {
+    $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+  }
+  reg_test extract : [ glob $(TESTS)/extract.* ] : ../phrase-extract//extract : @reg_test_extract ;
+
+
+  actions reg_test_extractrules {
+    $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+  }
+  reg_test extractrules : [ glob $(TESTS)/extract-rules.* : $(TESTS)/extract-rules.hierarchical ] : ../phrase-extract//extract-rules : @reg_test_extractrules ;
+
 
   actions reg_test_mert {
-    $(TOP)/regression-testing/run-test-mert.perl --mert-dir=$(TOP)/mert --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
+    $(TOP)/regression-testing/run-test-mert.perl --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
   }
-  reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//legacy : @reg_test_mert ;
+  reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//mert : @reg_test_mert ;
 
-  alias all : phrase chart mert score ;
+   alias all : phrase chart mert score extract extractrules ;
 }

regression-testing/run-test-extract.perl

@@ -53,7 +53,7 @@ BEGIN
 
 if (-e $outPath)
 {
-  my $cmd = "diff --exclude=.DS_Store $outPath/ $truthPath/ | wc -l";
+  my $cmd = "diff --exclude=.DS_Store --exclude=._* $outPath/ $truthPath/ | wc -l";
   my $numDiff = `$cmd`;
 
   if ($numDiff == 0)

scripts/Jamfile

@@ -1,7 +1,6 @@
 #See ../Jamroot for options.  
 import option path ;
 
-build-project ems/biconcor ;
 build-project training ;
 
 with-giza = [ option.get "with-giza" ] ;
@@ -27,7 +26,7 @@ if $(with-giza) {
   check-for-bin mkcls ;
 } else {
   if $(CLEANING) = no {
-    echo "If you want scripts/training/train-model.perl, pass --with-giza=/path/to/giza-pp" ;
+    #echo "If you want scripts/training/train-model.perl, pass --with-giza=/path/to/giza-pp" ;
   }
   constant WITH-GIZA : "no" ;
 }