better handling of cache-based models with inconsistent parameters

moses-smt · Dec 15, 2014 · 4e77665 · 4e77665
2 parents e4eb201 + dfd6cd2
commit 4e77665
Show file tree

Hide file tree

Showing 37 changed files with 516 additions and 421 deletions.
diff --git a/contrib/other-builds/consolidate/.cproject b/contrib/other-builds/consolidate/.cproject
@@ -5,12 +5,12 @@
 			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.2091728208" moduleId="org.eclipse.cdt.core.settings" name="Debug">
 				<externalSettings/>
 				<extensions>
-					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
@@ -48,6 +48,7 @@
 									<listOptionValue builtIn="false" value="boost_filesystem"/>
 									<listOptionValue builtIn="false" value="pthread"/>
 									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="bz2"/>
 									<listOptionValue builtIn="false" value="dl"/>
 									<listOptionValue builtIn="false" value="rt"/>
 								</option>
@@ -86,12 +87,12 @@
 			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.185559773" moduleId="org.eclipse.cdt.core.settings" name="Release">
 				<externalSettings/>
 				<extensions>
-					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
@@ -156,4 +157,5 @@
 		</configuration>
 	</storageModule>
 	<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
+	<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
 </cproject>
diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project
@@ -1855,6 +1855,16 @@
 			<type>1</type>
 			<locationURI>PARENT-3-PROJECT_LOC/moses/Syntax/KBestExtractor.h</locationURI>
 		</link>
+		<link>
+			<name>Syntax/Manager.cpp</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/moses/Syntax/Manager.cpp</locationURI>
+		</link>
+		<link>
+			<name>Syntax/Manager.h</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/moses/Syntax/Manager.h</locationURI>
+		</link>
 		<link>
 			<name>Syntax/NonTerminalMap.h</name>
 			<type>1</type>

diff --git a/moses/BitmapContainer.cpp b/moses/BitmapContainer.cpp
@@ -56,19 +56,15 @@ class HypothesisScoreOrdererWithDistortion
     m_transOptRange(transOptRange) {
     m_totalWeightDistortion = 0;
     const StaticData &staticData = StaticData::Instance();
-    const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
-    std::vector<FeatureFunction*>::const_iterator iter;
+
+    const std::vector<const DistortionScoreProducer*> &ffs = DistortionScoreProducer::GetDistortionFeatureFunctions();
+    std::vector<const DistortionScoreProducer*>::const_iterator iter;
     for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
-      const FeatureFunction *ff = *iter;
+      const DistortionScoreProducer *ff = *iter;
 
-      const DistortionScoreProducer *model = dynamic_cast<const DistortionScoreProducer*>(ff);
-      if (model) {
-        float weight =staticData.GetAllWeights().GetScoreForProducer(model);
-        m_totalWeightDistortion += weight;
-      }
+      float weight =staticData.GetAllWeights().GetScoreForProducer(ff);
+      m_totalWeightDistortion += weight;
     }
-
-
   }
 
   const WordsRange* m_transOptRange;
@@ -223,7 +219,7 @@ Hypothesis *BackwardsEdge::CreateHypothesis(const Hypothesis &hypothesis, const
 bool
 BackwardsEdge::SeenPosition(const size_t x, const size_t y)
 {
-  std::set< int >::iterator iter = m_seenPosition.find((x<<16) + y);
+  boost::unordered_set< int >::iterator iter = m_seenPosition.find((x<<16) + y);
   return (iter != m_seenPosition.end());
 }
 

diff --git a/moses/BitmapContainer.h b/moses/BitmapContainer.h
@@ -33,6 +33,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "TypeDef.h"
 #include "WordsBitmap.h"
 
+#include <boost/unordered_set.hpp>
+
 namespace Moses
 {
 
@@ -165,7 +167,7 @@ class BackwardsEdge
   const SquareMatrix &m_futurescore;
 
   std::vector< const Hypothesis* > m_hypotheses;
-  std::set< int > m_seenPosition;
+  boost::unordered_set< int > m_seenPosition;
 
   // We don't want to instantiate "empty" objects.
   BackwardsEdge();

diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp
@@ -300,6 +300,16 @@ void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream)
   WriteSearchGraph(writer);
 }
 
+void ChartManager::OutputBest(OutputCollector *collector) const
+{
+  const ChartHypothesis *bestHypo = GetBestHypothesis();
+  if (collector && bestHypo) {
+	  const size_t translationId = m_source.GetTranslationId();
+	  const ChartHypothesis *bestHypo = GetBestHypothesis();
+	  OutputBestHypo(collector, bestHypo, translationId);
+  }
+}
+
 void ChartManager::OutputNBest(OutputCollector *collector) const
 {
 	const StaticData &staticData = StaticData::Instance();
@@ -807,4 +817,61 @@ void ChartManager::OutputSearchGraphHypergraph() const
   }
 }
 
+void ChartManager::OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const
+{
+  if (!collector)
+    return;
+  std::ostringstream out;
+  FixPrecision(out);
+  if (hypo != NULL) {
+    VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
+    VERBOSE(3,"Best path: ");
+    Backtrack(hypo);
+    VERBOSE(3,"0" << std::endl);
+
+    if (StaticData::Instance().GetOutputHypoScore()) {
+      out << hypo->GetTotalScore() << " ";
+    }
+
+    if (StaticData::Instance().IsPathRecoveryEnabled()) {
+      out << "||| ";
+    }
+    Phrase outPhrase(ARRAY_SIZE_INCR);
+    hypo->GetOutputPhrase(outPhrase);
+
+    // delete 1st & last
+    UTIL_THROW_IF2(outPhrase.GetSize() < 2,
+  		  "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+
+    outPhrase.RemoveWord(0);
+    outPhrase.RemoveWord(outPhrase.GetSize() - 1);
+
+    const std::vector<FactorType> outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
+    string output = outPhrase.GetStringRep(outputFactorOrder);
+    out << output << endl;
+  } else {
+    VERBOSE(1, "NO BEST TRANSLATION" << endl);
+
+    if (StaticData::Instance().GetOutputHypoScore()) {
+      out << "0 ";
+    }
+
+    out << endl;
+  }
+  collector->Write(translationId, out.str());
+}
+
+void ChartManager::Backtrack(const ChartHypothesis *hypo) const
+{
+  const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+  vector<const ChartHypothesis*>::const_iterator iter;
+  for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
+    const ChartHypothesis *prevHypo = *iter;
+
+    VERBOSE(3,prevHypo->GetId() << " <= ");
+    Backtrack(prevHypo);
+  }
+}
+
 } // namespace Moses
diff --git a/moses/ChartManager.h b/moses/ChartManager.h
@@ -99,6 +99,8 @@ class ChartManager : public BaseManager
 		  const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList,
 		  const Sentence &sentence,
 		  long translationId) const;
+  void OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const;
+  void Backtrack(const ChartHypothesis *hypo) const;
 
 public:
   ChartManager(InputType const& source);
@@ -143,6 +145,7 @@ class ChartManager : public BaseManager
   const ChartParser &GetParser() const { return m_parser; }
 
   // outputs
+  void OutputBest(OutputCollector *collector) const;
   void OutputNBest(OutputCollector *collector) const;
   void OutputLatticeSamples(OutputCollector *collector) const
   {}

diff --git a/moses/FF/BleuScoreFeature.cpp b/moses/FF/BleuScoreFeature.cpp
@@ -118,7 +118,7 @@ void BleuScoreFeature::SetParameter(const std::string& key, const std::string& v
       }
       string line;
       while (getline(in,line)) {
-        /*  if (GetSearchAlgorithm() == ChartDecoding) {
+        /*  if (GetSearchAlgorithm() == CYKPlus) {
         stringstream tmp;
         tmp << "<s> " << line << " </s>";
         line = tmp.str();

diff --git a/moses/FF/ConstrainedDecoding.cpp b/moses/FF/ConstrainedDecoding.cpp
@@ -43,7 +43,7 @@ ConstrainedDecoding::ConstrainedDecoding(const std::string &line)
 void ConstrainedDecoding::Load()
 {
   const StaticData &staticData = StaticData::Instance();
-  bool addBeginEndWord = (staticData.GetSearchAlgorithm() == ChartDecoding) || (staticData.GetSearchAlgorithm() == ChartIncremental);
+  bool addBeginEndWord = (staticData.GetSearchAlgorithm() == CYKPlus) || (staticData.GetSearchAlgorithm() == ChartIncremental);
 
   for(size_t i = 0; i < m_paths.size(); ++i) {
     InputFileStream constraintFile(m_paths[i]);

diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp
@@ -22,9 +22,12 @@ struct DistortionState_traditional : public FFState {
   }
 };
 
+std::vector<const DistortionScoreProducer*> DistortionScoreProducer::s_staticColl;
+
 DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
   : StatefulFeatureFunction(1, line)
 {
+  s_staticColl.push_back(this);
   ReadParameters();
 }
 

diff --git a/moses/FF/DistortionScoreProducer.h b/moses/FF/DistortionScoreProducer.h
@@ -16,7 +16,14 @@ class WordsRange;
  */
 class DistortionScoreProducer : public StatefulFeatureFunction
 {
+protected:
+  static std::vector<const DistortionScoreProducer*> s_staticColl;
+
 public:
+  static const std::vector<const DistortionScoreProducer*>& GetDistortionFeatureFunctions() {
+    return s_staticColl;
+  }
+
   DistortionScoreProducer(const std::string &line);
 
   bool IsUseable(const FactorMask &mask) const {

diff --git a/moses/FF/DynamicCacheBasedLanguageModel.h b/moses/FF/DynamicCacheBasedLanguageModel.h
@@ -78,8 +78,8 @@ class DynamicCacheBasedLanguageModel : public StatelessFeatureFunction
   void Print() const;
 
 protected:
-  static DynamicCacheBasedLanguageModel *s_instance;
-  static std::map< const std::string, DynamicCacheBasedLanguageModel * > s_instance_map;
+  static DynamicCacheBasedLanguageModel* s_instance;
+  static std::map< const std::string, DynamicCacheBasedLanguageModel* > s_instance_map;
 
 public:
   DynamicCacheBasedLanguageModel(const std::string &line);
@@ -92,15 +92,18 @@ class DynamicCacheBasedLanguageModel : public StatelessFeatureFunction
     m_name = name;
   }
 
-  static const DynamicCacheBasedLanguageModel& Instance(const std::string name) {
-    UTIL_THROW_IF2(s_instance_map.find(name) == s_instance_map.end(), "The DynamicCacheBasedLanguageModel feature named " + name + " does not exist!");
-    return *(s_instance_map[name]);
+  static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
+    if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
+    return s_instance_map[name];
   }
 
-  static DynamicCacheBasedLanguageModel& InstanceNonConst(const std::string name) {
-    UTIL_THROW_IF2(s_instance_map.find(name) == s_instance_map.end(), "The DynamicCacheBasedLanguageModel feature named " + name + " does not exist!");
-    return *(s_instance_map[name]);
+  static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
+    if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
+    return s_instance_map[name];
   }
+
+
+
   static const DynamicCacheBasedLanguageModel& Instance() {
     return *s_instance;
   }

diff --git a/moses/FF/FeatureFunction.cpp b/moses/FF/FeatureFunction.cpp
@@ -7,6 +7,7 @@
 #include "moses/Manager.h"
 #include "moses/TranslationOption.h"
 #include "moses/Util.h"
+#include "moses/FF/DistortionScoreProducer.h"
 
 using namespace std;
 

diff --git a/moses/FF/FeatureFunction.h b/moses/FF/FeatureFunction.h
@@ -22,6 +22,7 @@ class WordsRange;
 class FactorMask;
 class InputPath;
 class StackVec;
+class DistortionScoreProducer;
 
 /** base class for all feature functions.
  */
@@ -46,6 +47,7 @@ class FeatureFunction
   static const std::vector<FeatureFunction*>& GetFeatureFunctions() {
     return s_staticColl;
   }
+
   static FeatureFunction &FindFeatureFunction(const std::string& name);
   static void Destroy();