Permalink
Browse files

Integration of 4-grams for LM queries.

  • Loading branch information...
1 parent da755d3 commit 7ae7c9c35ac426e6faf2d5c8de85c12da4d94b4b @julosaure committed Sep 28, 2012
Showing with 13 additions and 5 deletions.
  1. +1 −1 src/decloder/blm.clj
  2. +12 −4 src/decloder/translator.clj
View
@@ -12,7 +12,7 @@
;(def LM_BIN "/Users/julien/workspaces/clojure/berkeleylm-1.1.2/examples/big_test.binary")
-(def LM_BIN "/Users/julien/workspaces/xp/europarl/europarl-v7.fr-en.fr.tok.low.se.berk_lmbin")
+(def LM_BIN "/Users/julien/workspaces/xp/europarl/europarl-v7.fr-en.fr.tok.low.se.n4.wb.berk_lmbin")
;; UTILS
@@ -33,10 +33,18 @@
;(println "bi-gram to score: " bi-gram " -> " lm-score)
(+ lex-prob (* 0.5 (:score pred-hypo)) (* 0.1 lm-score))
)
- (let [tri-gram (str (:token (:pred pred-hypo)) " " bi-gram)
- lm-score (decloder.blm/score-ngrams (model :lm) tri-gram)]
- ;(println "tri-gram to score: " tri-gram " -> " lm-score)
- (+ lex-prob (* 0.5 (:score pred-hypo)) (* 0.1 lm-score))
+ (let [tri-gram (str (:token (:pred pred-hypo)) " " bi-gram)]
+ (if (nil? (:pred (:pred pred-hypo)))
+ (let [lm-score (decloder.blm/score-ngrams (model :lm) tri-gram)]
+ ;(println "tri-gram to score: " tri-gram " -> " lm-score)
+ (+ lex-prob (* 0.3 (:score pred-hypo)) (* 0.3 lm-score))
+ )
+ (let [quad-gram (str (:token (:pred (:pred pred-hypo))) " " tri-gram)
+ lm-score (decloder.blm/score-ngrams (model :lm) quad-gram)]
+ ;(println "4-gram to score: " quad-gram " -> " lm-score)
+ (+ lex-prob (* 0.3 (:score pred-hypo)) (* 0.3 lm-score))
+ )
+ )
)
)
)

0 comments on commit 7ae7c9c

Please sign in to comment.