This repository has been archived by the owner on Dec 13, 2021. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- GrammarPacker now uses the appropriate {Moses,Hiero}FormatReader objects in explore() and binarize() passes, instead of doing its own parsing
- MosesFormatReader chains to HieroFormatReader after munging input, removes some redundancy
- Updated test case- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,6 +37,9 @@ | |
| import java.util.logging.Logger; | ||
|
|
||
| import joshua.corpus.Vocabulary; | ||
| import joshua.decoder.ff.tm.Rule; | ||
| import joshua.decoder.ff.tm.format.HieroFormatReader; | ||
| import joshua.decoder.ff.tm.format.MosesFormatReader; | ||
| import joshua.util.FormatUtils; | ||
| import joshua.util.encoding.EncoderConfiguration; | ||
| import joshua.util.encoding.FeatureTypeAnalyzer; | ||
|
|
@@ -154,13 +157,12 @@ private void readConfig(String config_filename) throws IOException { | |
| */ | ||
| public void pack() throws IOException { | ||
| logger.info("Beginning exploration pass."); | ||
|
|
||
| // Explore pass. Learn vocabulary and feature value histograms. | ||
| logger.info("Exploring: " + grammar); | ||
|
|
||
| HieroFormatReader grammarReader = getGrammarReader(); | ||
| explore(grammarReader); | ||
|
|
||
| logger.info("Exploration pass complete. Freezing vocabulary and finalizing encoders."); | ||
| if (dump != null) { | ||
|
|
@@ -194,78 +196,56 @@ public void pack() throws IOException { | |
|
|
||
| logger.info("Beginning packing pass."); | ||
| // Actual binarization pass. Slice and pack source, target and data. | ||
| grammarReader = getGrammarReader(); | ||
| LineReader alignment_reader = null; | ||
| if (packAlignments && !grammarAlignments) | ||
| alignment_reader = new LineReader(alignments); | ||
| binarize(grammarReader, alignment_reader); | ||
| logger.info("Packing complete."); | ||
|
|
||
| logger.info("Packed grammar in: " + output); | ||
| logger.info("Done."); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a reader that turns whatever file format is found into Hiero grammar rules. | ||
| * | ||
| * @param grammarFile | ||
| * @return | ||
| * @throws IOException | ||
| */ | ||
| private HieroFormatReader getGrammarReader() throws IOException { | ||
| LineReader reader = new LineReader(grammar); | ||
| String line = reader.next(); | ||
| if (line.startsWith("[")) { | ||
| return new HieroFormatReader(grammar); | ||
| } else { | ||
| return new MosesFormatReader(grammar); | ||
| } | ||
| } | ||
|
|
||
| private void explore(HieroFormatReader reader) { | ||
| int counter = 0; | ||
| // We always assume a labeled grammar. Unlabeled features are assumed to be dense and to always | ||
| // appear in the same order. They are assigned numeric names in order of appearance. | ||
| this.types.setLabeled(true); | ||
|
|
||
| for (Rule rule: reader) { | ||
| counter++; | ||
|
|
||
| max_source_len = Math.max(max_source_len, rule.getFrench().length); | ||
|
|
||
| /* Add symbols to vocabulary. | ||
| * NOTE: In case of nonterminals, we add both stripped versions ("[X]") | ||
| * and "[X,1]" to the vocabulary. | ||
| * | ||
| * TODO: MJP May 2016: do we need to add [X,1]? If so, should be done in FormatReaders. | ||
This comment has been minimized.
Sorry, something went wrong. 
fhieber
Contributor
|
||
| */ | ||
|
|
||
| // Add feature names to vocabulary and pass the value through the | ||
| // appropriate encoder. | ||
| int feature_counter = 0; | ||
| String[] features = rule.getFeatureString().split("\\s+"); | ||
| for (int f = 0; f < features.length; ++f) { | ||
| if (features[f].contains("=")) { | ||
| String[] fe = features[f].split("="); | ||
|
|
@@ -278,6 +258,7 @@ private void explore(LineReader grammar) { | |
| } | ||
| } | ||
| } | ||
| System.err.println("COUNTER: " + counter); | ||
This comment has been minimized.
Sorry, something went wrong. |
||
| } | ||
|
|
||
| /** | ||
|
|
@@ -288,7 +269,7 @@ private String getFirstTwoSourceWords(final String[] source_words) { | |
| return source_words[0] + SOURCE_WORDS_SEPARATOR + ((source_words.length > 1) ? source_words[1] : ""); | ||
| } | ||
|
|
||
| private void binarize(HieroFormatReader grammarReader, LineReader alignment_reader) throws IOException { | ||
| int counter = 0; | ||
| int slice_counter = 0; | ||
| int num_slices = 0; | ||
|
|
@@ -306,36 +287,14 @@ private void binarize(LineReader grammar_reader, LineReader alignment_reader) th | |
| alignment_buffer = new AlignmentBuffer(); | ||
|
|
||
| TreeMap<Integer, Float> features = new TreeMap<Integer, Float>(); | ||
| for (Rule rule: grammarReader) { | ||
| counter++; | ||
| slice_counter++; | ||
|
|
||
| String lhs_word = Vocabulary.word(rule.getLHS()); | ||
| String[] source_words = rule.getFrenchWords().split("\\s+"); | ||
This comment has been minimized.
Sorry, something went wrong.
fhieber
Contributor
|
||
| String[] target_words = rule.getEnglishWords().split("\\s+"); | ||
| String[] feature_entries = rule.getFeatureString().split("\\s+"); | ||
|
|
||
| // Reached slice limit size, indicate that we're closing up. | ||
| if (!ready_to_flush | ||
|
|
@@ -373,7 +332,7 @@ private void binarize(LineReader grammar_reader, LineReader alignment_reader) th | |
| if (packAlignments) { | ||
| String alignment_line; | ||
| if (grammarAlignments) { | ||
| alignment_line = rule.getAlignmentString(); | ||
| } else { | ||
| if (!alignment_reader.hasNext()) { | ||
| logger.severe("No more alignments starting in line " + counter); | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| max-source-len = 3 |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
1 comment
on commit f5adcde
There was a problem hiding this comment.
I love this!
you can initialize the StringBuffer directly with the following line:
new StringBuffer("[X] ||| [X,1] " + fields[0] + " ||| [X,1] " + fields[1] + " |||")