Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 100 lines (83 sloc) 2.802 kB
677f0f4 copyright notice & svn properties
hieuhoang1972 authored
1 // $Id$
2 /***********************************************************************
3 Moses - factored phrase-based, hierarchical and syntactic language decoder
4 Copyright (C) 2009 Hieu Hoang
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
5
677f0f4 copyright notice & svn properties
hieuhoang1972 authored
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
10
677f0f4 copyright notice & svn properties
hieuhoang1972 authored
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
15
677f0f4 copyright notice & svn properties
hieuhoang1972 authored
16 You should have received a copy of the GNU Lesser General Public
17 License along with this library; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 ***********************************************************************/
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
20 #include <string>
21 #include <fstream>
22 #include "OnDiskWrapper.h"
23 #include "Vocab.h"
24
25 using namespace std;
26
27 namespace OnDiskPt
28 {
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
29
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
30 bool Vocab::Load(OnDiskWrapper &onDiskWrapper)
31 {
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
32 fstream &file = onDiskWrapper.GetFileVocab();
33
34 string line;
35 while(getline(file, line)) {
36 vector<string> tokens;
37 Moses::Tokenize(tokens, line);
bf78f7a @kpu Replace assert with CHECK until people learn how to use assert properly
kpu authored
38 CHECK(tokens.size() == 2);
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
39 const string &key = tokens[0];
40 m_vocabColl[key] = Moses::Scan<UINT64>(tokens[1]);
41 }
42
43 // create lookup
44 // assume contiguous vocab id
45 m_lookup.resize(m_vocabColl.size() + 1);
1ef0d6b bug. Unknown words given id starting from 1
Hieu Hoang authored
46 m_nextId = m_lookup.size();
47
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
48 CollType::const_iterator iter;
49 for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) {
50 UINT32 vocabId = iter->second;
51 const std::string &word = iter->first;
52
53 m_lookup[vocabId] = word;
54 }
55
56 return true;
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
57 }
58
59 void Vocab::Save(OnDiskWrapper &onDiskWrapper)
60 {
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
61 fstream &file = onDiskWrapper.GetFileVocab();
62 CollType::const_iterator iterVocab;
63 for (iterVocab = m_vocabColl.begin(); iterVocab != m_vocabColl.end(); ++iterVocab) {
64 const string &word = iterVocab->first;
65 UINT32 vocabId = iterVocab->second;
66
67 file << word << " " << vocabId << endl;
68 }
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
69 }
70
e71f76a delete unused function
Hieu Hoang authored
71 UINT64 Vocab::AddVocabId(const std::string &str)
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
72 {
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
73 // find string id
e71f76a delete unused function
Hieu Hoang authored
74 CollType::const_iterator iter = m_vocabColl.find(str);
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
75 if (iter == m_vocabColl.end()) {
76 // add new vocab entry
e71f76a delete unused function
Hieu Hoang authored
77 m_vocabColl[str] = m_nextId;
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
78 return m_nextId++;
79 } else {
80 // return existing entry
81 return iter->second;
82 }
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
83 }
84
e71f76a delete unused function
Hieu Hoang authored
85 UINT64 Vocab::GetVocabId(const std::string &str, bool &found) const
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
86 {
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
87 // find string id
e71f76a delete unused function
Hieu Hoang authored
88 CollType::const_iterator iter = m_vocabColl.find(str);
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
89 if (iter == m_vocabColl.end()) {
90 found = false;
91 return 0; //return whatever
92 } else {
93 // return existing entry
94 found = true;
95 return iter->second;
96 }
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
97 }
a3d9758 run beautify.perl. Consistent formatting for .h & .cpp files
hieuhoang1972 authored
98
c117ef7 Copy in changes from the chart_merge branch (doing it manually becaus…
hieuhoang1972 authored
99 }
Something went wrong with that request. Please try again.