/
tessdatamanager.h
299 lines (269 loc) · 11.1 KB
/
tessdatamanager.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
///////////////////////////////////////////////////////////////////////
// File: tessdatamanager.h
// Description: Functions to handle loading/combining tesseract data files.
// Author: Daria Antonova
// Created: Wed Jun 03 11:26:43 PST 2009
//
// (C) Copyright 2009, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////
#ifndef TESSERACT_CCUTIL_TESSDATAMANAGER_H_
#define TESSERACT_CCUTIL_TESSDATAMANAGER_H_
#include <stdio.h>
#include "host.h"
#include "strngs.h"
#include "tprintf.h"
static const char kTrainedDataSuffix[] = "traineddata";
// When adding new tessdata types and file suffixes, please make sure to
// update TessdataType enum, kTessdataFileSuffixes and kTessdataFileIsText.
static const char kLangConfigFileSuffix[] = "config";
static const char kUnicharsetFileSuffix[] = "unicharset";
static const char kAmbigsFileSuffix[] = "unicharambigs";
static const char kBuiltInTemplatesFileSuffix[] = "inttemp";
static const char kBuiltInCutoffsFileSuffix[] = "pffmtable";
static const char kNormProtoFileSuffix[] = "normproto";
static const char kPuncDawgFileSuffix[] = "punc-dawg";
static const char kSystemDawgFileSuffix[] = "word-dawg";
static const char kNumberDawgFileSuffix[] = "number-dawg";
static const char kFreqDawgFileSuffix[] = "freq-dawg";
static const char kFixedLengthDawgsFileSuffix[] = "fixed-length-dawgs";
static const char kCubeUnicharsetFileSuffix[] = "cube-unicharset";
static const char kCubeSystemDawgFileSuffix[] = "cube-word-dawg";
static const char kShapeTableFileSuffix[] = "shapetable";
static const char kBigramDawgFileSuffix[] = "bigram-dawg";
static const char kUnambigDawgFileSuffix[] = "unambig-dawg";
static const char kParamsModelFileSuffix[] = "params-model";
namespace tesseract {
enum TessdataType {
TESSDATA_LANG_CONFIG, // 0
TESSDATA_UNICHARSET, // 1
TESSDATA_AMBIGS, // 2
TESSDATA_INTTEMP, // 3
TESSDATA_PFFMTABLE, // 4
TESSDATA_NORMPROTO, // 5
TESSDATA_PUNC_DAWG, // 6
TESSDATA_SYSTEM_DAWG, // 7
TESSDATA_NUMBER_DAWG, // 8
TESSDATA_FREQ_DAWG, // 9
TESSDATA_FIXED_LENGTH_DAWGS, // 10 // deprecated
TESSDATA_CUBE_UNICHARSET, // 11
TESSDATA_CUBE_SYSTEM_DAWG, // 12
TESSDATA_SHAPE_TABLE, // 13
TESSDATA_BIGRAM_DAWG, // 14
TESSDATA_UNAMBIG_DAWG, // 15
TESSDATA_PARAMS_MODEL, // 16
TESSDATA_NUM_ENTRIES
};
/**
* kTessdataFileSuffixes[i] indicates the file suffix for
* tessdata of type i (from TessdataType enum).
*/
static const char * const kTessdataFileSuffixes[] = {
kLangConfigFileSuffix, // 0
kUnicharsetFileSuffix, // 1
kAmbigsFileSuffix, // 2
kBuiltInTemplatesFileSuffix, // 3
kBuiltInCutoffsFileSuffix, // 4
kNormProtoFileSuffix, // 5
kPuncDawgFileSuffix, // 6
kSystemDawgFileSuffix, // 7
kNumberDawgFileSuffix, // 8
kFreqDawgFileSuffix, // 9
kFixedLengthDawgsFileSuffix, // 10 // deprecated
kCubeUnicharsetFileSuffix, // 11
kCubeSystemDawgFileSuffix, // 12
kShapeTableFileSuffix, // 13
kBigramDawgFileSuffix, // 14
kUnambigDawgFileSuffix, // 15
kParamsModelFileSuffix, // 16
};
/**
* If kTessdataFileIsText[i] is true - the tessdata component
* of type i (from TessdataType enum) is text, and is binary otherwise.
*/
static const bool kTessdataFileIsText[] = {
true, // 0
true, // 1
true, // 2
false, // 3
true, // 4
true, // 5
false, // 6
false, // 7
false, // 8
false, // 9
false, // 10 // deprecated
true, // 11
false, // 12
false, // 13
false, // 14
false, // 15
true, // 16
};
/**
* TessdataType could be updated to contain more entries, however
* we do not expect that number to be astronomically high.
* In order to automatically detect endianness TessdataManager will
* flip the bits if actual_tessdata_num_entries_ is larger than
* kMaxNumTessdataEntries.
*/
static const int kMaxNumTessdataEntries = 1000;
class TessdataManager {
public:
TessdataManager() {
data_file_ = NULL;
actual_tessdata_num_entries_ = 0;
for (int i = 0; i < TESSDATA_NUM_ENTRIES; ++i) {
offset_table_[i] = -1;
}
}
~TessdataManager() {}
int DebugLevel() { return debug_level_; }
/**
* Opens the given data file and reads the offset table.
* @return true on success.
*/
bool Init(const char *data_file_name, int debug_level);
// Return the name of the underlying data file.
const STRING &GetDataFileName() const { return data_file_name_; }
/** Returns data file pointer. */
inline FILE *GetDataFilePtr() const { return data_file_; }
/**
* Returns false if there is no data of the given type.
* Otherwise does a seek on the data_file_ to position the pointer
* at the start of the data of the given type.
*/
inline bool SeekToStart(TessdataType tessdata_type) {
if (debug_level_) {
tprintf("TessdataManager: seek to offset %lld - start of tessdata"
"type %d (%s))\n", offset_table_[tessdata_type],
tessdata_type, kTessdataFileSuffixes[tessdata_type]);
}
if (offset_table_[tessdata_type] < 0) {
return false;
} else {
ASSERT_HOST(fseek(data_file_,
static_cast<size_t>(offset_table_[tessdata_type]),
SEEK_SET) == 0);
return true;
}
}
/** Returns the end offset for the given tesseract data file type. */
inline inT64 GetEndOffset(TessdataType tessdata_type) const {
int index = tessdata_type + 1;
while (index < actual_tessdata_num_entries_ && offset_table_[index] == -1) {
++index; // skip tessdata types not present in the combined file
}
if (debug_level_) {
tprintf("TessdataManager: end offset for type %d is %lld\n",
tessdata_type,
(index == actual_tessdata_num_entries_) ? -1
: offset_table_[index]);
}
return (index == actual_tessdata_num_entries_) ? -1 : offset_table_[index] - 1;
}
/** Closes data_file_ (if it was opened by Init()). */
inline void End() {
if (data_file_ != NULL) {
fclose(data_file_);
data_file_ = NULL;
}
}
bool swap() const {
return swap_;
}
/** Writes the number of entries and the given offset table to output_file.
* Returns false on error.
*/
static bool WriteMetadata(inT64 *offset_table,
const char *language_data_path_prefix,
FILE *output_file);
/**
* Reads all the standard tesseract config and data files for a language
* at the given path and bundles them up into one binary data file.
* Returns true if the combined traineddata file was successfully written.
*/
static bool CombineDataFiles(const char *language_data_path_prefix,
const char *output_filename);
/**
* Gets the individual components from the data_file_ with which the class was
* initialized. Overwrites the components specified by component_filenames.
* Writes the updated traineddata file to new_traineddata_filename.
*/
bool OverwriteComponents(const char *new_traineddata_filename,
char **component_filenames,
int num_new_components);
/**
* Extracts tessdata component implied by the name of the input file from
* the combined traineddata loaded into TessdataManager.
* Writes the extracted component to the file indicated by the file name.
* E.g. if the filename given is somepath/somelang.unicharset, unicharset
* will be extracted from the data loaded into the TessdataManager and will
* be written to somepath/somelang.unicharset.
* @return true if the component was successfully extracted, false if the
* component was not present in the traineddata loaded into TessdataManager.
*/
bool ExtractToFile(const char *filename);
/**
* Copies data from the given input file to the output_file provided.
* If num_bytes_to_copy is >= 0, only num_bytes_to_copy is copied from
* the input file, otherwise all the data in the input file is copied.
*/
static void CopyFile(FILE *input_file, FILE *output_file,
bool newline_end, inT64 num_bytes_to_copy);
/**
* Fills type with TessdataType of the tessdata component represented by the
* given file name. E.g. tessdata/eng.unicharset -> TESSDATA_UNICHARSET.
* Sets *text_file to true if the component is in text format (e.g.
* unicharset, unichar ambigs, config, etc).
* @return true if the tessdata component type could be determined
* from the given file name.
*/
static bool TessdataTypeFromFileSuffix(const char *suffix,
TessdataType *type,
bool *text_file);
/**
* Tries to determine tessdata component file suffix from filename,
* returns true on success.
*/
static bool TessdataTypeFromFileName(const char *filename,
TessdataType *type,
bool *text_file);
private:
/**
* Opens the file whose name is a concatenation of language_data_path_prefix
* and file_suffix. Returns a file pointer to the opened file.
*/
static FILE *GetFilePtr(const char *language_data_path_prefix,
const char *file_suffix, bool text_file);
/**
* Each offset_table_[i] contains a file offset in the combined data file
* where the data of TessdataFileType i is stored.
*/
inT64 offset_table_[TESSDATA_NUM_ENTRIES];
/**
* Actual number of entries in the tessdata table. This value can only be
* same or smaller than TESSDATA_NUM_ENTRIES, but can never be larger,
* since then it would be impossible to interpret the type of tessdata at
* indices same and higher than TESSDATA_NUM_ENTRIES.
* This parameter is used to allow for backward compatibility
* when new tessdata types are introduced.
*/
inT32 actual_tessdata_num_entries_;
STRING data_file_name_; // name of the data file.
FILE *data_file_; ///< pointer to the data file.
int debug_level_;
// True if the bytes need swapping.
bool swap_;
};
} // namespace tesseract
#endif // TESSERACT_CCUTIL_TESSDATAMANAGER_H_