Permalink
Browse files

fix issue #101

If using zlib, either compressed or uncompressed model files can be loaded.
They may even be mixed and matched (some compressed, some not) if that happens
to be the case.

For saving, compression is chosen if zlib exists.
  • Loading branch information...
smassung committed Jun 19, 2015
1 parent 97b06c3 commit a7e21013956236f8d127a45003aaa246245c3bfb
@@ -268,6 +268,12 @@ class sr_parser
best_transitions(const feature_vector& features, const state& state,
size_t num, bool check_legality = false) const;
/**
* Loads the parser model file from the given stream.
* @param model The input stream to read from
*/
void load(std::istream& model);
/**
* Storage for the ids for each transition
*/
@@ -75,6 +75,13 @@ class transition_map
};
private:
/**
* Loads the transitions from the given file.
* @param store The transitions model input stream
*/
void load(std::istream& store);
/**
* The map from transition to id.
*/
@@ -318,6 +318,12 @@ class sequence_analyzer
*/
void load_feature_id_mapping(const std::string& prefix);
/**
* Loads the feature_id mapping from disk using an input stream.
* @param input The input stream of the feature_id mapping
*/
void load_feature_id_mapping(std::istream& input);
/**
* Loads the label_id mapping from disk.
* @param prefix The folder to load the mapping from
@@ -16,6 +16,7 @@
#include "parser/trees/internal_node.h"
#include "parser/trees/leaf_node.h"
#include "parser/trees/visitors/debinarizer.h"
#include "util/filesystem.h"
#include "util/progress.h"
#include "util/range.h"
#include "util/time.h"
@@ -451,11 +452,19 @@ void sr_parser::save(const std::string& prefix) const
void sr_parser::load(const std::string& prefix)
{
#ifdef META_HAS_ZLIB
io::gzifstream model{prefix + "/parser.model.gz"};
#else
std::ifstream model{prefix + "/parser.model", std::ios::binary};
if (filesystem::file_exists(prefix + "/parser.model.gz"))
{
io::gzifstream model{prefix + "/parser.model.gz"};
load(model);
return;
}
#endif
std::ifstream model{prefix + "/parser.model", std::ios::binary};
load(model);
}
void sr_parser::load(std::istream& model)
{
if (!model)
throw exception{"model file not found"};
@@ -8,6 +8,7 @@
#include "io/binary.h"
#include "parser/transition_map.h"
#include "util/filesystem.h"
#ifdef META_HAS_ZLIB
#include "io/gzstream.h"
@@ -21,11 +22,19 @@ namespace parser
transition_map::transition_map(const std::string& prefix)
{
#ifdef META_HAS_ZLIB
io::gzifstream store{prefix + "/parser.trans.gz"};
#else
std::ifstream store{prefix + "/parser.trans", std::ios::binary};
if (filesystem::file_exists(prefix + "/parser.trans.gz"))
{
io::gzifstream store{prefix + "/parser.trans.gz"};
load(store);
return;
}
#endif
std::ifstream store{prefix + "/parser.trans", std::ios::binary};
load(store);
}
void transition_map::load(std::istream& store)
{
if (!store)
throw exception{"missing transitions model file"};
@@ -7,6 +7,7 @@
#include "sequence/perceptron.h"
#include "utf/utf.h"
#include "util/filesystem.h"
#include "util/progress.h"
#include "util/time.h"
@@ -51,10 +52,14 @@ perceptron::perceptron(const std::string& prefix) : perceptron()
analyzer_.load(prefix);
#if META_HAS_ZLIB
io::gzifstream file{prefix + "/tagger.model.gz"};
#else
std::ifstream file{prefix + "/tagger.model"};
if (filesystem::file_exists(prefix + "/tagger.model.gz"))
{
io::gzifstream file{prefix + "/tagger.model.gz"};
model_.load(file);
return;
}
#endif
std::ifstream file{prefix + "/tagger.model"};
model_.load(file);
}
@@ -37,11 +37,19 @@ void sequence_analyzer::load(const std::string& prefix)
void sequence_analyzer::load_feature_id_mapping(const std::string& prefix)
{
#if META_HAS_ZLIB
io::gzifstream input{prefix + "/feature.mapping.gz"};
#else
std::ifstream input{prefix + "/feature.mapping", std::ios::binary};
if (filesystem::file_exists(prefix + "/feature.mapping.gz"))
{
io::gzifstream input{prefix + "/feature.mapping.gz"};
load_feature_id_mapping(input);
return;
}
#endif
std::ifstream input{prefix + "/feature.mapping", std::ios::binary};
load_feature_id_mapping(input);
}
void sequence_analyzer::load_feature_id_mapping(std::istream& input)
{
if (!input)
throw exception{"missing feature id mapping"};

0 comments on commit a7e2101

Please sign in to comment.