Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

More updates to LSTM related unittests #2193

Merged
merged 2 commits into from
Jan 24, 2019
Merged

More updates to LSTM related unittests #2193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3690606
Add LF to INFO msgs in lstm_test
Shreeshrii Jan 24, 2019
dbb12d6
more updates to lstm related unittests
Shreeshrii Jan 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions unittest/lstm_recode_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ TEST_F(LSTMTrainerTest, RecodeTestKorBase) {
double kor_full_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(kor_full_err, 88);
// EXPECT_GT(kor_full_err, 85);
LOG(INFO) << "********** Expected < 88 ************\n" ;
}

TEST_F(LSTMTrainerTest, RecodeTestKor) {
Expand All @@ -31,6 +32,7 @@ TEST_F(LSTMTrainerTest, RecodeTestKor) {
"kor.Arial_Unicode_MS.exp0.lstmf", true, true, 5e-4, false);
double kor_recode_err = TrainIterations(kTrainerIterations);
EXPECT_LT(kor_recode_err, 60);
LOG(INFO) << "********** Expected < 60 ************\n" ;
}

// Tests that the given string encodes and decodes back to the same
Expand Down
2 changes: 1 addition & 1 deletion unittest/lstm_squashed_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ TEST_F(LSTMTrainerTest, TestSquashed) {
"SQU-2-layer-lstm", /*recode*/ true, /*adam*/ true);
double lstm_2d_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(lstm_2d_err, 80);
LOG(INFO) << "********** < 80 ************" ;
LOG(INFO) << "********** < 80 ************\n" ;
TestIntMode(kTrainerIterations);
}

Expand Down
43 changes: 23 additions & 20 deletions unittest/lstm_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
//
// Use --xsize 800 for text2image to be similar to original training data.
//
// src/training/tesstrain.sh --fonts_dir /usr/share/fonts --lang eng --linedata_only --noextract_font_properties --langdata_dir ../langdata_lstm --tessdata_dir ../tessdata --output_dir ~/tesseract/test/testdata --fontlist "Arial" --maxpages 10
// src/training/tesstrain.sh --fonts_dir /usr/share/fonts --lang eng \
// --linedata_only --noextract_font_properties --langdata_dir ../langdata_lstm \
// --tessdata_dir ../tessdata --output_dir ~/tesseract/test/testdata \
// --fontlist "Arial" --maxpages 10
//

#include "lstm_test.h"
Expand All @@ -30,15 +33,15 @@ TEST_F(LSTMTrainerTest, BasicTest) {
"Ct1,1,64O1c1]",
"no-lstm", "eng/eng.unicharset", "eng.Arial.exp0.lstmf", false, false,
2e-4, false);
double non_lstm_err = TrainIterations(kTrainerIterations * 3);
double non_lstm_err = TrainIterations(kTrainerIterations * 4);
EXPECT_LT(non_lstm_err, 98);
LOG(INFO) << "********** Expected < 98 ************" ;
LOG(INFO) << "********** Expected < 98 ************\n" ;

// A basic single-layer, single direction LSTM.
SetupTrainerEng("[1,1,0,32 Lfx100 O1c1]", "1D-lstm", false, false);
double lstm_uni_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(lstm_uni_err, 86);
LOG(INFO) << "********** Expected < 86 ************" ;
LOG(INFO) << "********** Expected < 86 ************\n" ;
// Beats the convolver. (Although it does have a lot more weights, it still
// iterates faster.)
EXPECT_LT(lstm_uni_err, non_lstm_err);
Expand All @@ -51,16 +54,16 @@ TEST_F(LSTMTrainerTest, ColorTest) {
"2D-color-lstm", true, true);
double lstm_uni_err = TrainIterations(kTrainerIterations);
EXPECT_LT(lstm_uni_err, 85);
EXPECT_GT(lstm_uni_err, 66);
LOG(INFO) << "********** Expected > 66 ** < 85 ************" ;
// EXPECT_GT(lstm_uni_err, 66);
LOG(INFO) << "********** Expected < 85 ************\n" ;
}

TEST_F(LSTMTrainerTest, BidiTest) {
// A basic single-layer, bi-di 1d LSTM.
SetupTrainerEng("[1,1,0,32 Lbx100 O1c1]", "bidi-lstm", false, false);
double lstm_bi_err = TrainIterations(kTrainerIterations);
EXPECT_LT(lstm_bi_err, 75);
LOG(INFO) << "********** Expected < 75 ************" ;
LOG(INFO) << "********** Expected < 75 ************\n" ;
// Int mode training is dead, so convert the trained network to int and check
// that its error rate is close to the float version.
TestIntMode(kTrainerIterations);
Expand All @@ -72,10 +75,10 @@ TEST_F(LSTMTrainerTest, Test2D) {
// A 2-layer LSTM with a 2-D feature-extracting LSTM on the bottom.
SetupTrainerEng("[1,32,0,1 S4,2 L2xy16 Ct1,1,16 S8,1 Lbx100 O1c1]",
"2-D-2-layer-lstm", false, false);
double lstm_2d_err = TrainIterations(kTrainerIterations);
double lstm_2d_err = TrainIterations(kTrainerIterations * 3 / 2 );
EXPECT_LT(lstm_2d_err, 98);
EXPECT_GT(lstm_2d_err, 90);
LOG(INFO) << "********** Expected > 90 ** < 98 ************" ;
// EXPECT_GT(lstm_2d_err, 90);
LOG(INFO) << "********** Expected < 98 ************\n" ;
// Int mode training is dead, so convert the trained network to int and check
// that its error rate is close to the float version.
TestIntMode(kTrainerIterations);
Expand All @@ -89,7 +92,7 @@ TEST_F(LSTMTrainerTest, TestAdam) {
"2-D-2-layer-lstm", false, true);
double lstm_2d_err = TrainIterations(kTrainerIterations);
EXPECT_LT(lstm_2d_err, 70);
LOG(INFO) << "********** Expected < 70 ************" ;
LOG(INFO) << "********** Expected < 70 ************\n" ;
TestIntMode(kTrainerIterations);
}

Expand All @@ -100,23 +103,23 @@ TEST_F(LSTMTrainerTest, SpeedTest) {
"O1c1]",
"2-D-2-layer-lstm", false, true);
TrainIterations(kTrainerIterations);
LOG(INFO) << "********** *** ************" ;
LOG(INFO) << "********** *** ************\n" ;
}

// Tests that two identical networks trained the same get the same results.
// Also tests that the same happens with a serialize/deserialize in the middle.
TEST_F(LSTMTrainerTest, DeterminismTest) {
SetupTrainerEng("[1,32,0,1 S4,2 L2xy16 Ct1,1,16 S8,1 Lbx100 O1c1]",
"2-D-2-layer-lstm", false, false);
double lstm_2d_err_a = TrainIterations(kTrainerIterations / 3);
double lstm_2d_err_a = TrainIterations(kTrainerIterations);
double act_error_a = trainer_->ActivationError();
double char_error_a = trainer_->CharError();
GenericVector<char> trainer_a_data;
EXPECT_TRUE(trainer_->SaveTrainingDump(NO_BEST_TRAINER, trainer_.get(),
&trainer_a_data));
SetupTrainerEng("[1,32,0,1 S4,2 L2xy16 Ct1,1,16 S8,1 Lbx100 O1c1]",
"2-D-2-layer-lstm", false, false);
double lstm_2d_err_b = TrainIterations(kTrainerIterations / 3);
double lstm_2d_err_b = TrainIterations(kTrainerIterations);
double act_error_b = trainer_->ActivationError();
double char_error_b = trainer_->CharError();
EXPECT_FLOAT_EQ(lstm_2d_err_a, lstm_2d_err_b);
Expand All @@ -136,7 +139,7 @@ TEST_F(LSTMTrainerTest, DeterminismTest) {
EXPECT_FLOAT_EQ(lstm_2d_err_a, lstm_2d_err_b);
EXPECT_FLOAT_EQ(act_error_a, act_error_b);
EXPECT_FLOAT_EQ(char_error_a, char_error_b);
LOG(INFO) << "********** *** ************" ;
LOG(INFO) << "********** *** ************\n" ;
}

// The baseline network against which to test the built-in softmax.
Expand All @@ -145,15 +148,15 @@ TEST_F(LSTMTrainerTest, SoftmaxBaselineTest) {
SetupTrainerEng("[1,1,0,32 Lfx96 O1c1]", "1D-lstm", false, true);
double lstm_uni_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(lstm_uni_err, 60);
EXPECT_GT(lstm_uni_err, 48);
LOG(INFO) << "********** Expected > 48 ** < 60 ************" ;
// EXPECT_GT(lstm_uni_err, 48);
LOG(INFO) << "********** Expected < 60 ************\n" ;
// Check that it works in int mode too.
TestIntMode(kTrainerIterations);
// If we run TestIntMode again, it tests that int_mode networks can
// serialize and deserialize correctly.
double delta = TestIntMode(kTrainerIterations);
// The two tests (both of int mode this time) should be almost identical.
LOG(INFO) << "Delta in Int mode error rates = " << delta;
LOG(INFO) << "Delta in Int mode error rates = " << delta << "\n";
EXPECT_LT(delta, 0.01);
}

Expand All @@ -165,7 +168,7 @@ TEST_F(LSTMTrainerTest, SoftmaxTest) {
SetupTrainerEng("[1,1,0,32 LS96]", "Lstm-+-softmax", false, true);
double lstm_sm_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(lstm_sm_err, 49.0);
LOG(INFO) << "********** Expected < 49 ************" ;
LOG(INFO) << "********** Expected < 49 ************\n" ;
// Check that it works in int mode too.
TestIntMode(kTrainerIterations);
}
Expand All @@ -177,7 +180,7 @@ TEST_F(LSTMTrainerTest, EncodedSoftmaxTest) {
SetupTrainerEng("[1,1,0,32 LE96]", "Lstm-+-softmax", false, true);
double lstm_sm_err = TrainIterations(kTrainerIterations * 2);
EXPECT_LT(lstm_sm_err, 62.0);
LOG(INFO) << "********** Expected < 62 ************" ;
LOG(INFO) << "********** Expected < 62 ************\n" ;
// Check that it works in int mode too.
TestIntMode(kTrainerIterations);
}
Expand Down
6 changes: 3 additions & 3 deletions unittest/lstm_test.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class LSTMTrainerTest : public testing::Test {
GenericVector<STRING> filenames;
filenames.push_back(STRING(TestDataNameToPath(lstmf_file).c_str()));
EXPECT_TRUE(trainer_->LoadAllTrainingData(filenames, CS_SEQUENTIAL, false));
LOG(INFO) << "Setup network:" << model_name;
LOG(INFO) << "Setup network:" << model_name << "\n" ;
}
// Trains for a given number of iterations and returns the char error rate.
double TrainIterations(int max_iterations) {
Expand All @@ -108,8 +108,8 @@ class LSTMTrainerTest : public testing::Test {
iteration = trainer_->training_iteration();
mean_error *= 100.0 / kBatchIterations;
LOG(INFO) << log_str.string();
LOG(INFO) << "Best error = " << best_error;
LOG(INFO) << "Mean error = " << mean_error;
LOG(INFO) << "Best error = " << best_error << "\n" ;
LOG(INFO) << "Mean error = " << mean_error << "\n" ;
if (mean_error < best_error) best_error = mean_error;
} while (iteration < iteration_limit);
LOG(INFO) << "Trainer error rate = " << best_error << "\n";
Expand Down