Add additional Encoder and Decoder alphabet test (GH #562)

basecode.cpp

@@ -181,16 +181,16 @@ void BaseN_Decoder::InitializeDecodingLookupArray(int *lookup, const byte *alpha
 
 	for (unsigned int i=0; i<base; i++)
 	{
+		// Debug asserts for 'lookup[alphabet[i]] == -1' removed because the self tests
+		// have unusal tests that try to break the encoders and decoders. Tests include
+		// a string of the same characters. I.,e., a string of stars like '********...'.
 		if (caseInsensitive && isalpha(alphabet[i]))
 		{
-			CRYPTOPP_ASSERT(lookup[toupper(alphabet[i])] == -1);
 			lookup[toupper(alphabet[i])] = i;
-			CRYPTOPP_ASSERT(lookup[tolower(alphabet[i])] == -1);
 			lookup[tolower(alphabet[i])] = i;
 		}
 		else
 		{
-			CRYPTOPP_ASSERT(lookup[alphabet[i]] == -1);
 			lookup[alphabet[i]] = i;
 		}
 	}

validat1.cpp

@@ -3222,19 +3222,36 @@ void MyDecoder::IsolatedInitialize(const NameValuePairs &parameters)
 		MakeParameters(Name::DecodingLookupArray(), GetDecodingLookupArray(), false)(Name::Log2Base(), 6, true)));
 }
 
+struct MyDecoderAlphabet
+{
+	MyDecoderAlphabet() {
+		std::fill(tab, tab+COUNTOF(tab), '*');
+	}
+	byte tab[64];
+};
+
+struct MyDecoderArray
+{
+	MyDecoderArray() {
+		std::fill(tab, tab+COUNTOF(tab), -1);
+	}
+	int tab[256];
+};
+
 const int * MyDecoder::GetDecodingLookupArray()
 {
-	static volatile bool s_initialized = false;
-	static byte s_star[64];
-	static int s_array[256];
+	static bool s_initialized = false;
+	static MyDecoderAlphabet s_alpha;
+	static MyDecoderArray s_array;
 
+	MEMORY_BARRIER();
 	if (!s_initialized)
 	{
-		memset(s_star, '*', 64);
-		InitializeDecodingLookupArray(s_array, s_star, 64, false);
+		InitializeDecodingLookupArray(s_array.tab, s_alpha.tab, COUNTOF(s_alpha.tab), false);
 		s_initialized = true;
+		MEMORY_BARRIER();
 	}
-	return s_array;
+	return s_array.tab;
 }
 
 bool ValidateEncoder()
@@ -3243,6 +3260,7 @@ bool ValidateEncoder()
 	// string of '*'. To round trip a string both IsolatedInitialize
 	// must be called and work correctly.
 	std::cout << "\nCustom encoder validation running...\n\n";
+	bool pass1 = true, pass2 = false;
 
 	int lookup[256];
 	const char alphabet[64+1] =
@@ -3272,11 +3290,37 @@ bool ValidateEncoder()
 	decoder.Put((const byte*) str1.data(), str1.size());
 	decoder.MessageEnd();
 
-	bool pass = (str1 == std::string(expected)) && (str2 == std::string(alphabet, 64));
-	std::cout << (pass ? "passed:" : "FAILED:");
-	std::cout << "  Encoder encode and Decoder decode\n";
+	pass1 = (str1 == std::string(expected)) && pass1;
+	pass1 = (str2 == std::string(alphabet, 64)) && pass1;
 
-	return pass;
+	std::cout << (pass1 ? "passed:" : "FAILED:");
+	std::cout << "  Encode and decode\n";
+
+	// Try forcing an empty message. This is the Monero bug
+	// at https://github.com/weidai11/cryptopp/issues/562.
+	{
+		MyDecoder decoder2;
+		SecByteBlock empty;
+
+		AlgorithmParameters dparams2 = MakeParameters(Name::DecodingLookupArray(),(const int*)lookup);
+		decoder2.IsolatedInitialize(dparams2);
+
+		decoder2.Detach(new Redirector(TheBitBucket()));
+		decoder2.Put(empty.BytePtr(), empty.SizeInBytes());
+		decoder2.MessageEnd();
+
+		// Tame the optimizer
+		volatile lword size = decoder2.MaxRetrievable();
+		lword shadow = size;
+		CRYPTOPP_UNUSED(shadow);
+
+		pass2 = true;
+	}
+
+	std::cout << (pass2 ? "passed:" : "FAILED:");
+	std::cout << "  0-length message\n";
+
+	return pass1 && pass2;
 }
 
 bool ValidateSHACAL2()