Add support for OpenSSL 1.1 and replace deprecated function calls

See #16 for details. Thanks to
Chris West @FauxFaux for the initial patch!

appveyor.yml

@@ -19,7 +19,7 @@ configuration:
 environment:
   VisualStudioVersion: 10.0
   Generator: Visual Studio 10
-  OPENSSL_VERSION: 1.0.2f
+  OPENSSL_VERSION: 1.1.0f
   PCRE_VERSION: 8.38
   CMAKE_UNIBUILD_DIR: '%APPVEYOR_BUILD_FOLDER%\..\cmake'
 

lib/crypto/CipherContext.cpp

@@ -50,7 +50,7 @@ CipherContext::~CipherContext()
 	if(mInitialised)
 	{
 		// Clean up
-		EVP_CIPHER_CTX_cleanup(&ctx);
+		BOX_OPENSSL_CLEANUP_CTX(ctx);
 		mInitialised = false;
 	}
 #ifdef HAVE_OLD_SSL
@@ -109,15 +109,15 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes
 	mFunction = Function;
 
 	// Initialise the cipher
-#ifndef HAVE_OLD_SSL
-	EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does
-
-	if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL,
-		(mFunction == Encrypt) ? 1 : 0) != 1)
-#else
+#ifdef HAVE_OLD_SSL
 	// Use old version of init call
 	if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL,
 		(mFunction == Encrypt) ? 1 : 0) != 1)
+#else
+	BOX_OPENSSL_INIT_CTX(ctx);
+
+	if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), rDescription.GetCipher(), NULL, NULL, NULL,
+		(mFunction == Encrypt) ? 1 : 0) != 1)
 #endif
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure,
@@ -130,7 +130,7 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes
 		mCipherName = rDescription.GetFullName();
 #ifndef HAVE_OLD_SSL
 		// Let the description set up everything else
-		rDescription.SetupParameters(&ctx);
+		rDescription.SetupParameters(BOX_OPENSSL_CTX(ctx));
 #else
 		// With the old version, a copy needs to be taken first.
 		mpDescription = rDescription.Clone();
@@ -145,7 +145,7 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure,
 			"Failed to configure " << mCipherName << " cipher: " <<
 			LogError("configuring cipher"));
-		EVP_CIPHER_CTX_cleanup(&ctx);
+		BOX_OPENSSL_CLEANUP_CTX(ctx);
 		throw;
 	}
 
@@ -166,7 +166,7 @@ void CipherContext::Reset()
 	if(mInitialised)
 	{
 		// Clean up
-		EVP_CIPHER_CTX_cleanup(&ctx);
+		EVP_CIPHER_CTX_cleanup(BOX_OPENSSL_CTX(ctx));
 		mInitialised = false;
 	}
 #ifdef HAVE_OLD_SSL
@@ -203,7 +203,7 @@ void CipherContext::Begin()
 	}
 
 	// Initialise the cipher context again
-	if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1)
+	if(EVP_CipherInit(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, -1) != 1)
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure,
 			"Failed to reset " << mCipherName << " cipher: " <<
@@ -251,14 +251,15 @@ int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuf
 	}
 
 	// Check output buffer size
-	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx)))
+	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx))))
 	{
 		THROW_EXCEPTION(CipherException, OutputBufferTooSmall);
 	}
 
 	// Do the transform
 	int outLength = OutLength;
-	if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1)
+	if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength,
+		(unsigned char*)pInBuffer, InLength) != 1)
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure,
 			"Failed to " << GetFunction() << " (update) " <<
@@ -300,15 +301,15 @@ int CipherContext::Final(void *pOutBuffer, int OutLength)
 	}
 
 	// Check output buffer size
-	if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx)))
+	if(OutLength < (2 * EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx))))
 	{
 		THROW_EXCEPTION(CipherException, OutputBufferTooSmall);
 	}
 
 	// Do the transform
 	int outLength = OutLength;
 #ifndef HAVE_OLD_SSL
-	if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1)
+	if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength) != 1)
 	{
 		mWithinTransform = false;
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure,
@@ -340,11 +341,11 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut)
 	// Old version needs to use a different form, and then set up the cipher again for next time around
 	int outLength = rOutLengthOut;
 	// Have to emulate padding off...
-	int blockSize = EVP_CIPHER_CTX_block_size(&ctx);
+	int blockSize = EVP_CIPHER_CTX_block_size(ctx);
 	if(mPaddingOn)
 	{
 		// Just use normal final call
-		if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1)
+		if(EVP_CipherFinal(ctx, Buffer, &outLength) != 1)
 		{
 			THROW_EXCEPTION(CipherException, EVPFinalFailure)
 		}
@@ -357,13 +358,13 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut)
 		{
 			// NASTY -- fiddling around with internals like this is bad.
 			// But only way to get this working on old versions of OpenSSL.
-			if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0)
+			if(!EVP_EncryptUpdate(ctx,Buffer,&outLength,ctx.buf,0)
 				|| outLength != blockSize)
 			{
 				THROW_EXCEPTION(CipherException, EVPFinalFailure)
 			}
 			// Clean up
-			EVP_CIPHER_CTX_cleanup(&ctx);
+			EVP_CIPHER_CTX_free(ctx);
 		}
 		else
 		{
@@ -421,7 +422,7 @@ int CipherContext::InSizeForOutBufferSize(int OutLength)
 
 	// Strictly speaking, the *2 is unnecessary. However... 
 	// Final() is paranoid, and requires two input blocks of space to work.
-	return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2);
+	return OutLength - (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 2);
 }
 
 // --------------------------------------------------------------------------
@@ -442,7 +443,7 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength)
 
 	// Final() is paranoid, and requires two input blocks of space to work, and so we need to add
 	// three blocks on to be absolutely sure.
-	return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3);
+	return InLength + (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 3);
 }
 
 
@@ -469,7 +470,7 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p
 	}
 
 	// Check output buffer size
-	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx)))
+	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx))))
 	{
 		// Check if padding is off, in which case the buffer can be smaller
 		if(!mPaddingOn && OutLength <= InLength)
@@ -483,7 +484,7 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p
 	}
 
 	// Initialise the cipher context again
-	if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1)
+	if(EVP_CipherInit(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, -1) != 1)
 	{
 		THROW_EXCEPTION(CipherException, EVPInitFailure)
 	}
@@ -493,7 +494,8 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p
 
 	// Update
 	outLength = OutLength;
-	if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1)
+	if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength,
+		(unsigned char*)pInBuffer, InLength) != 1)
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure,
 			"Failed to " << GetFunction() << " (update) " <<
@@ -502,15 +504,16 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p
 
 	// Finalise
 	int outLength2 = OutLength - outLength;
-#ifndef HAVE_OLD_SSL
-	if(EVP_CipherFinal(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1)
+#ifdef HAVE_OLD_SSL
+	OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2);
+#else
+	if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), ((unsigned char*)pOutBuffer) + outLength,
+		&outLength2) != 1)
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure,
 			"Failed to " << GetFunction() << " (final) " <<
 			mCipherName << " cipher: " << LogError(GetFunction()));
 	}
-#else
-	OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2);
 #endif
 	outLength += outLength2;
 
@@ -533,7 +536,7 @@ int CipherContext::GetIVLength()
 		THROW_EXCEPTION(CipherException, NotInitialised)
 	}
 
-	return EVP_CIPHER_CTX_iv_length(&ctx);
+	return EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx));
 }
 
 
@@ -560,7 +563,7 @@ void CipherContext::SetIV(const void *pIV)
 	}
 
 	// Set IV
-	if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1)
+	if(EVP_CipherInit(BOX_OPENSSL_CTX(ctx), NULL, NULL, (unsigned char *)pIV, -1) != 1)
 	{
 		THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure,
 			"Failed to " << GetFunction() << " (set IV) " <<
@@ -601,7 +604,7 @@ const void *CipherContext::SetRandomIV(int &rLengthOut)
 	}
 
 	// Get length of IV
-	unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx);
+	unsigned int ivLen = EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx));
 	if(ivLen > sizeof(mGeneratedIV))
 	{
 		THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded)
@@ -628,7 +631,7 @@ const void *CipherContext::SetRandomIV(int &rLengthOut)
 void CipherContext::UsePadding(bool Padding)
 {
 #ifndef HAVE_OLD_SSL
-	if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1)
+	if(EVP_CIPHER_CTX_set_padding(BOX_OPENSSL_CTX(ctx), Padding) != 1)
 	{
 		THROW_EXCEPTION(CipherException, EVPSetPaddingFailure)
 	}

lib/crypto/CipherContext.h

@@ -19,6 +19,22 @@ class CipherDescription;
 
 #define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH		32
 
+// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See
+// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h
+// for the gory details.
+#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1
+#	define BOX_OPENSSL_INIT_CTX(ctx) ctx = EVP_CIPHER_CTX_new();
+#	define BOX_OPENSSL_CTX(ctx) ctx
+#	define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_free(ctx)
+typedef EVP_CIPHER_CTX* BOX_EVP_CIPHER_CTX;
+#else // OpenSSL < 1.1
+#	define BOX_OPENSSL_INIT_CTX(ctx) EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does
+#	define BOX_OPENSSL_CTX(ctx) &ctx
+#	define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_cleanup(&ctx)
+typedef EVP_CIPHER_CTX BOX_EVP_CIPHER_CTX;
+#endif
+
+
 // --------------------------------------------------------------------------
 //
 // Class
@@ -74,7 +90,7 @@ class CipherContext
 #endif
 
 private:
-	EVP_CIPHER_CTX ctx;
+	BOX_EVP_CIPHER_CTX ctx;
 	bool mInitialised;
 	bool mWithinTransform;
 	bool mPaddingOn;

lib/server/TLSContext.cpp

@@ -23,6 +23,17 @@
 #define MAX_VERIFICATION_DEPTH		2
 #define CIPHER_LIST					"ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"
 
+// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See
+// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h
+// for the gory details.
+#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1
+#	define BOX_TLS_SERVER_METHOD TLS_server_method
+#	define BOX_TLS_CLIENT_METHOD TLS_client_method
+#else // OpenSSL < 1.1
+#	define BOX_TLS_SERVER_METHOD TLSv1_server_method
+#	define BOX_TLS_CLIENT_METHOD TLSv1_client_method
+#endif
+
 // --------------------------------------------------------------------------
 //
 // Function
@@ -67,7 +78,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c
 		::SSL_CTX_free(mpContext);
 	}
 
-	mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method());
+	mpContext = ::SSL_CTX_new(AsServer ? BOX_TLS_SERVER_METHOD() : BOX_TLS_CLIENT_METHOD());
 	if(mpContext == NULL)
 	{
 		THROW_EXCEPTION(ServerException, TLSAllocationFailed)