diff --git a/src/lib-ssl-iostream/iostream-openssl.c b/src/lib-ssl-iostream/iostream-openssl.c
index 8dd0541af8..45de412858 100644
--- a/src/lib-ssl-iostream/iostream-openssl.c
+++ b/src/lib-ssl-iostream/iostream-openssl.c
@@ -303,6 +303,8 @@ openssl_iostream_create(struct ssl_iostream_context *ctx, const char *host,
 	o_stream_uncork(ssl_io->plain_output);
 
 	*input = openssl_i_stream_create_ssl(ssl_io);
+	ssl_io->ssl_input = *input;
+
 	*output = openssl_o_stream_create_ssl(ssl_io);
 	i_stream_set_name(*input, t_strconcat("SSL ",
 		i_stream_get_name(ssl_io->plain_input), NULL));
@@ -312,7 +314,6 @@ openssl_iostream_create(struct ssl_iostream_context *ctx, const char *host,
 	if (ssl_io->plain_output->real_stream->error_handling_disabled)
 		o_stream_set_no_error_handling(*output, TRUE);
 
-	ssl_io->ssl_input = *input;
 	ssl_io->ssl_output = *output;
 	*iostream_r = ssl_io;
 	return 0;
diff --git a/src/lib-ssl-iostream/ostream-openssl.c b/src/lib-ssl-iostream/ostream-openssl.c
index 3a71758c89..e6e8aa5686 100644
--- a/src/lib-ssl-iostream/ostream-openssl.c
+++ b/src/lib-ssl-iostream/ostream-openssl.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "buffer.h"
+#include "istream.h"
 #include "ostream-private.h"
 #include "iostream-openssl.h"
 
@@ -23,8 +24,10 @@ o_stream_ssl_close(struct iostream_private *stream, bool close_parent)
 static void o_stream_ssl_destroy(struct iostream_private *stream)
 {
 	struct ssl_ostream *sstream = (struct ssl_ostream *)stream;
+	struct istream *ssl_input = sstream->ssl_io->ssl_input;
 
 	sstream->ssl_io->ssl_output = NULL;
+	i_stream_unref(&ssl_input);
 	ssl_iostream_unref(&sstream->ssl_io);
 	buffer_free(&sstream->buffer);
 }
@@ -245,6 +248,13 @@ struct ostream *openssl_o_stream_create_ssl(struct ssl_iostream *ssl_io)
 
 	ssl_io->refcount++;
 
+	/* When ostream is destroyed, it's flushed. With iostream-ssl the
+	   flushing requires both istream and ostream to be available. The
+	   istream is referenced here to make sure it's not destroyed before
+	   the ostream. */
+	i_assert(ssl_io->ssl_input != NULL);
+	i_stream_ref(ssl_io->ssl_input);
+
 	sstream = i_new(struct ssl_ostream, 1);
 	sstream->ssl_io = ssl_io;
 	sstream->ostream.max_buffer_size =