diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 52df028ce3f..996d8e8fefd 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -34,6 +34,7 @@
 
 #include "avcodec.h"
 #include "internal.h"
+#include "qsv.h"
 #include "qsv_internal.h"
 #include "qsvdec.h"
 
@@ -48,37 +49,28 @@ int ff_qsv_map_pixfmt(enum AVPixelFormat format)
     }
 }
 
-static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession session)
-{
-    if (!session) {
-        if (!q->internal_qs.session) {
-            int ret = ff_qsv_init_internal_session(avctx, &q->internal_qs, NULL);
-            if (ret < 0)
-                return ret;
-        }
-
-        q->session = q->internal_qs.session;
-    } else {
-        q->session = session;
-    }
-
-    /* make sure the decoder is uninitialized */
-    MFXVideoDECODE_Close(q->session);
-
-    return 0;
-}
-
-int ff_qsv_decode_init(AVCodecContext *avctx, QSVContext *q, mfxSession session)
+int ff_qsv_decode_init(AVCodecContext *avctx, QSVContext *q)
 {
     mfxVideoParam param = { { 0 } };
     int ret;
 
-    ret = qsv_init_session(avctx, q, session);
-    if (ret < 0) {
-        av_log(avctx, AV_LOG_ERROR, "Error initializing an MFX session\n");
-        return ret;
+    q->iopattern  = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
+
+    if (avctx->hwaccel_context) {
+        AVQSVContext *qsv = avctx->hwaccel_context;
+
+        q->session        = qsv->session;
+        q->iopattern      = qsv->iopattern;
+        q->ext_buffers    = qsv->ext_buffers;
+        q->nb_ext_buffers = qsv->nb_ext_buffers;
     }
+    if (!q->session) {
+        ret = ff_qsv_init_internal_session(avctx, &q->internal_qs, NULL);
+        if (ret < 0)
+            return ret;
 
+        q->session = q->internal_qs.session;
+    }
 
     ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
     if (ret < 0)
diff --git a/libavcodec/qsvdec.h b/libavcodec/qsvdec.h
index 210e9a97860..4184d3391af 100644
--- a/libavcodec/qsvdec.h
+++ b/libavcodec/qsvdec.h
@@ -57,7 +57,7 @@ typedef struct QSVContext {
 
 int ff_qsv_map_pixfmt(enum AVPixelFormat format);
 
-int ff_qsv_decode_init(AVCodecContext *s, QSVContext *q, mfxSession session);
+int ff_qsv_decode_init(AVCodecContext *s, QSVContext *q);
 
 int ff_qsv_decode(AVCodecContext *s, QSVContext *q,
                   AVFrame *frame, int *got_frame,
diff --git a/libavcodec/qsvdec_h264.c b/libavcodec/qsvdec_h264.c
index 7eb7a6c134c..1e9dff13c12 100644
--- a/libavcodec/qsvdec_h264.c
+++ b/libavcodec/qsvdec_h264.c
@@ -33,9 +33,7 @@
 
 #include "avcodec.h"
 #include "internal.h"
-#include "qsv_internal.h"
 #include "qsvdec.h"
-#include "qsv.h"
 
 typedef struct QSVH264Context {
     AVClass *class;
@@ -130,8 +128,6 @@ static av_cold int qsv_decode_init(AVCodecContext *avctx)
     }
     s->parser->flags |= PARSER_FLAG_COMPLETE_FRAMES;
 
-    s->qsv.iopattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
-
     return 0;
 fail:
     qsv_decode_close(avctx);
@@ -157,7 +153,6 @@ static int qsv_process_data(AVCodecContext *avctx, AVFrame *frame,
     if (s->parser->format       != s->orig_pix_fmt    ||
         s->parser->coded_width  != avctx->coded_width ||
         s->parser->coded_height != avctx->coded_height) {
-        mfxSession session = NULL;
 
         enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_QSV,
                                            AV_PIX_FMT_NONE,
@@ -187,15 +182,7 @@ static int qsv_process_data(AVCodecContext *avctx, AVFrame *frame,
 
         avctx->pix_fmt = ret;
 
-        if (avctx->hwaccel_context) {
-            AVQSVContext *user_ctx = avctx->hwaccel_context;
-            session               = user_ctx->session;
-            s->qsv.iopattern      = user_ctx->iopattern;
-            s->qsv.ext_buffers    = user_ctx->ext_buffers;
-            s->qsv.nb_ext_buffers = user_ctx->nb_ext_buffers;
-        }
-
-        ret = ff_qsv_decode_init(avctx, &s->qsv, session);
+        ret = ff_qsv_decode_init(avctx, &s->qsv);
         if (ret < 0)
             goto reinit_fail;
     }