fix memory leak - transcoding

Fixes coverity scan issues: 551230, 551229, 507422 and 507421

Author: uknunknown

src/transcoding/transcode/audio.c

@@ -85,7 +85,7 @@ _audio_context_channel_layout(TVHContext *self, AVDictionary **opts, AVChannelLa
 {
     const AVChannelLayout *channel_layouts =
         tvh_codec_profile_audio_get_channel_layouts(self->profile);
-    AVChannelLayout ilayout;
+    AVChannelLayout ilayout = {0};
     av_channel_layout_copy(&ilayout, &self->iavctx->ch_layout);
     AVChannelLayout olayout;
     av_channel_layout_default(&olayout, 0);
@@ -193,7 +193,6 @@ tvh_audio_context_open_encoder(TVHContext *self, AVDictionary **opts)
                         "audio encoder has no suitable channel layout");
         return -1;
     }
-    self->oavctx->ch_layout.nb_channels = self->oavctx->ch_layout.nb_channels;
 #else
     self->oavctx->channel_layout = _audio_context_channel_layout(self, opts);
     if (!self->oavctx->channel_layout) {

src/transcoding/transcode/hwaccels/vaapi.c

@@ -637,6 +637,9 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
         tvherror(LS_VAAPI, "Decode: Failed to Open VAAPI device and create an AVHWDeviceContext for device: "
                             "%s with error code: %s", 
                             ctx->hw_accel_device, av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
@@ -645,6 +648,12 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
     if (!avctx->hw_device_ctx) {
         tvherror(LS_VAAPI, "Decode: Failed to create a hardware device reference for device: %s.", 
                         ctx->hw_accel_device);
+        // unref hw_device_ref
+        av_buffer_unref(&self->hw_device_ref);
+        self->hw_device_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return AVERROR(ENOMEM);
     }
     ctx->hw_accel_ictx = self;
@@ -767,13 +776,22 @@ vaapi_encode_setup_context(AVCodecContext *avctx)
     if ((ret = av_hwdevice_ctx_create(&self->hw_frame_ref, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to open VAAPI device and create an AVHWDeviceContext for it."
                 "Error code: %s",av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
     /* set hw_frames_ctx for encoder's AVCodecContext */
     if ((ret = set_hwframe_ctx(avctx, self->hw_frame_ref)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to set hwframe context."
                 "Error code: %s",av_err2str(ret));
+        // unref hw_frame_ref
+        av_buffer_unref(&self->hw_frame_ref);
+        self->hw_frame_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
     ctx->hw_device_octx = av_buffer_ref(self->hw_frame_ref);