[FFmpeg-devel] [PATCH 2/2] avcodec/v4l2_m2m{enc, dec}: Fix leaks when init fails

[Andriy Gelman]

From: Andriy Gelman <andriy.gelman at gmail.com>

v4l2_m2m_{enc,dec} doesn't call its close function when initialization
fails because FF_CODEC_CAP_INIT_CLEANUP is not set. This causes a couple
possible leaks, which are fixed in the commit.

Signed-off-by: Andriy Gelman <andriy.gelman at gmail.com>
---
 libavcodec/v4l2_m2m_dec.c | 9 ++++++---
 libavcodec/v4l2_m2m_enc.c | 7 ++++++-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
index d666edffe46..a515f3ca720 100644
--- a/libavcodec/v4l2_m2m_dec.c
+++ b/libavcodec/v4l2_m2m_dec.c
@@ -205,13 +205,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx)
     ret = ff_v4l2_m2m_codec_init(priv);
     if (ret) {
         av_log(avctx, AV_LOG_ERROR, "can't configure decoder\n");
-        s->self_ref = NULL;
-        av_buffer_unref(&priv->context_ref);
+        ff_v4l2_m2m_codec_end(priv);
 
         return ret;
     }
 
-    return v4l2_prepare_decoder(s);
+    ret = v4l2_prepare_decoder(s);
+    if (ret < 0)
+        ff_v4l2_m2m_codec_end(priv);
+
+    return ret;
 }
 
 static av_cold int v4l2_decode_close(AVCodecContext *avctx)
diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c
index ff9ff267ea2..ca58669a976 100644
--- a/libavcodec/v4l2_m2m_enc.c
+++ b/libavcodec/v4l2_m2m_enc.c
@@ -329,10 +329,15 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx)
     if (pix_fmt_output != avctx->pix_fmt) {
         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt_output);
         av_log(avctx, AV_LOG_ERROR, "Encoder requires %s pixel format.\n", desc->name);
+        ff_v4l2_m2m_codec_end(priv);
         return AVERROR(EINVAL);
     }
 
-    return v4l2_prepare_encoder(s);
+    ret = v4l2_prepare_encoder(s);
+    if (ret < 0)
+        ff_v4l2_m2m_codec_end(priv);
+
+    return ret;
 }
 
 static av_cold int v4l2_encode_close(AVCodecContext *avctx)
-- 
2.25.0