[FFmpeg-devel] [PATCH 19/19] lavc: sanitize decoder return value.

[Nicolas George]

With this change, we can guarantee that the return value of
avcodec_decode_<type>() is either an error code or between
0 and the packet size, and not 0 if no frame was decoded.
It is therefore suitable for advancing in the packet data and
looping.

For builds with assert-level >= 2, an assert failure is raised,
making it easier to detect the problems.

Signed-off-by: Nicolas George <nicolas.george at normalesup.org>
---
 libavcodec/utils.c |   13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 53fda1f..1f82a33 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -1498,6 +1498,16 @@ static void apply_param_change(AVCodecContext *avctx, AVPacket *avpkt)
     }
 }
 
+/* Note: we are still vulnerable to decoders decoding a frame and
+   returning 0 for success instead of the used bytes. */
+#define SANITIZE_DECODED_SIZE(ret, pkt, got_frame) {                   \
+    av_assert2((ret) <= (pkt).size);                                   \
+    (ret) = FFMIN((ret), (pkt).size);                                  \
+    av_assert2(!(pkt).size || (got_frame) || (ret));                   \
+    if ((pkt).size && !(got_frame) && !(ret))                          \
+        ret = AVERROR_BUG;                                             \
+}
+
 int attribute_align_arg avcodec_decode_video2(AVCodecContext *avctx, AVFrame *picture,
                          int *got_picture_ptr,
                          const AVPacket *avpkt)
@@ -1540,6 +1550,7 @@ int attribute_align_arg avcodec_decode_video2(AVCodecContext *avctx, AVFrame *pi
             if (picture->format == PIX_FMT_NONE)
                 picture->format = avctx->pix_fmt;
         }
+        SANITIZE_DECODED_SIZE(ret, tmp, *got_picture_ptr);
 
         emms_c(); //needed to avoid an emms_c() call before every return;
 
@@ -1638,6 +1649,7 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
 
         avctx->pkt = &tmp;
         ret = avctx->codec->decode(avctx, frame, got_frame_ptr, &tmp);
+        SANITIZE_DECODED_SIZE(ret, tmp, *got_frame_ptr);
         if (ret >= 0 && *got_frame_ptr) {
             avctx->frame_number++;
             frame->pkt_dts = avpkt->dts;
@@ -1712,6 +1724,7 @@ int avcodec_decode_subtitle2(AVCodecContext *avctx, AVSubtitle *sub,
     *got_sub_ptr = 0;
     avcodec_get_subtitle_defaults(sub);
     ret = avctx->codec->decode(avctx, sub, got_sub_ptr, avpkt);
+    SANITIZE_DECODED_SIZE(ret, *avpkt, *got_sub_ptr);
     if (*got_sub_ptr)
         avctx->frame_number++;
     return ret;
-- 
1.7.10.4