[FFmpeg-devel] [PATCH 2/2] vmdav: unbreak decoding of samples from game The Last Dynasty

[Paul B Mahol]

This fixes video output with samples HG060808.VMD and
HG060810.VMD. Regression since c0cbe36b18ab3e.

While here show warning if decoding is aborted for some reason.

Signed-off-by: Paul B Mahol <onemda at gmail.com>
---
 libavcodec/vmdav.c | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index f73a8eb..40882ed 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -192,7 +192,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
     return bytestream2_tell(&gb);
 }
 
-static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
+static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
 {
     int i;
     unsigned int *palette32;
@@ -211,18 +211,20 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
 
     frame_x = AV_RL16(&s->buf[6]);
     frame_y = AV_RL16(&s->buf[8]);
-    frame_width = AV_RL16(&s->buf[10]) - frame_x + 1;
-    frame_height = AV_RL16(&s->buf[12]) - frame_y + 1;
+    frame_width = FFMIN(AV_RL16(&s->buf[10]) - frame_x + 1, s->avctx->width - frame_x);
+    frame_height = FFMIN(AV_RL16(&s->buf[12]) - frame_y + 1, s->avctx->height - frame_y);
     if (frame_x < 0 || frame_width < 0 ||
         frame_x >= s->avctx->width ||
         frame_width > s->avctx->width ||
-        frame_x + frame_width > s->avctx->width)
-        return;
+        frame_x + frame_width > s->avctx->width) {
+        return AVERROR_INVALIDDATA;
+    }
     if (frame_y < 0 || frame_height < 0 ||
         frame_y >= s->avctx->height ||
         frame_height > s->avctx->height ||
-        frame_y + frame_height > s->avctx->height)
-        return;
+        frame_y + frame_height > s->avctx->height) {
+        return AVERROR_INVALIDDATA;
+    }
 
     if ((frame_width == s->avctx->width && frame_height == s->avctx->height) &&
         (frame_x || frame_y)) {
@@ -262,7 +264,7 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
         /* originally UnpackFrame in VAG's code */
         bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer);
         if (bytestream2_get_bytes_left(&gb) < 1)
-            return;
+            return AVERROR_INVALIDDATA;
         meth = bytestream2_get_byteu(&gb);
         if (meth & 0x80) {
             lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb),
@@ -282,13 +284,13 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
                     if (len & 0x80) {
                         len = (len & 0x7F) + 1;
                         if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len)
-                            return;
+                            return AVERROR_INVALIDDATA;
                         bytestream2_get_bufferu(&gb, &dp[ofs], len);
                         ofs += len;
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -328,7 +330,7 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -343,6 +345,8 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
             break;
         }
     }
+
+    return 0;
 }
 
 static av_cold int vmdvideo_decode_init(AVCodecContext *avctx)
@@ -405,7 +409,8 @@ static int vmdvideo_decode_frame(AVCodecContext *avctx,
     if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0)
         return ret;
 
-    vmd_decode(s, frame);
+    if (vmd_decode(s, frame) < 0)
+        av_log(avctx, AV_LOG_WARNING, "decode error\n");
 
     /* make the palette available on the way out */
     memcpy(frame->data[1], s->palette, PALETTE_COUNT * 4);
-- 
1.7.11.2