[FFmpeg-cvslog] avcodec/rl2: Fix undefined pointer arithmetic

[Andreas Rheinhardt]

ffmpeg | branch: master | Andreas Rheinhardt <andreas.rheinhardt at outlook.com> | Wed Sep 28 17:58:00 2022 +0200| [87f243911e0769b560162bce1a356699784e1fea] | committer: Andreas Rheinhardt

avcodec/rl2: Fix undefined pointer arithmetic

Don't increment back_frame if it does not correspond
to a real buffer. To do this, handle copying from
the back frame separately from the "use coded value"
codepath; also use memcpy for the former, as the
chunks here are typically worth it.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=87f243911e0769b560162bce1a356699784e1fea
---

 libavcodec/rl2.c | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c
index 76982f0426..7938ef1d92 100644
--- a/libavcodec/rl2.c
+++ b/libavcodec/rl2.c
@@ -69,13 +69,16 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
     uint8_t *line_end;
 
     /** copy start of the background frame */
+    if (s->back_frame) {
     for (i = 0; i <= base_y; i++) {
-        if (s->back_frame)
             memcpy(out, back_frame, s->avctx->width);
         out        += stride;
         back_frame += s->avctx->width;
     }
     back_frame += base_x - s->avctx->width;
+    } else {
+        out += stride * (base_y + 1);
+    }
     line_end    = out - stride_adj;
     out        += base_x - stride;
 
@@ -89,16 +92,32 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
             len = *in++;
             if (!len)
                 break;
+            val &= 0x7F;
         }
 
-        if (s->back_frame)
+        if (back_frame) {
+            if (!val) {
+                do {
+                    size_t copy = FFMIN(line_end - out, len);
+                    memcpy(out, back_frame, copy);
+                    out        += copy;
+                    back_frame += copy;
+                    len        -= copy;
+                    if (out == line_end) {
+                        if (out == out_end)
+                            return;
+                        out      += stride_adj;
+                        line_end += stride;
+                    }
+                } while (len > 0);
+                continue;
+            }
+            back_frame += len;
             val |= 0x80;
-        else
-            val &= ~0x80;
+        }
 
         while (len--) {
-            *out++ = (val == 0x80) ? *back_frame : val;
-            back_frame++;
+            *out++ = val;
             if (out == line_end) {
                 if (out == out_end)
                     return;
@@ -164,7 +183,9 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
     back_size = avctx->extradata_size - EXTRADATA1_SIZE;
 
     if (back_size > 0) {
-        uint8_t *back_frame = av_mallocz(avctx->width*avctx->height);
+        /* The 254 are padding to ensure that pointer arithmetic stays within
+         * the buffer. */
+        uint8_t *back_frame = av_mallocz(avctx->width * avctx->height + 254);
         if (!back_frame)
             return AVERROR(ENOMEM);
         rl2_rle_decode(s, avctx->extradata + EXTRADATA1_SIZE, back_size,