[FFmpeg-cvslog] Merge commit 'f5d46d332258dcd8ca623019ece1d5e5bb74142b'

[Clément Bœsch]

ffmpeg | branch: master | Clément Bœsch <u at pkh.me> | Sun Mar 19 17:24:49 2017 +0100| [1080b7162f2c6d1e911b91e8097b352a7836b207] | committer: Clément Bœsch

Merge commit 'f5d46d332258dcd8ca623019ece1d5e5bb74142b'

* commit 'f5d46d332258dcd8ca623019ece1d5e5bb74142b':
  vmnc: check that subrectangles fit into their containing rectangles

See 6ba02602aa7fc7d38db582e75b8b093fb3c1608d

This merge keeps our condition against w-i and h-j instead of bw and bh.
One may be more correct than the other, but I'm keeping our behaviour
here for safety reasons.

The style and formatting is merged.

Merged-by: Clément Bœsch <u at pkh.me>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=1080b7162f2c6d1e911b91e8097b352a7836b207
---

 libavcodec/vmnc.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c
index 49abb77..a756dab 100644
--- a/libavcodec/vmnc.c
+++ b/libavcodec/vmnc.c
@@ -287,17 +287,24 @@ static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb,
                     return AVERROR_INVALIDDATA;
                 }
                 for (k = 0; k < rects; k++) {
+                    int rect_x, rect_y, rect_w, rect_h;
                     if (color)
                         fg = vmnc_get_pixel(gb, bpp, c->bigendian);
                     xy = bytestream2_get_byte(gb);
                     wh = bytestream2_get_byte(gb);
-                    if (   (xy >> 4) + (wh >> 4) + 1 > w - i
-                        || (xy & 0xF) + (wh & 0xF)+1 > h - j) {
+
+                    rect_x = xy >> 4;
+                    rect_y = xy & 0xF;
+                    rect_w = (wh >> 4) + 1;
+                    rect_h = (wh & 0xF) + 1;
+
+                    if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) {
                         av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");
                         return AVERROR_INVALIDDATA;
                     }
-                    paint_rect(dst2, xy >> 4, xy & 0xF,
-                               (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
+
+                    paint_rect(dst2, rect_x, rect_y,
+                               rect_w, rect_h, fg, bpp, stride);
                 }
             }
         }


======================================================================

diff --cc libavcodec/vmnc.c
index 49abb77,7a01f1e..a756dab
--- a/libavcodec/vmnc.c
+++ b/libavcodec/vmnc.c
@@@ -291,13 -292,19 +292,19 @@@ static int decode_hextile(VmncContext *
                          fg = vmnc_get_pixel(gb, bpp, c->bigendian);
                      xy = bytestream2_get_byte(gb);
                      wh = bytestream2_get_byte(gb);
-                     if (   (xy >> 4) + (wh >> 4) + 1 > w - i
-                         || (xy & 0xF) + (wh & 0xF)+1 > h - j) {
+ 
+                     rect_x = xy >> 4;
+                     rect_y = xy & 0xF;
+                     rect_w = (wh >> 4) + 1;
+                     rect_h = (wh & 0xF) + 1;
+ 
 -                    if (rect_x + rect_w > bw || rect_y + rect_h > bh) {
 -                        av_log(c->avctx, AV_LOG_ERROR, "Invalid subrect\n");
++                    if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) {
 +                        av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");
                          return AVERROR_INVALIDDATA;
                      }
-                     paint_rect(dst2, xy >> 4, xy & 0xF,
-                                (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
+ 
+                     paint_rect(dst2, rect_x, rect_y,
+                                rect_w, rect_h, fg, bpp, stride);
                  }
              }
          }