[FFmpeg-cvslog] avcodec/smcenc: more y<height checks to fix invalid reads

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sat Nov 12 23:03:22 2022 +0100| [163f1949ed1eac8598737ba0e917449f20ef2b2d] | committer: Paul B Mahol

avcodec/smcenc: more y<height checks to fix invalid reads

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=163f1949ed1eac8598737ba0e917449f20ef2b2d
---

 libavcodec/smcenc.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/libavcodec/smcenc.c b/libavcodec/smcenc.c
index 33549b8ab4..59b9d9e0ff 100644
--- a/libavcodec/smcenc.c
+++ b/libavcodec/smcenc.c
@@ -302,11 +302,12 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint8_t value = s->color_pairs[color_table_index][1];
                 uint16_t flags = 0;
                 int shift = 15;
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         flags |= (value == pixel_ptr[x + y * stride]) << shift;
                         shift--;
@@ -350,6 +351,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint32_t flags = 0;
                 uint8_t quad[4];
                 int shift = 30;
@@ -357,7 +359,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                 for (int k = 0; k < 4; k++)
                     quad[k] = s->color_quads[color_table_index][k];
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint32_t idx = 0;
@@ -417,6 +419,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint64_t flags = 0;
                 uint8_t octet[8];
                 int shift = 45;
@@ -424,7 +427,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                 for (int k = 0; k < 8; k++)
                     octet[k] = s->color_octets[color_table_index][k];
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint64_t idx = 0;
@@ -451,11 +454,17 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         default:
             bytestream2_put_byte(pb, 0xE0 | (blocks - 1));
             for (int i = 0; i < blocks; i++) {
-                for (int y = 0; y < 4; y++) {
+                const int y_size = FFMIN(4, height - cur_y);
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++)
                         bytestream2_put_byte(pb, pixel_ptr[x + y * stride]);
                 }
 
+                for (int y = y_size; y < 4; y++) {
+                    for (int x = 0; x < 4; x++)
+                        bytestream2_put_byte(pb, 0);
+                }
+
                 ADVANCE_BLOCK(pixel_ptr, row_ptr, 1)
             }
             break;