[FFmpeg-cvslog] avcodec/dvbsubenc: Check nb_colors before using it

[Andreas Rheinhardt]

ffmpeg | branch: master | Andreas Rheinhardt <andreas.rheinhardt at outlook.com> | Wed Apr 16 11:03:53 2025 +0200| [3cf21217b5063d9758582d1cab325e3c91c848f2] | committer: Andreas Rheinhardt

avcodec/dvbsubenc: Check nb_colors before using it

Avoids a potential overflow when multiplying nb_colors by 6.
Also make the nb_colors check a bit more strict.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=3cf21217b5063d9758582d1cab325e3c91c848f2
---

 libavcodec/dvbsubenc.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/libavcodec/dvbsubenc.c b/libavcodec/dvbsubenc.c
index 4844c3deb5..113bb320a9 100644
--- a/libavcodec/dvbsubenc.c
+++ b/libavcodec/dvbsubenc.c
@@ -329,24 +329,23 @@ static int dvbsub_encode(AVCodecContext *avctx, uint8_t *outbuf, int buf_size,
 
     if (h->num_rects) {
         for (clut_id = 0; clut_id < h->num_rects; clut_id++) {
-            if (buf_size < 6 + h->rects[clut_id]->nb_colors * 6)
-                return AVERROR_BUFFER_TOO_SMALL;
-
             /* CLUT segment */
 
-            if (h->rects[clut_id]->nb_colors <= 4) {
+            if (h->rects[clut_id]->nb_colors <= 4U) {
                 /* 2 bpp, some decoders do not support it correctly */
                 bpp_index = 0;
-            } else if (h->rects[clut_id]->nb_colors <= 16) {
+            } else if (h->rects[clut_id]->nb_colors <= 16U) {
                 /* 4 bpp, standard encoding */
                 bpp_index = 1;
-            } else if (h->rects[clut_id]->nb_colors <= 256) {
+            } else if (h->rects[clut_id]->nb_colors <= 256U) {
                 /* 8 bpp, standard encoding */
                 bpp_index = 2;
             } else {
                 return AVERROR(EINVAL);
             }
 
+            if (buf_size < 6 + h->rects[clut_id]->nb_colors * 6)
+                return AVERROR_BUFFER_TOO_SMALL;
 
             /* CLUT segment */
             *q++ = 0x0f; /* sync byte */