[FFmpeg-devel] [PATCH 1/5] avcodec/bsf/(hevc|vvc)_mp4toannexb: Ensure extradata_size < INT_MAX

[Andreas Rheinhardt]

AVCodecParameters.extradata_size is an int.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>
---
 libavcodec/bsf/hevc_mp4toannexb.c | 2 +-
 libavcodec/bsf/vvc_mp4toannexb.c  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libavcodec/bsf/hevc_mp4toannexb.c b/libavcodec/bsf/hevc_mp4toannexb.c
index 8eec18f31e..c0df2b79a6 100644
--- a/libavcodec/bsf/hevc_mp4toannexb.c
+++ b/libavcodec/bsf/hevc_mp4toannexb.c
@@ -69,7 +69,7 @@ static int hevc_extradata_to_annexb(AVBSFContext *ctx)
 
             if (!nalu_len ||
                 nalu_len > bytestream2_get_bytes_left(&gb) ||
-                4 + AV_INPUT_BUFFER_PADDING_SIZE + nalu_len > SIZE_MAX - new_extradata_size) {
+                4 + nalu_len > FFMIN(INT_MAX, SIZE_MAX) - AV_INPUT_BUFFER_PADDING_SIZE - new_extradata_size) {
                 ret = AVERROR_INVALIDDATA;
                 goto fail;
             }
diff --git a/libavcodec/bsf/vvc_mp4toannexb.c b/libavcodec/bsf/vvc_mp4toannexb.c
index 36bdae8f49..1b851f3223 100644
--- a/libavcodec/bsf/vvc_mp4toannexb.c
+++ b/libavcodec/bsf/vvc_mp4toannexb.c
@@ -159,7 +159,7 @@ static int vvc_extradata_to_annexb(AVBSFContext *ctx)
 
             if (!nalu_len ||
                 nalu_len > bytestream2_get_bytes_left(&gb) ||
-                4 + AV_INPUT_BUFFER_PADDING_SIZE + nalu_len > SIZE_MAX - new_extradata_size) {
+                4 + nalu_len > FFMIN(INT_MAX, SIZE_MAX) - AV_INPUT_BUFFER_PADDING_SIZE - new_extradata_size) {
                 ret = AVERROR_INVALIDDATA;
                 goto fail;
             }
-- 
2.34.1