[FFmpeg-devel] [PATCH 2/3] avcodec/cbs: Fix potential overflow

[Andreas Rheinhardt]

The number of bits in a PutBitsContext must fit into an int, yet the
various cbs functions to write units did not check that the argument in
init_put_bits is in the range 0..INT_MAX / 8. This has been changed.

Furthermore, the check 8 * data_size > data_bit_start that there is
data beyond the initial padding could also overflow, so divide it by 8
to get an equivalent check without this problem.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at gmail.com>
---
 libavcodec/cbs_av1.c   | 4 +++-
 libavcodec/cbs_h2645.c | 6 ++++--
 libavcodec/cbs_jpeg.c  | 4 +++-
 libavcodec/cbs_mpeg2.c | 6 ++++--
 libavcodec/cbs_vp9.c   | 4 +++-
 5 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index cca112ecf8..f091c5ac41 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -1239,7 +1239,9 @@ static int cbs_av1_write_unit(CodedBitstreamContext *ctx,
     err = cbs_av1_write_obu(ctx, unit, &pbc);
     if (err == AVERROR(ENOSPC)) {
         // Overflow.
-        priv->write_buffer_size *= 2;
+        if (priv->write_buffer_size == INT_MAX / 8)
+            return AVERROR(ENOMEM);
+        priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8);
         goto reallocate_and_try_again;
     }
     if (err < 0)
diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index 5dd371153a..6025035e22 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -1101,7 +1101,7 @@ static int cbs_h2645_write_slice_data(CodedBitstreamContext *ctx,
     const uint8_t *pos = data + data_bit_start / 8;
 
     av_assert0(data_bit_start >= 0 &&
-               8 * data_size > data_bit_start);
+               data_size > data_bit_start / 8);
 
     if (data_size * 8 + 8 > put_bits_left(pbc))
         return AVERROR(ENOSPC);
@@ -1411,7 +1411,9 @@ static int cbs_h2645_write_nal_unit(CodedBitstreamContext *ctx,
 
     if (err == AVERROR(ENOSPC)) {
         // Overflow.
-        priv->write_buffer_size *= 2;
+        if (priv->write_buffer_size == INT_MAX / 8)
+            return AVERROR(ENOMEM);
+        priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8);
         goto reallocate_and_try_again;
     }
     // Overflow but we didn't notice.
diff --git a/libavcodec/cbs_jpeg.c b/libavcodec/cbs_jpeg.c
index a20f062f1b..71679ef5df 100644
--- a/libavcodec/cbs_jpeg.c
+++ b/libavcodec/cbs_jpeg.c
@@ -406,7 +406,9 @@ static int cbs_jpeg_write_unit(CodedBitstreamContext *ctx,
 
     if (err == AVERROR(ENOSPC)) {
         // Overflow.
-        priv->write_buffer_size *= 2;
+        if (priv->write_buffer_size == INT_MAX / 8)
+            return AVERROR(ENOMEM);
+        priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8);
         goto reallocate_and_try_again;
     }
     if (err < 0) {
diff --git a/libavcodec/cbs_mpeg2.c b/libavcodec/cbs_mpeg2.c
index 52252093b3..8591ada7bf 100644
--- a/libavcodec/cbs_mpeg2.c
+++ b/libavcodec/cbs_mpeg2.c
@@ -337,7 +337,7 @@ static int cbs_mpeg2_write_slice(CodedBitstreamContext *ctx,
         uint8_t *pos = slice->data + slice->data_bit_start / 8;
 
         av_assert0(slice->data_bit_start >= 0 &&
-                   8 * slice->data_size > slice->data_bit_start);
+                   slice->data_size > slice->data_bit_start / 8);
 
         if (slice->data_size * 8 + 8 > put_bits_left(pbc))
             return AVERROR(ENOSPC);
@@ -400,7 +400,9 @@ static int cbs_mpeg2_write_unit(CodedBitstreamContext *ctx,
 
     if (err == AVERROR(ENOSPC)) {
         // Overflow.
-        priv->write_buffer_size *= 2;
+        if (priv->write_buffer_size == INT_MAX / 8)
+            return AVERROR(ENOMEM);
+        priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8);
         goto reallocate_and_try_again;
     }
     if (err < 0) {
diff --git a/libavcodec/cbs_vp9.c b/libavcodec/cbs_vp9.c
index ebb4c573ea..88191fee3d 100644
--- a/libavcodec/cbs_vp9.c
+++ b/libavcodec/cbs_vp9.c
@@ -548,7 +548,9 @@ static int cbs_vp9_write_unit(CodedBitstreamContext *ctx,
 
     err = cbs_vp9_write_frame(ctx, &pbc, frame);
     if (err == AVERROR(ENOSPC)) {
-        priv->write_buffer_size *= 2;
+        if (priv->write_buffer_size == INT_MAX / 8)
+            return AVERROR(ENOMEM);
+        priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8);
         goto reallocate_and_try_again;
     }
     if (err < 0)
-- 
2.20.1