[FFmpeg-cvslog] avcodec/cbs: Fix potential overflow

[Andreas Rheinhardt]

ffmpeg | branch: release/4.2 | Andreas Rheinhardt <andreas.rheinhardt at gmail.com> | Sun Nov 17 08:34:36 2019 +0100| [4667920455c0d52c25835ff81098254213f2d018] | committer: James Almer

avcodec/cbs: Fix potential overflow

The number of bits in a PutBitContext must fit into an int, yet nothing
guaranteed the size argument cbs_write_unit_data() uses in init_put_bits()
to be in the range 0..INT_MAX / 8. This has been changed.

Furthermore, the check 8 * data_size > data_bit_start that there is
data beyond the initial padding when writing mpeg2 or H.264/5 slices
could also overflow, so divide it by 8 to get an equivalent check
without this problem.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at gmail.com>
(cherry picked from commit cda3e8ca04c0e343f5b60fda8fb467936e176f33)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4667920455c0d52c25835ff81098254213f2d018
---

 libavcodec/cbs.c       | 4 +++-
 libavcodec/cbs_h2645.c | 2 +-
 libavcodec/cbs_mpeg2.c | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libavcodec/cbs.c b/libavcodec/cbs.c
index 047f4fb898..0ce66170ec 100644
--- a/libavcodec/cbs.c
+++ b/libavcodec/cbs.c
@@ -309,7 +309,9 @@ static int cbs_write_unit_data(CodedBitstreamContext *ctx,
     if (ret < 0) {
         if (ret == AVERROR(ENOSPC)) {
             // Overflow.
-            ctx->write_buffer_size *= 2;
+            if (ctx->write_buffer_size == INT_MAX / 8)
+                return AVERROR(ENOMEM);
+            ctx->write_buffer_size = FFMIN(2 * ctx->write_buffer_size, INT_MAX / 8);
             goto reallocate_and_try_again;
         }
         // Write failed for some other reason.
diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index ba442b62ed..e272b0bcf6 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -1101,7 +1101,7 @@ static int cbs_h2645_write_slice_data(CodedBitstreamContext *ctx,
     const uint8_t *pos = data + data_bit_start / 8;
 
     av_assert0(data_bit_start >= 0 &&
-               8 * data_size > data_bit_start);
+               data_size > data_bit_start / 8);
 
     if (data_size * 8 + 8 > put_bits_left(pbc))
         return AVERROR(ENOSPC);
diff --git a/libavcodec/cbs_mpeg2.c b/libavcodec/cbs_mpeg2.c
index a49a403b26..98a22e3a68 100644
--- a/libavcodec/cbs_mpeg2.c
+++ b/libavcodec/cbs_mpeg2.c
@@ -301,7 +301,7 @@ static int cbs_mpeg2_write_slice(CodedBitstreamContext *ctx,
         uint8_t *pos = slice->data + slice->data_bit_start / 8;
 
         av_assert0(slice->data_bit_start >= 0 &&
-                   8 * slice->data_size > slice->data_bit_start);
+                   slice->data_size > slice->data_bit_start / 8);
 
         if (slice->data_size * 8 + 8 > put_bits_left(pbc))
             return AVERROR(ENOSPC);