[FFmpeg-devel] [PATCH] Fix quadratic memory use in ff_h2645_extract_rbsp() when multiple NALUs exist in packet.

[Nikolas Bowe]

Found via fuzzing.
/tmp/poc is a 1 MB mpegts file generated via fuzzing, where 1 packet has many NALUs
Before this change:
  $ /usr/bin/time -f "\t%M Max Resident Set Size (Kb)"  ./ffprobe /tmp/poc 2>&1 | tail -n 1
  	2158192 Max Resident Set Size (Kb)
After this change:
  $ /usr/bin/time -f "\t%M Max Resident Set Size (Kb)"  ./ffprobe /tmp/poc 2>&1 | tail -n 1
  	1046812 Max Resident Set Size (Kb)
---
 libavcodec/h2645_parse.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavcodec/h2645_parse.c b/libavcodec/h2645_parse.c
index b0d9ff66f0..e77689f347 100644
--- a/libavcodec/h2645_parse.c
+++ b/libavcodec/h2645_parse.c
@@ -32,7 +32,7 @@
 int ff_h2645_extract_rbsp(const uint8_t *src, int length,
                           H2645NAL *nal, int small_padding)
 {
-    int i, si, di;
+    int i, si, di, nsc;
     uint8_t *dst;
     int64_t padding = small_padding ? 0 : MAX_MBPAIR_SIZE;
 
@@ -91,8 +91,17 @@ int ff_h2645_extract_rbsp(const uint8_t *src, int length,
     } else if (i > length)
         i = length;
 
+    // Find next NAL start code, if present, to reduce rbsp_buffer size when
+    // multiple NALUs.
+    for (nsc = i; nsc + 2 < length; nsc++) {
+        if (src[nsc] == 0 && src[nsc + 1] == 0 && src[nsc + 2] == 1)
+          break;
+    }
+    if (nsc + 2 == length)
+        nsc = length;
+
     av_fast_padded_malloc(&nal->rbsp_buffer, &nal->rbsp_buffer_size,
-                          length + padding);
+                          nsc + padding);
     if (!nal->rbsp_buffer)
         return AVERROR(ENOMEM);
 
-- 
2.15.0.rc1.287.g2b38de12cc-goog