[FFmpeg-cvslog] avformat/asfdec_o: limit recursion depth in asf_read_unknown()

[Michael Niedermayer]

ffmpeg | branch: release/3.2 | Michael Niedermayer <michael at niedermayer.cc> | Wed Aug 31 01:21:38 2022 +0200| [9bf2cb6d850c43b1294648eceac68783f79d1798] | committer: Michael Niedermayer

avformat/asfdec_o: limit recursion depth in asf_read_unknown()

The threshold of 5 is arbitrary, both smaller and larger should work fine

Fixes: Stack overflow
Fixes: 50603/clusterfuzz-testcase-minimized-ffmpeg_dem_ASF_O_fuzzer-6049302564175872

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 1f1a368169ef9d945dc4b4764f5c60ba9bbc9134)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=9bf2cb6d850c43b1294648eceac68783f79d1798
---

 libavformat/asfdec_o.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libavformat/asfdec_o.c b/libavformat/asfdec_o.c
index ac95a8327d..6260308b8a 100644
--- a/libavformat/asfdec_o.c
+++ b/libavformat/asfdec_o.c
@@ -113,6 +113,7 @@ typedef struct ASFContext {
     int64_t data_offset;
     int64_t first_packet_offset; // packet offset
     int64_t unknown_offset;   // for top level header objects or subobjects without specified behavior
+    int in_asf_read_unknown;
 
     // ASF file must not contain more than 128 streams according to the specification
     ASFStream *asf_st[ASF_MAX_STREAMS];
@@ -177,7 +178,7 @@ static int asf_read_unknown(AVFormatContext *s, const GUIDParseTable *g)
     uint64_t size   = avio_rl64(pb);
     int ret;
 
-    if (size > INT64_MAX)
+    if (size > INT64_MAX || asf->in_asf_read_unknown > 5)
         return AVERROR_INVALIDDATA;
 
     if (asf->is_header)
@@ -186,8 +187,11 @@ static int asf_read_unknown(AVFormatContext *s, const GUIDParseTable *g)
     if (!g->is_subobject) {
         if (!(ret = strcmp(g->name, "Header Extension")))
             avio_skip(pb, 22); // skip reserved fields and Data Size
-        if ((ret = detect_unknown_subobject(s, asf->unknown_offset,
-                                            asf->unknown_size)) < 0)
+        asf->in_asf_read_unknown ++;
+        ret = detect_unknown_subobject(s, asf->unknown_offset,
+                                            asf->unknown_size);
+        asf->in_asf_read_unknown --;
+        if (ret < 0)
             return ret;
     } else {
         if (size < 24) {