[FFmpeg-cvslog] avcodec/mpeg4videodec: Remove use of FF_PROFILE_MPEG4_SIMPLE_STUDIO as indicator of studio profile

[Michael Niedermayer]

ffmpeg | branch: release/4.0 | Michael Niedermayer <michael at niedermayer.cc> | Tue Jul  3 00:27:04 2018 +0200| [5fd1dce39a70340b9fd508154e48985902602e25] | committer: Michael Niedermayer

avcodec/mpeg4videodec: Remove use of FF_PROFILE_MPEG4_SIMPLE_STUDIO as indicator of studio profile

The profile field is changed by code inside and outside the decoder,
its not a reliable indicator of the internal codec state.
Maintaining it consistency with studio_profile is messy.
Its easier to just avoid it and use only studio_profile

Fixes: assertion failure
Fixes: ffmpeg_crash_9.avi

Found-by: Thuan Pham, Marcel Böhme, Andrew Santosa and Alexandru Razvan Caciulescu with AFLSmart
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit bd27a9364ca274ca97f1df6d984e88a0700fb235)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=5fd1dce39a70340b9fd508154e48985902602e25
---

 libavcodec/error_resilience.c | 3 +--
 libavcodec/h263dec.c          | 6 ++++--
 libavcodec/mpeg4videodec.c    | 1 -
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libavcodec/error_resilience.c b/libavcodec/error_resilience.c
index 339042e206..1abae53f41 100644
--- a/libavcodec/error_resilience.c
+++ b/libavcodec/error_resilience.c
@@ -814,8 +814,7 @@ static int er_supported(ERContext *s)
 {
     if(s->avctx->hwaccel && s->avctx->hwaccel->decode_slice           ||
        !s->cur_pic.f                                                  ||
-       s->cur_pic.field_picture                                       ||
-       s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO
+       s->cur_pic.field_picture
     )
         return 0;
     return 1;
diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
index 5b6f834d5a..a2326ac1ab 100644
--- a/libavcodec/h263dec.c
+++ b/libavcodec/h263dec.c
@@ -47,9 +47,10 @@
 
 static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
 {
+    MpegEncContext *s = avctx->priv_data;
     /* MPEG-4 Studio Profile only, not supported by hardware */
     if (avctx->bits_per_raw_sample > 8) {
-        av_assert1(avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
+        av_assert1(s->studio_profile);
         return avctx->pix_fmt;
     }
 
@@ -669,7 +670,8 @@ retry:
 
     av_assert1(s->bitstream_buffer_size == 0);
 frame_end:
-    ff_er_frame_end(&s->er);
+    if (!s->studio_profile)
+        ff_er_frame_end(&s->er);
 
     if (avctx->hwaccel) {
         ret = avctx->hwaccel->end_frame(avctx);
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 54a8496244..2df525e03a 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -3244,7 +3244,6 @@ end:
     s->avctx->has_b_frames = !s->low_delay;
 
     if (s->studio_profile) {
-        av_assert0(s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
         if (!s->avctx->bits_per_raw_sample) {
             av_log(s->avctx, AV_LOG_ERROR, "Missing VOL header\n");
             return AVERROR_INVALIDDATA;