[FFmpeg-devel] [PATCH 2/2] mmaldec: restore extradata destroyed by bitstream filter

[wm4]

This matters because avformat_find_stream_info() can open this decoder
under certain circumstances. It will not create a copy of the context,
and all changes done to AVCodecContext must be undone on closing.
---
Shitty avformat_find_stream_info() implementation meats shitty BSF API,
wasting developer time.
---
 libavcodec/mmaldec.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index df1f1ba..57f6765 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -23,7 +23,7 @@
  * @file
  * MMAL Video Decoder
  */
-
+#pragma GCC optimize("O0")
 #include <bcm_host.h>
 #include <interface/mmal/mmal.h>
 #include <interface/mmal/util/mmal_util.h>
@@ -85,6 +85,9 @@ typedef struct MMALDecodeContext {
     int64_t frames_output;
     int eos_received;
     int eos_sent;
+
+    uint8_t *orig_extradata;
+    int orig_extradata_size;
 } MMALDecodeContext;
 
 // Assume decoder is guaranteed to produce output after at least this many
@@ -183,8 +186,18 @@ static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
     if (ctx->bsfc)
         av_bitstream_filter_close(ctx->bsfc);
 
-    mmal_vc_deinit();
+    /*
+     * Restore original extradata, so that if the decoder is
+     * reinitialised, the bitstream detection and filtering
+     * will work as expected.
+     */
+    if (ctx->orig_extradata) {
+        av_free(avctx->extradata);
+        avctx->extradata = ctx->orig_extradata;
+        avctx->extradata_size = ctx->orig_extradata_size;
+    }
 
+    mmal_vc_deinit();
     return 0;
 }
 
@@ -355,6 +368,13 @@ static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
     if (avctx->codec->id == AV_CODEC_ID_H264 && avctx->extradata && avctx->extradata[0] == 1) {
         uint8_t *dummy_p;
         int dummy_int;
+        ctx->orig_extradata = av_mallocz(avctx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE);
+        if (!ctx->orig_extradata) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        ctx->orig_extradata_size = avctx->extradata_size;
+        memcpy(ctx->orig_extradata, avctx->extradata, avctx->extradata_size);
         ctx->bsfc = av_bitstream_filter_init("h264_mp4toannexb");
         if (!ctx->bsfc) {
             av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
-- 
2.5.0