[FFmpeg-devel] [PATCH 4/6] lavf: add automatic bitstream filtering

[Rodger Combs]

This solves the problem discussed in https://ffmpeg.org/pipermail/ffmpeg-devel/2015-September/179238.html
by allowing AVCodec::write_header to be delayed until after packets have been
run through required bitstream filters in order to generate global extradata.

It also provides a mechanism by which a muxer can add a bitstream filter to a
stream automatically, rather than prompting the user to do so.
---
 libavformat/avformat.h | 37 +++++++++++++++++++++++++++++++++++++
 libavformat/mux.c      | 47 +++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index d94191e..bb3a674 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -600,6 +600,29 @@ typedef struct AVOutputFormat {
      */
     int (*free_device_capabilities)(struct AVFormatContext *s, struct AVDeviceCapabilitiesQuery *caps);
     enum AVCodecID data_codec; /**< default data codec */
+    /**
+     * Initialize format. May allocate data here, and set any AVFormatContext or
+     * AVStream parameters that need to be set before packets are sent.
+     * This method must not write output.
+     *
+     * Any allocations made here must be freed in deinit().
+     */
+    int (*init)(struct AVFormatContext *);
+    /**
+     * Deinitialize format. If present, this is called whenever the muxer is being
+     * destroyed, regardless of whether or not the header has been written.
+     *
+     * If a trailer is being written, this is called after write_trailer().
+     *
+     * This is called if init() fails as well.
+     */
+    void (*deinit)(struct AVFormatContext *);
+    /**
+     * Set up any necessary bitstream filtering and extract any extra data needed
+     * for the global header.
+     * Return 0 if more packets from this stream must be checked; 1 if not.
+     */
+    int (*check_bitstream)(struct AVFormatContext *, const AVPacket *pkt);
 } AVOutputFormat;
 /**
  * @}
@@ -1177,6 +1200,18 @@ typedef struct AVStream {
      * Must not be accessed in any way by callers.
      */
     AVStreamInternal *internal;
+
+    /**
+     * bitstream filter to run on stream
+     * - encoding: Set by muxer or user using av_stream_add_bitstream_filter
+     * - decoding: unused
+     */
+    AVBitStreamFilterContext *bsfc;
+
+    /**
+     * internal check if check_bitstream should still be run on each packet
+     */
+    int bitstream_checked;
 } AVStream;
 
 AVRational av_stream_get_r_frame_rate(const AVStream *s);
@@ -1792,6 +1827,8 @@ typedef struct AVFormatContext {
      * Demuxing: Set by user.
      */
     int (*open_cb)(struct AVFormatContext *s, AVIOContext **p, const char *url, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options);
+
+    int header_written;
 } AVFormatContext;
 
 int av_format_get_probe_score(const AVFormatContext *s);
diff --git a/libavformat/mux.c b/libavformat/mux.c
index 05d4170..56ada97 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -405,6 +405,11 @@ FF_ENABLE_DEPRECATION_WARNINGS
          *options = tmp;
     }
 
+    if (s->oformat->init && (ret = s->oformat->init(s)) < 0) {
+        s->oformat->deinit(s);
+        goto fail;
+    }
+
     return 0;
 
 fail:
@@ -456,7 +461,7 @@ int avformat_write_header(AVFormatContext *s, AVDictionary **options)
     if ((ret = init_muxer(s, options)) < 0)
         return ret;
 
-    if (s->oformat->write_header) {
+    if (s->oformat->write_header && !s->oformat->check_bitstream) {
         ret = s->oformat->write_header(s);
         if (ret >= 0 && s->pb && s->pb->error < 0)
             ret = s->pb->error;
@@ -464,6 +469,7 @@ int avformat_write_header(AVFormatContext *s, AVDictionary **options)
             return ret;
         if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
             avio_flush(s->pb);
+        s->header_written = 1;
     }
 
     if ((ret = init_pts(s)) < 0)
@@ -1021,6 +1027,17 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             ret = AVERROR(EINVAL);
             goto fail;
         }
+
+        if (s->oformat->check_bitstream) {
+            if (!st->bitstream_checked) {
+                if ((ret = s->oformat->check_bitstream(s, pkt)) < 0)
+                    goto fail;
+                else if (ret == 1)
+                    st->bitstream_checked = 1;
+            }
+        }
+
+        av_apply_bitstream_filters(st->codec, pkt, st->bsfc);
     } else {
         av_log(s, AV_LOG_TRACE, "av_interleaved_write_frame FLUSH\n");
         flush = 1;
@@ -1037,10 +1054,22 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
         if (ret <= 0) //FIXME cleanup needed for ret<0 ?
             return ret;
 
+        if (!s->header_written && s->oformat->write_header) {
+            ret = s->oformat->write_header(s);
+            if (ret >= 0 && s->pb && s->pb->error < 0)
+                ret = s->pb->error;
+            if (ret < 0)
+                goto fail2;
+            if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
+                avio_flush(s->pb);
+            s->header_written = 1;
+        }
+
         ret = write_packet(s, &opkt);
         if (ret >= 0)
             s->streams[opkt.stream_index]->nb_frames++;
 
+fail2:
         av_packet_unref(&opkt);
 
         if (ret < 0)
@@ -1077,14 +1106,28 @@ int av_write_trailer(AVFormatContext *s)
             goto fail;
     }
 
+    if (!s->header_written && s->oformat->write_header) {
+        ret = s->oformat->write_header(s);
+        if (ret >= 0 && s->pb && s->pb->error < 0)
+            ret = s->pb->error;
+        if (ret < 0)
+            goto fail;
+        if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
+            avio_flush(s->pb);
+        s->header_written = 1;
+    }
+
 fail:
-    if (s->oformat->write_trailer)
+    if (s->header_written && s->oformat->write_trailer)
         if (ret >= 0) {
         ret = s->oformat->write_trailer(s);
         } else {
             s->oformat->write_trailer(s);
         }
 
+    if (s->oformat->deinit)
+        s->oformat->deinit(s);
+
     if (s->pb)
        avio_flush(s->pb);
     if (ret == 0)
-- 
2.6.3