[FFmpeg-cvslog] avcodec/bsf/h264_mp4toannexb: Fix mixed bitstream format

[Zhao Zhili]

ffmpeg | branch: release/7.1 | Zhao Zhili <zhilizhao at tencent.com> | Fri Apr 25 01:08:36 2025 +0800| [94784d0918932a25a8f013f29e6c862b61ecfa45] | committer: Zhao Zhili

avcodec/bsf/h264_mp4toannexb: Fix mixed bitstream format

This bsf converts AV_PKT_DATA_NEW_EXTRADATA side data in avcc format
to in-band annexb format. However, the side data wasn't been removed
and copied from input packet to output packet. So the output packet
has mixed bitstream format. We don't support mixed bitstream format.
For example, h264_metadata report error in the following case:

ffmpeg -i foo.flv \
  -bsf:v "h264_mp4toannexb,h264_metadata" \
  -c copy -f null

This patch removed NEW_EXTRADATA side data after process.

This patch also add a check so only NEW_EXTRADATA in avcc format is
processed. NEW_EXTRADATA in annexb format is copied to output as is.

Reported-by: jiangjie <jiangjie618 at gmail.com>
Signed-off-by: Zhao Zhili <zhilizhao at tencent.com>
(cherry picked from commit 25812d303398552a7768dac04f2992bcb09ea000)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=94784d0918932a25a8f013f29e6c862b61ecfa45
---

 libavcodec/bsf/h264_mp4toannexb.c | 22 ++++++++++++----------
 tests/fate/h264.mak               |  4 ++--
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/libavcodec/bsf/h264_mp4toannexb.c b/libavcodec/bsf/h264_mp4toannexb.c
index dda064287e..36b09b4dd6 100644
--- a/libavcodec/bsf/h264_mp4toannexb.c
+++ b/libavcodec/bsf/h264_mp4toannexb.c
@@ -93,6 +93,11 @@ static int h264_extradata_to_annexb(AVBSFContext *ctx,
     const int padding                   = AV_INPUT_BUFFER_PADDING_SIZE;
     int length_size, pps_offset = 0;
 
+    if (extradata_size < 7) {
+        av_log(ctx, AV_LOG_ERROR, "Invalid extradata size: %d\n", extradata_size);
+        return AVERROR_INVALIDDATA;
+    }
+
     bytestream2_init(gb, extradata, extradata_size);
 
     bytestream2_skipu(gb, 4);
@@ -262,16 +267,11 @@ static int h264_mp4toannexb_init(AVBSFContext *ctx)
         (extra_size >= 4 && AV_RB32(ctx->par_in->extradata) == 1)) {
         av_log(ctx, AV_LOG_VERBOSE,
                "The input looks like it is Annex B already\n");
-    } else if (extra_size >= 7) {
-        return h264_extradata_to_annexb(ctx,
-                                        ctx->par_in->extradata,
-                                        ctx->par_in->extradata_size);
-    } else {
-        av_log(ctx, AV_LOG_ERROR, "Invalid extradata size: %d\n", extra_size);
-        return AVERROR_INVALIDDATA;
+        return 0;
     }
-
-    return 0;
+    return h264_extradata_to_annexb(ctx,
+                                    ctx->par_in->extradata,
+                                    ctx->par_in->extradata_size);
 }
 
 static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
@@ -293,10 +293,12 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
 
     extradata = av_packet_get_side_data(in, AV_PKT_DATA_NEW_EXTRADATA,
                                         &extradata_size);
-    if (extradata) {
+    if (extradata && extradata[0] == 1) {
         ret = h264_extradata_to_annexb(ctx, extradata, extradata_size);
         if (ret < 0)
             goto fail;
+        av_packet_side_data_remove(in->side_data, &in->side_data_elems,
+                                   AV_PKT_DATA_NEW_EXTRADATA);
     }
 
     /* nothing to filter */
diff --git a/tests/fate/h264.mak b/tests/fate/h264.mak
index e7038387ea..d93751f093 100644
--- a/tests/fate/h264.mak
+++ b/tests/fate/h264.mak
@@ -224,7 +224,7 @@ FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-unescaped-extradata
 # this sample contains field-coded frames, with both fields in a single packet
 FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-twofields-packet
 
-FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF SCALE_FILTER) += fate-h264-bsf-mp4toannexb-new-extradata
+FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF H264_METADATA_BSF SCALE_FILTER) += fate-h264-bsf-mp4toannexb-new-extradata
 
 FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF) += fate-h264-bsf-mp4toannexb \
                                                              fate-h264-bsf-mp4toannexb-2 \
@@ -438,7 +438,7 @@ fate-h264-bsf-mp4toannexb:                        CMD = md5 -i $(TARGET_SAMPLES)
 fate-h264-bsf-mp4toannexb-2:                      CMD = md5 -i $(TARGET_SAMPLES)/h264/ps_prefix_first_idr.mp4 -c:v copy -f h264
 fate-h264-bsf-mp4toannexb-2:                      CMP = oneline
 fate-h264-bsf-mp4toannexb-2:                      REF = cffcfa6a2d0b58c9de1f5785f099f41d
-fate-h264-bsf-mp4toannexb-new-extradata:          CMD = stream_remux mov $(TARGET_SAMPLES)/h264/extradata-reload-multi-stsd.mov "" h264 "-map 0:v"
+fate-h264-bsf-mp4toannexb-new-extradata:          CMD = stream_remux mov $(TARGET_SAMPLES)/h264/extradata-reload-multi-stsd.mov "" h264 "-bsf h264_mp4toannexb,h264_metadata -map 0:v"
 fate-h264-bsf-dts2pts:                            CMD = transcode "h264" $(TARGET_SAMPLES)/h264-conformance/CAPAMA3_Sand_F.264 \
                                                         mov "-c:v copy -bsf:v dts2pts -frames:v 50" "-c:v copy"
 fate-h264_mp4toannexb_ticket5927:                 CMD = transcode "mp4" $(TARGET_SAMPLES)/h264/thezerotheorem-cut.mp4 \