[FFmpeg-devel] [PATCH] libavformat/mpegtsenc.c -- correctly re-emit extradata ahead of IDR pictures

[John Coiner]

Trying again to upload the same patch, this time inline:

>From 09c4bbd3d4ef6fbcb1558fce5cff4d15d7839526 Mon Sep 17 00:00:00 2001
From: John Coiner <jcoiner at google.com>
Date: Thu, 26 Jan 2023 13:34:24 -0500
Subject: [PATCH] Proposed fix for https://trac.ffmpeg.org/ticket/10148

---
 libavformat/mpegtsenc.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/libavformat/mpegtsenc.c b/libavformat/mpegtsenc.c
index 48d39e6a7d..6cb398f99c 100644
--- a/libavformat/mpegtsenc.c
+++ b/libavformat/mpegtsenc.c
@@ -1886,17 +1886,21 @@ static int
mpegts_write_packet_internal(AVFormatContext *s, AVPacket *pkt)
         if (extradd && AV_RB24(st->codecpar->extradata) > 1)
             extradd = 0;

-        do {
+        while (p < buf_end
+               && extradd > 0
+               && (state & 0x1f) != 5  // IDR picture
+               && (state & 0x1f) != 1  // non-IDR picture
+               ) {
             p = avpriv_find_start_code(p, buf_end, &state);
             av_log(s, AV_LOG_TRACE, "nal %"PRId32"\n", state & 0x1f);
-            if ((state & 0x1f) == 7)
+            if ((state & 0x1f) == 7)  // SPS NAL
                 extradd = 0;
-        } while (p < buf_end && (state & 0x1f) != 9 &&
-                 (state & 0x1f) != 5 && (state & 0x1f) != 1);
-
-        if ((state & 0x1f) != 5)
+        }
+        if ((state & 0x1f) != 5) {
+            // Not an IDR picture
             extradd = 0;
-        if ((state & 0x1f) != 9) { // AUD NAL
+        }
+        if (extradd > 0) {
             data = av_malloc(pkt->size + 6 + extradd);
             if (!data)
                 return AVERROR(ENOMEM);
-- 
2.39.1.456.gfc5497dd1b-goog