[FFmpeg-cvslog] avutil/avframe: fix channel layout checks in av_frame_copy()

[James Almer]

ffmpeg | branch: master | James Almer <jamrial at gmail.com> | Wed May 25 12:27:59 2022 -0300| [5929ea6d4b1d979a5d888c3692fe24957a57e9c4] | committer: James Almer

avutil/avframe: fix channel layout checks in av_frame_copy()

Normally, both the source and dest frame would have only the old API fields
set, only the new API fields set, or both set. But in some cases, like when
calling av_frame_ref() using a non reference counted source frame where only
the old channel layout API fields were populated, the result would be the dst
frame having both the new and old fields populated.

This commit takes this into account and fixes the checks by calling
av_channel_layout_compare() only if the source frame has the new API fields
set, and doing sanity checks for the source frame old API fields if the new
ones are not set.

Signed-off-by: James Almer <jamrial at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=5929ea6d4b1d979a5d888c3692fe24957a57e9c4
---

 libavutil/frame.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libavutil/frame.c b/libavutil/frame.c
index fbb869fffa..4c16488c66 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -727,19 +727,28 @@ static int frame_copy_audio(AVFrame *dst, const AVFrame *src)
 
 #if FF_API_OLD_CHANNEL_LAYOUT
 FF_DISABLE_DEPRECATION_WARNINGS
-    if (!channels) {
+    if (!channels || !src->ch_layout.nb_channels) {
         if (dst->channels       != src->channels ||
             dst->channel_layout != src->channel_layout)
             return AVERROR(EINVAL);
+        CHECK_CHANNELS_CONSISTENCY(src);
+    }
+    if (!channels) {
         channels = dst->channels;
         planes = planar ? channels : 1;
-        CHECK_CHANNELS_CONSISTENCY(src);
     }
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
 
     if (dst->nb_samples != src->nb_samples ||
+#if FF_API_OLD_CHANNEL_LAYOUT
+        (av_channel_layout_check(&dst->ch_layout) &&
+         av_channel_layout_check(&src->ch_layout) &&
+#endif
         av_channel_layout_compare(&dst->ch_layout, &src->ch_layout))
+#if FF_API_OLD_CHANNEL_LAYOUT
+        )
+#endif
         return AVERROR(EINVAL);
 
     for (i = 0; i < planes; i++)
@@ -763,7 +772,7 @@ FF_DISABLE_DEPRECATION_WARNINGS
     else if (dst->nb_samples > 0 &&
              (av_channel_layout_check(&dst->ch_layout)
 #if FF_API_OLD_CHANNEL_LAYOUT
-              || dst->channel_layout || dst->channels
+              || dst->channels > 0
 #endif
             ))
         return frame_copy_audio(dst, src);