[FFmpeg-cvslog] libavcodec/dcadec: adjust the `ch_layout` when downmix is active

[Geoffrey McRae]

ffmpeg | branch: master | Geoffrey McRae <geoff at hostfission.com> | Sun Nov 26 15:33:22 2023 +1100| [a8677bcc8f2f9561be070cb9412c5a2be9e5be7f] | committer: James Almer

libavcodec/dcadec: adjust the `ch_layout` when downmix is active

Applications making use of this codec with the `downmix` option are
segfaulting unless the `ch_layout` is overridden after `avcodec_open2`
as can be seen in projects like MythTV[1]

This patch fixes this by overriding the ch_layout as done in other
decoders such as AC3.

1: https://github.com/MythTV/mythtv/blob/af6f362a140cd59b9ed784a8c639fd456b5f6967/mythtv/libs/libmythtv/decoders/avformatdecoder.cpp#L4607

Signed-off-by: Geoffrey McRae <geoff at hostfission.com>
Signed-off-by: James Almer <jamrial at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a8677bcc8f2f9561be070cb9412c5a2be9e5be7f
---

 libavcodec/dcadec.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libavcodec/dcadec.c b/libavcodec/dcadec.c
index 1fee49cf4d..cc23731b90 100644
--- a/libavcodec/dcadec.c
+++ b/libavcodec/dcadec.c
@@ -366,12 +366,19 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
     if (s->downmix_layout.nb_channels) {
         if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO) ||
-            !av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX))
+            !av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX)) {
             s->request_channel_layout = DCA_SPEAKER_LAYOUT_STEREO;
-        else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0))
+            av_channel_layout_uninit(&avctx->ch_layout);
+            avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO;
+        } else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0)) {
             s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT0;
-        else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1))
+            av_channel_layout_uninit(&avctx->ch_layout);
+            avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0;
+        } else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1)) {
             s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT1;
+            av_channel_layout_uninit(&avctx->ch_layout);
+            avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1;
+        }
         else
             av_log(avctx, AV_LOG_WARNING, "Invalid downmix layout\n");
     }