[FFmpeg-cvslog] avcodec/libx265: ignore user set alpha x265-param

[James Almer]

ffmpeg | branch: master | James Almer <jamrial at gmail.com> | Thu Dec 26 13:41:04 2024 -0300| [7e778586e7efceacae33b12b9b3673838060f8f2] | committer: James Almer

avcodec/libx265: ignore user set alpha x265-param

It makes no difference when the input has an alpha plane, and may end up in
crashes or undefined behavior if it doesn't.

Signed-off-by: James Almer <jamrial at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7e778586e7efceacae33b12b9b3673838060f8f2
---

 libavcodec/cbs_h2645.c |  6 +++---
 libavcodec/libx265.c   | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index 369e3ac876..647b1c785e 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -499,9 +499,9 @@ static int cbs_h2645_fragment_add_nals(CodedBitstreamContext *ctx,
         size_t size = nal->size;
         enum AVCodecID codec_id = ctx->codec->codec_id;
 
-        if (codec_id == AV_CODEC_ID_HEVC && nal->nuh_layer_id > 0 &&
-            (nal->type < HEVC_NAL_VPS || nal->type > HEVC_NAL_PPS))
-            continue;
+//        if (codec_id == AV_CODEC_ID_HEVC && nal->nuh_layer_id > 0 &&
+  //          (nal->type < HEVC_NAL_VPS || nal->type > HEVC_NAL_PPS))
+    //        continue;
 
         // Remove trailing zeroes.
         while (size > 0 && nal->data[size - 1] == 0)
diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index aa9b9e4c46..db0f0972df 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -505,8 +505,22 @@ FF_ENABLE_DEPRECATION_WARNINGS
     {
         const AVDictionaryEntry *en = NULL;
         while ((en = av_dict_iterate(ctx->x265_opts, en))) {
-            int parse_ret = ctx->api->param_parse(ctx->params, en->key, en->value);
+            int parse_ret;
 
+            // ignore forced alpha option. The pixel format is all we need.
+            if (!strncmp(en->key, "alpha", 5)) {
+                if (desc->nb_components == 4) {
+                    av_log(avctx, AV_LOG_WARNING,
+                           "Ignoring redundant \"alpha\" option.\n");
+                    continue;
+                }
+                av_log(avctx, AV_LOG_ERROR,
+                       "Alpha encoding was requested through an unsupported "
+                       "option when no alpha plane is present\n");
+                return AVERROR(EINVAL);
+            }
+
+            parse_ret = ctx->api->param_parse(ctx->params, en->key, en->value);
             switch (parse_ret) {
             case X265_PARAM_BAD_NAME:
                 av_log(avctx, AV_LOG_WARNING,