[FFmpeg-devel] [PATCH 2/2] avutil/hwcontext_vaapi: return ENOSYS for unsupported operation

[Philip Langdale]

On Fri, 27 Oct 2023 23:37:01 +0800
Zhao Zhili <quinkblack at foxmail.com> wrote:

> From: Zhao Zhili <zhilizhao at tencent.com>
> 
> av_hwframe_transfer_data try with src_ctx first. If the operation
> failed with AVERROR(ENOSYS), it will try again with dst_ctx. Return
> AVERROR(EINVAL) makes the second step being skipped.
> ---
>  libavutil/hwcontext_vaapi.c | 10 +++++-----
>  1 file changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
> index 558fed94c6..bb28bcf588 100644
> --- a/libavutil/hwcontext_vaapi.c
> +++ b/libavutil/hwcontext_vaapi.c
> @@ -217,7 +217,7 @@ static int
> vaapi_get_image_format(AVHWDeviceContext *hwdev, return 0;
>          }
>      }
> -    return AVERROR(EINVAL);
> +    return AVERROR(ENOSYS);
>  }
>  
>  static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
> @@ -805,19 +805,19 @@ static int vaapi_map_frame(AVHWFramesContext
> *hwfc, 
>      if (!ctx->derive_works && (flags & AV_HWFRAME_MAP_DIRECT)) {
>          // Requested direct mapping but it is not possible.
> -        return AVERROR(EINVAL);
> +        return AVERROR(ENOSYS);
>      }
>      if (dst->format == AV_PIX_FMT_NONE)
>          dst->format = hwfc->sw_format;
>      if (dst->format != hwfc->sw_format && (flags &
> AV_HWFRAME_MAP_DIRECT)) { // Requested direct mapping but the formats
> do not match.
> -        return AVERROR(EINVAL);
> +        return AVERROR(ENOSYS);
>      }
>  
>      err = vaapi_get_image_format(hwfc->device_ctx, dst->format,
> &image_format); if (err < 0) {
>          // Requested format is not a valid output format.
> -        return AVERROR(EINVAL);
> +        return err;
>      }
>  
>      map = av_malloc(sizeof(*map));
> @@ -992,7 +992,7 @@ static int vaapi_map_to_memory(AVHWFramesContext
> *hwfc, AVFrame *dst, if (dst->format != AV_PIX_FMT_NONE) {
>          err = vaapi_get_image_format(hwfc->device_ctx, dst->format,
> NULL); if (err < 0)
> -            return AVERROR(ENOSYS);
> +            return err;
>      }
>  
>      err = vaapi_map_frame(hwfc, dst, src, flags);

LGTM.


--phil