[FFmpeg-cvslog] avutil/hwcontext_vaapi: return ENOSYS for unsupported operation

[Zhao Zhili]

ffmpeg | branch: release/6.1 | Zhao Zhili <zhilizhao at tencent.com> | Sat Oct 28 14:33:31 2023 +0800| [1e84d9c5da685af9ca54ef31a6091f2bff8d646c] | committer: Zhao Zhili

avutil/hwcontext_vaapi: return ENOSYS for unsupported operation

av_hwframe_transfer_data try with src_ctx first. If the operation
failed with AVERROR(ENOSYS), it will try again with dst_ctx. Return
AVERROR(EINVAL) makes the second step being skipped.

Signed-off-by: Zhao Zhili <zhilizhao at tencent.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=1e84d9c5da685af9ca54ef31a6091f2bff8d646c
---

 libavutil/hwcontext_vaapi.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 558fed94c6..12bc95119a 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -217,7 +217,7 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
             return 0;
         }
     }
-    return AVERROR(EINVAL);
+    return AVERROR(ENOSYS);
 }
 
 static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
@@ -817,7 +817,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     err = vaapi_get_image_format(hwfc->device_ctx, dst->format, &image_format);
     if (err < 0) {
         // Requested format is not a valid output format.
-        return AVERROR(EINVAL);
+        return err;
     }
 
     map = av_malloc(sizeof(*map));
@@ -992,7 +992,7 @@ static int vaapi_map_to_memory(AVHWFramesContext *hwfc, AVFrame *dst,
     if (dst->format != AV_PIX_FMT_NONE) {
         err = vaapi_get_image_format(hwfc->device_ctx, dst->format, NULL);
         if (err < 0)
-            return AVERROR(ENOSYS);
+            return err;
     }
 
     err = vaapi_map_frame(hwfc, dst, src, flags);