[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle video parameters

[pkoshevoy at gmail.com]

From: Pavel Koshevoy <pkoshevoy at gmail.com>

Evidently CUVID doesn't support decoding 422 or 444 chroma formats,
and only a limited set of resolutions per codec are supported.

Unfortunately CUVID silently drops packets for video of unsupported
resolution.  However, it will error-out at cuvidCreateDecoder call
if the indicated video resolution is not supported.

Given that stream resolution and pixel format are typically known as
a result of probing it is better to use this information during
avcodec_open2 call to fail immediately, rather than proceeding to
decode and never receiving any frames from the decoder nor receiving
any indication of decode failure.
---
 libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 5 deletions(-)

diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
index 8fc713d..f9c29a1 100644
--- a/libavcodec/cuvid.c
+++ b/libavcodec/cuvid.c
@@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
     return 0;
 }
 
-static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
+static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
+                                    const CUVIDPARSERPARAMS *cuparseinfo,
+                                    cudaVideoChromaFormat probed_chroma_format,
+                                    int probed_width,
+                                    int probed_height)
 {
     CuvidContext *ctx = avctx->priv_data;
     CUVIDDECODECREATEINFO cuinfo;
@@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
     memset(&cuinfo, 0, sizeof(cuinfo));
 
     cuinfo.CodecType = cuparseinfo->CodecType;
-    cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
+    cuinfo.ChromaFormat = probed_chroma_format;
     cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
 
-    cuinfo.ulWidth = 1280;
-    cuinfo.ulHeight = 720;
+    cuinfo.ulWidth = probed_width;
+    cuinfo.ulHeight = probed_height;
     cuinfo.ulTargetWidth = cuinfo.ulWidth;
     cuinfo.ulTargetHeight = cuinfo.ulHeight;
 
@@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
     return 0;
 }
 
+static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt,
+                                               cudaVideoChromaFormat *out)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
+    if (!(out && desc &&
+          (desc->nb_components == 1 || desc->nb_components == 3) &&
+          (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2)))
+    {
+        return AVERROR(EINVAL);
+    }
+
+    if (desc->nb_components == 1)
+    {
+        *out = cudaVideoChromaFormat_Monochrome;
+    }
+    else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR)
+    {
+        *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 :
+                (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 :
+                cudaVideoChromaFormat_420);
+    }
+    else
+    {
+        return AVERROR(EINVAL);
+    }
+
+    // unfortunately, 420 is the only one that works:
+    return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL;
+}
+
 static av_cold int cuvid_decode_init(AVCodecContext *avctx)
 {
     CuvidContext *ctx = avctx->priv_data;
@@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
     CUcontext cuda_ctx = NULL;
     CUcontext dummy;
     const AVBitStreamFilter *bsf;
+    cudaVideoChromaFormat probed_chroma_format;
+    int probed_width;
+    int probed_height;
     int ret = 0;
 
     enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
                                        AV_PIX_FMT_NV12,
                                        AV_PIX_FMT_NONE };
 
+    enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt <= 0 ?
+                                         AV_PIX_FMT_YUV420P :
+                                         avctx->pix_fmt);
+
+    ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format);
+    if (ret < 0) {
+        // pixel format is not supported:
+        return ret;
+    }
+    probed_width = avctx->coded_width ? avctx->coded_width : 1280;
+    probed_height = avctx->coded_height ? avctx->coded_height : 720;
+
     // Accelerated transcoding scenarios with 'ffmpeg' require that the
     // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
     // pix_fmt for non-accelerated transcoding, do not need to be correct
@@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
     if (ret < 0)
         goto error;
 
-    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
+    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
+                                   probed_chroma_format,
+                                   probed_width,
+                                   probed_height);
     if (ret < 0)
         goto error;
 
-- 
2.9.2