[FFmpeg-devel] [PATCH 2/2] ffmpeg: Require output format when using cuvid hwaccel

[Philip Langdale]

This is another approach to dealing with the behavioural change
when switching to generic hwaccel. Full hardware transcoding will
not happen if no output format is specified, where this was not
true previously.

So, let's force the user to deal with this change by refusing to
run without an output format.

Signed-off-by: Philip Langdale <philipl at overt.org>
---
 ffmpeg.h     |  1 +
 ffmpeg_opt.c | 29 ++++++++++++++++++++++-------
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/ffmpeg.h b/ffmpeg.h
index fa81427471..1a0aec5862 100644
--- a/ffmpeg.h
+++ b/ffmpeg.h
@@ -76,6 +76,7 @@ typedef struct HWAccel {
     enum HWAccelID id;
     enum AVPixelFormat pix_fmt;
     enum AVHWDeviceType device_type;
+    int require_output_format;
 } HWAccel;
 
 typedef struct HWDevice {
diff --git a/ffmpeg_opt.c b/ffmpeg_opt.c
index 6dc4ad43d2..abdd8bae58 100644
--- a/ffmpeg_opt.c
+++ b/ffmpeg_opt.c
@@ -68,31 +68,38 @@
 const HWAccel hwaccels[] = {
 #if HAVE_VDPAU_X11
     { "vdpau", hwaccel_decode_init, HWACCEL_VDPAU, AV_PIX_FMT_VDPAU,
-      AV_HWDEVICE_TYPE_VDPAU },
+      AV_HWDEVICE_TYPE_VDPAU,
+      .require_output_format = 0 },
 #endif
 #if HAVE_DXVA2_LIB
     { "dxva2", dxva2_init, HWACCEL_DXVA2, AV_PIX_FMT_DXVA2_VLD,
-      AV_HWDEVICE_TYPE_NONE },
+      AV_HWDEVICE_TYPE_NONE,
+      .require_output_format = 0 },
 #endif
 #if CONFIG_VDA
     { "vda",   videotoolbox_init,   HWACCEL_VDA,   AV_PIX_FMT_VDA,
-      AV_HWDEVICE_TYPE_NONE },
+      AV_HWDEVICE_TYPE_NONE,
+      .require_output_format = 0 },
 #endif
 #if CONFIG_VIDEOTOOLBOX
     { "videotoolbox",   videotoolbox_init,   HWACCEL_VIDEOTOOLBOX,   AV_PIX_FMT_VIDEOTOOLBOX,
-      AV_HWDEVICE_TYPE_NONE },
+      AV_HWDEVICE_TYPE_NONE,
+      .require_output_format = 0 },
 #endif
 #if CONFIG_LIBMFX
     { "qsv",   qsv_init,   HWACCEL_QSV,   AV_PIX_FMT_QSV,
-      AV_HWDEVICE_TYPE_NONE },
+      AV_HWDEVICE_TYPE_NONE,
+      .require_output_format = 0 },
 #endif
 #if CONFIG_VAAPI
     { "vaapi", hwaccel_decode_init, HWACCEL_VAAPI, AV_PIX_FMT_VAAPI,
-      AV_HWDEVICE_TYPE_VAAPI },
+      AV_HWDEVICE_TYPE_VAAPI,
+      .require_output_format = 0 },
 #endif
 #if CONFIG_CUVID
     { "cuvid", hwaccel_decode_init, HWACCEL_CUVID, AV_PIX_FMT_CUDA,
-      AV_HWDEVICE_TYPE_CUDA },
+      AV_HWDEVICE_TYPE_CUDA,
+      .require_output_format = 1 },
 #endif
     { 0 },
 };
@@ -708,6 +715,7 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
         char *discard_str = NULL;
         const AVClass *cc = avcodec_get_class();
         const AVOption *discard_opt = av_opt_find(&cc, "skip_frame", NULL, 0, 0);
+        int require_output_format;
 
         if (!ist)
             exit_program(1);
@@ -805,6 +813,7 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
                     for (i = 0; hwaccels[i].name; i++) {
                         if (!strcmp(hwaccels[i].name, hwaccel)) {
                             ist->hwaccel_id = hwaccels[i].id;
+                            require_output_format = hwaccels[i].require_output_format;
                             break;
                         }
                     }
@@ -836,6 +845,12 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
                     av_log(NULL, AV_LOG_FATAL, "Unrecognised hwaccel output "
                            "format: %s", hwaccel_output_format);
                 }
+            } else if (require_output_format) {
+                av_log(NULL, AV_LOG_FATAL, "No hwaccel output format specified. "
+                       "Specify one using -hwaccel_output_format. To enable full "
+                       "hardware transcoding with cuvid/nvenc, use the 'cuda' "
+                       "format.\n");
+                exit_program(1);
             } else {
                 ist->hwaccel_output_format = AV_PIX_FMT_NONE;
             }
-- 
2.11.0