[FFmpeg-devel] [RFC] get rid of h264_vdpau codec

[Reimar Döffinger]

Hello,
Gwenole Beauchesne made me realize that the current way of selecting
hardware acceleration sucks, and that there is a better method already
available.
The idea is that, since the difference is basically having a different
PIX_FMT, we can simply let the AVCodecContext::get_format function choose.
Of course that means also you can't use CODEC_CAP_HWACCEL_VDPAU etc.
(and I probably missed a few places where this must be changed),
but I always considered that a rather idiotic approach anyway.
I have not actually tested how MPlayer would need to be changed to work
with that (and thus don't know for sure how well it would work), but one
advantage would certainly be, that the user no longer would have to
explicitly select a different codec but simply could do e.g. -vo vdpau
and get the best thing available.
What are your opinions?

Greetings,
Reimar D?ffinger
-------------- next part --------------
Index: libavcodec/h264.c
===================================================================
--- libavcodec/h264.c	(revision 17187)
+++ libavcodec/h264.c	(working copy)
@@ -2172,6 +2172,7 @@
 }
 
 static av_cold int decode_init(AVCodecContext *avctx){
+    const enum PixelFormat H264_pixfmts[] = {PIX_FMT_YUV420P, PIX_FMT_VDPAU_H264, PIX_FMT_NONE};
     H264Context *h= avctx->priv_data;
     MpegEncContext * const s = &h->s;
 
@@ -2190,10 +2191,8 @@
 
     if(avctx->codec_id == CODEC_ID_SVQ3)
         avctx->pix_fmt= PIX_FMT_YUVJ420P;
-    else if(s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU)
-        avctx->pix_fmt= PIX_FMT_VDPAU_H264;
     else
-        avctx->pix_fmt= PIX_FMT_YUV420P;
+        avctx->pix_fmt= avctx->get_format(avctx, H264_pixfmts);
 
     decode_init_vlc();
 
@@ -7322,7 +7321,7 @@
     H264Context *hx;
     int i;
 
-    if(s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU)
+    if(s->avctx->pix_fmt == PIX_FMT_VDPAU_H264)
         return;
     if(context_count == 1) {
         decode_slice(avctx, &h);
@@ -7454,7 +7453,7 @@
                && (avctx->skip_frame < AVDISCARD_BIDIR  || hx->slice_type_nos!=FF_B_TYPE)
                && (avctx->skip_frame < AVDISCARD_NONKEY || hx->slice_type_nos==FF_I_TYPE)
                && avctx->skip_frame < AVDISCARD_ALL){
-                if(CONFIG_H264_VDPAU_DECODER && s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU){
+                if(CONFIG_H264_VDPAU_DECODER && s->avctx->pix_fmt == PIX_FMT_VDPAU_H264){
                     static const uint8_t start_code[] = {0x00, 0x00, 0x01};
                     ff_vdpau_add_data_chunk(s, start_code, sizeof(start_code));
                     ff_vdpau_add_data_chunk(s, &buf[buf_index - consumed], consumed );
@@ -7655,7 +7654,7 @@
         s->current_picture_ptr->qscale_type= FF_QSCALE_TYPE_H264;
         s->current_picture_ptr->pict_type= s->pict_type;
 
-        if (CONFIG_H264_VDPAU_DECODER && s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU)
+        if (CONFIG_H264_VDPAU_DECODER && s->avctx->pix_fmt == PIX_FMT_VDPAU_H264)
             ff_vdpau_h264_set_reference_frames(s);
 
         if(!s->dropable) {
@@ -7666,7 +7665,7 @@
         h->prev_frame_num_offset= h->frame_num_offset;
         h->prev_frame_num= h->frame_num;
 
-        if (CONFIG_H264_VDPAU_DECODER && s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU)
+        if (CONFIG_H264_VDPAU_DECODER && s->avctx->pix_fmt == PIX_FMT_VDPAU_H264)
             ff_vdpau_h264_picture_complete(s);
 
         /*
@@ -8055,22 +8054,6 @@
     .long_name = NULL_IF_CONFIG_SMALL("H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10"),
 };
 
-#if CONFIG_H264_VDPAU_DECODER
-AVCodec h264_vdpau_decoder = {
-    "h264_vdpau",
-    CODEC_TYPE_VIDEO,
-    CODEC_ID_H264,
-    sizeof(H264Context),
-    decode_init,
-    NULL,
-    decode_end,
-    decode_frame,
-    CODEC_CAP_DR1 | CODEC_CAP_DELAY | CODEC_CAP_HWACCEL_VDPAU,
-    .flush= flush_dpb,
-    .long_name = NULL_IF_CONFIG_SMALL("H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 (VDPAU acceleration)"),
-};
-#endif
-
 #if CONFIG_SVQ3_DECODER
 #include "svq3.c"
 #endif