[MPlayer-dev-eng] [PATCH]Fail early if vdpau decoder creation will certainly fail

[Carl Eugen Hoyos]

Hi!

Attached patch intends to help in following situations:
If user starts more vdpau decoder sessions than supported by hardware.
If user tries to use vdpau decoder for an unsupported codec/unsupported 
size/not even enough video memory for 0 reference frames.

Please comment, Carl Eugen
-------------- next part --------------
Index: libvo/vo_vdpau.c
===================================================================
--- libvo/vo_vdpau.c	(revision 29008)
+++ libvo/vo_vdpau.c	(working copy)
@@ -163,6 +163,7 @@
 static int                                top_field_first;
 
 static VdpDecoder                         decoder;
+static VdpDecoderProfile                  vdp_decoder_profile;
 static int                                decoder_max_refs;
 
 static VdpRect                            src_rect_vid;
@@ -487,6 +488,7 @@
                   uint32_t d_height, uint32_t flags, char *title,
                   uint32_t format)
 {
+    VdpStatus vdp_st;
     XVisualInfo vinfo;
     XSetWindowAttributes xswa;
     XWindowAttributes attribs;
@@ -560,6 +562,30 @@
     vid_surface_num = -1;
     resize();
 
+    if (IMGFMT_IS_VDPAU(image_format)) {
+        switch (image_format) {
+            case IMGFMT_VDPAU_MPEG1:
+                vdp_decoder_profile = VDP_DECODER_PROFILE_MPEG1;
+                break;
+            case IMGFMT_VDPAU_MPEG2:
+                vdp_decoder_profile = VDP_DECODER_PROFILE_MPEG2_MAIN;
+                break;
+            case IMGFMT_VDPAU_H264:
+                vdp_decoder_profile = VDP_DECODER_PROFILE_H264_HIGH;
+                break;
+            case IMGFMT_VDPAU_WMV3:
+                vdp_decoder_profile = VDP_DECODER_PROFILE_VC1_MAIN;
+                break;
+            case IMGFMT_VDPAU_VC1:
+                vdp_decoder_profile = VDP_DECODER_PROFILE_VC1_ADVANCED;
+                break;
+        }
+        decoder_max_refs = 0;
+        vdp_st = vdp_decoder_create(vdp_device, vdp_decoder_profile, vid_width, vid_height,
+                                    decoder_max_refs, &decoder);
+        CHECK_ST_ERROR("Failed creating VDPAU decoder");
+    }
+
     return 0;
 }
 
@@ -792,28 +818,11 @@
     if (!IMGFMT_IS_VDPAU(image_format))
         return VO_FALSE;
     if (decoder == VDP_INVALID_HANDLE || decoder_max_refs < max_refs) {
-        VdpDecoderProfile vdp_decoder_profile;
         if (decoder != VDP_INVALID_HANDLE)
             vdp_decoder_destroy(decoder);
         decoder = VDP_INVALID_HANDLE;
-        switch (image_format) {
-            case IMGFMT_VDPAU_MPEG1:
-                vdp_decoder_profile = VDP_DECODER_PROFILE_MPEG1;
-                break;
-            case IMGFMT_VDPAU_MPEG2:
-                vdp_decoder_profile = VDP_DECODER_PROFILE_MPEG2_MAIN;
-                break;
-            case IMGFMT_VDPAU_H264:
-                vdp_decoder_profile = VDP_DECODER_PROFILE_H264_HIGH;
+        if (image_format == IMGFMT_VDPAU_H264)
                 mp_msg(MSGT_VO, MSGL_V, "[vdpau] Creating H264 hardware decoder for %d reference frames.\n", max_refs);
-                break;
-            case IMGFMT_VDPAU_WMV3:
-                vdp_decoder_profile = VDP_DECODER_PROFILE_VC1_MAIN;
-                break;
-            case IMGFMT_VDPAU_VC1:
-                vdp_decoder_profile = VDP_DECODER_PROFILE_VC1_ADVANCED;
-                break;
-        }
         vdp_st = vdp_decoder_create(vdp_device, vdp_decoder_profile, vid_width, vid_height, max_refs, &decoder);
         CHECK_ST_WARNING("Failed creating VDPAU decoder");
         if (vdp_st != VDP_STATUS_OK) {