[MPlayer-dev-eng] [PATCH] xvmc: handle shm xvimage allocation error

[Marcin Slusarz]

- handle xvimage/shm allocation failure and fallback to non-shm xv
- assert xvimage->data_size!=0 because 0 size will lead to memory corruptions
  (I hit it because of bug in Nouveau's early xvmc implementation)

Index: libvo/vo_xvmc.c
===================================================================
--- libvo/vo_xvmc.c	(wersja 34098)
+++ libvo/vo_xvmc.c	(kopia robocza)
@@ -181,6 +181,15 @@
     return cur_render->state || (cur_render->mpi && cur_render->mpi->usage_count);
 }
 
+static void allocate_xvimage_non_shm(int xvimage_width, int xvimage_height, int xv_format)
+{
+    xvimage = (XvImage *) XvCreateImage(mDisplay, xv_port, xv_format, NULL,
+                                        xvimage_width, xvimage_height);
+    assert(xvimage->data_size);
+    xvimage->data = malloc(xvimage->data_size);
+    XSync(mDisplay,False);
+}
+
 static void allocate_xvimage(int xvimage_width,int xvimage_height,int xv_format)
 {
  /*
@@ -188,7 +197,8 @@
   * mit-shm this will bomb... trzing to fix ::atmos
   */
 #ifdef HAVE_SHM
-    if ( mLocalDisplay && XShmQueryExtension( mDisplay ) ) Shmem_Flag = 1;
+    if ( mLocalDisplay && XShmQueryExtension( mDisplay ) )
+        Shmem_Flag = 0;
     else
     {
         Shmem_Flag = 0;
@@ -198,25 +208,41 @@
     {
         xvimage = (XvImage *) XvShmCreateImage(mDisplay, xv_port, xv_format,
                              NULL, xvimage_width, xvimage_height, &Shminfo);
+        if (!xvimage)
+            goto noshmimage;
 
+        assert(xvimage->data_size);
         Shminfo.shmid    = shmget(IPC_PRIVATE, xvimage->data_size, IPC_CREAT | 0777);
+        if (Shminfo.shmid == -1)
+            goto shmgetfail;
+
         Shminfo.shmaddr  = (char *) shmat(Shminfo.shmid, 0, 0);
+        if ((long)Shminfo.shmaddr == -1)
+            goto shmatfail;
+
         Shminfo.readOnly = False;
 
         xvimage->data = Shminfo.shmaddr;
-        XShmAttach(mDisplay, &Shminfo);
+        if (!XShmAttach(mDisplay, &Shminfo))
+            goto shmattachfail;
+
         XSync(mDisplay, False);
         shmctl(Shminfo.shmid, IPC_RMID, 0);
     }
     else
 #endif
-    {
-        xvimage = (XvImage *) XvCreateImage(mDisplay, xv_port, xv_format, NULL, xvimage_width, xvimage_height);
-        xvimage->data = malloc(xvimage->data_size);
-        XSync(mDisplay,False);
-    }
-// memset(xvimage->data,128,xvimage->data_size);
+    allocate_xvimage_non_shm(xvimage_width, xvimage_height, xv_format);
     return;
+
+shmattachfail:
+    shmdt(Shminfo.shmaddr);
+shmatfail:
+    shmctl(Shminfo.shmid, IPC_RMID, 0);
+shmgetfail:
+    XFree(xvimage);
+noshmimage:
+    Shmem_Flag = 0;
+    allocate_xvimage_non_shm(xvimage_width, xvimage_height, xv_format);
 }
 
 static void deallocate_xvimage(void)