[FFmpeg-devel] [PATCH] Fix segfault in x11grab when drawing Cursor on Xservers that don't support the XFixes extension

[Måns Rullgård]

Brenden Bain <brenden.bain at gmail.com> writes:

>> I'd just set nomouse here and carry on. ?If it failed once, it's
>> likely to fail again.
>
> Makes sense.
>
> Brenden.
>
> Index: libavdevice/x11grab.c
> ===================================================================
> --- libavdevice/x11grab.c	(revision 23733)
> +++ libavdevice/x11grab.c	(working copy)
> @@ -67,7 +67,7 @@
>      XImage *image;           /**< X11 image holding the grab */
>      int use_shm;             /**< !0 when using XShm extension */
>      XShmSegmentInfo shminfo; /**< When using XShm, keeps track of XShm infos */
> -    int nomouse;
> +    int nomouse;             /**< 0 when the mouse should be recoreded. */

Unrelated.

>  };
>  
>  /**
> @@ -91,7 +91,7 @@
>      XImage *image;
>      int x_off = 0;
>      int y_off = 0;
> -    int use_shm;
> +    int use_shm, ignore;
>      char *param, *offset;
>  
>      param = av_strdup(s1->filename);
> @@ -115,6 +115,11 @@
>          return AVERROR(EIO);
>      }
>  
> +    if (!XFixesQueryExtension(dpy, &ignore, &ignore)) {
> +        av_log(s1, AV_LOG_INFO, "Disabling cursor recording. Unable to query cursor shape.\n");
> +        x11grab->nomouse = 1;
> +    }
> +
>      st = av_new_stream(s1, 0);
>      if (!st) {
>          return AVERROR(ENOMEM);
> @@ -247,7 +252,7 @@
>   * @param y Mouse pointer coordinate
>   */
>  static void
> -paint_mouse_pointer(XImage *image, struct x11_grab *s)
> +paint_mouse_pointer(XImage *image, struct x11_grab *s, AVFormatContext *s1)

s1 is rather poor a name for that variable.

>  {
>      int x_off = s->x_off;
>      int y_off = s->y_off;
> @@ -262,6 +267,12 @@
>  
>      xcim = XFixesGetCursorImage(dpy);
>  
> +    if (xcim == NULL) {
> +        av_log(s1, AV_LOG_INFO, "Couldn't load mouse image. Unable to render cusor in video.\n");
> +        s->nomouse = 1;
> +        return;
> +    }
> +
>      x = xcim->x - xcim->xhot;
>      y = xcim->y - xcim->yhot;
>  
> @@ -388,7 +399,7 @@
>      }
>  
>      if(!s->nomouse){
> -        paint_mouse_pointer(image, s);
> +        paint_mouse_pointer(image, s, s1);

You could avoid passing the context around by having
paint_mouse_pointer() return an error code and handle it here.

-- 
M?ns Rullg?rd
mans at mansr.com