[FFmpeg-devel] Patch for IPC SHM

[Aran.Clauson at wwu.edu]

All,

I have encountered an error in libavdevice/xcbgrab.c that prevents NetBSD from
using X11grab.  When xcbgrab_frame_shm calls allocate_shm, it creates a private
shared memory region, attaches to that memory, sends the id to the server, but
then removes the shared memory ID.  On NetBSD with xorg, this causes the
subsequent call to xcb_shm_get_image to fail with BadShmSeg.  The fix is to keep
the ID valid until the memory is know to be shared, after the call to
xcb_shm_get_image_reply.  The following patch fixes the issue on my system.  It
is untested on others.

My guess is that Linux keeps the ID around until all attached processes
terminate.

For what it is worth, trac's registration system seems to be broken.  If you are
interested, please let me know and I can provide details.

Aran

*** libavdevice/xcbgrab.c.orig  Sun Jul 21 10:34:46 2019
--- libavdevice/xcbgrab.c       Sun Jul 21 10:34:55 2019
***************
*** 75,80 ****
--- 75,81 ----
      const char *framerate;

      int has_shm;
+     int shmid;
  } XCBGrabContext;

  #define FOLLOW_CENTER -1
***************
*** 221,226 ****
--- 222,235 ----
      return 0;
  }

+ static void rm_shmid(AVFormatContext *s) {
+     XCBGrabContext *c = s->priv_data;
+     if(c->shmid != -1) {
+       shmctl(c->shmid, IPC_RMID, 0);
+       c->shmid == -1;
+     }
+ }
+
  static int allocate_shm(AVFormatContext *s)
  {
      XCBGrabContext *c = s->priv_data;
***************
*** 230,236 ****

      if (c->buffer)
          return 0;
!     id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0777);
      if (id == -1) {
          char errbuf[1024];
          int err = AVERROR(errno);
--- 239,246 ----

      if (c->buffer)
          return 0;
!
!     id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0666);
      if (id == -1) {
          char errbuf[1024];
          int err = AVERROR(errno);
***************
*** 239,253 ****
                 size, errbuf);
          return err;
      }
      xcb_shm_attach(c->conn, c->segment, id, 0);
      data = shmat(id, NULL, 0);
!     shmctl(id, IPC_RMID, 0);
!     if ((intptr_t)data == -1 || !data)
!         return AVERROR(errno);
      c->buffer = data;
      return 0;
  }

  static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
  {
      XCBGrabContext *c = s->priv_data;
--- 249,268 ----
                 size, errbuf);
          return err;
      }
+
      xcb_shm_attach(c->conn, c->segment, id, 0);
      data = shmat(id, NULL, 0);
!
!     if ((intptr_t)data == -1 || !data) {
!       shmctl(id, IPC_RMID, 0);
!       return AVERROR(errno);
!     }
      c->buffer = data;
+     c->shmid = id;
      return 0;
  }

+
  static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
  {
      XCBGrabContext *c = s->priv_data;
***************
*** 268,273 ****
--- 283,290 ----

      xcb_flush(c->conn);

+     rm_shmid(s);
+
      if (e) {
          av_log(s, AV_LOG_ERROR,
                 "Cannot get the image data "
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 801 bytes
Desc: not available
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20190722/288dc2d4/attachment.sig>