[FFmpeg-devel] Patch for IPC SHM

[Aran.Clauson at wwu.edu]

From: Carl Eugen Hoyos <ceffmpeg at gmail.com>
Subject: Re: [FFmpeg-devel] Patch for IPC SHM
Date: Mon, 22 Jul 2019 19:23:35 +0200

>
>
>
>> Am 22.07.2019 um 18:10 schrieb <Aran.Clauson at wwu.edu> <Aran.Clauson at wwu.edu>:
>>
>> All,
>>
>> I have encountered an error in libavdevice/xcbgrab.c that prevents NetBSD from
>> using X11grab.  When xcbgrab_frame_shm calls allocate_shm, it creates a private
>> shared memory region, attaches to that memory, sends the id to the server, but
>> then removes the shared memory ID.  On NetBSD with xorg, this causes the
>> subsequent call to xcb_shm_get_image to fail with BadShmSeg.  The fix is to keep
>> the ID valid until the memory is know to be shared, after the call to
>> xcb_shm_get_image_reply.  The following patch fixes the issue on my system.  It
>> is untested on others.
>>
>> My guess is that Linux keeps the ID around until all attached processes
>> terminate.
>
> Please create your patch with “git format-patch”, we cannot read other patches.
>
> Thank you, Carl Eugen
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".

diff --git a/libavdevice/xcbgrab.c b/libavdevice/xcbgrab.c
index b7e689343e..c2119946e1 100644
--- a/libavdevice/xcbgrab.c
+++ b/libavdevice/xcbgrab.c
@@ -75,6 +75,7 @@ typedef struct XCBGrabContext {
     const char *framerate;

     int has_shm;
+    int shmid;
 } XCBGrabContext;

 #define FOLLOW_CENTER -1
@@ -221,6 +222,14 @@ static int check_shm(xcb_connection_t *conn)
     return 0;
 }

+static void rm_shmid(AVFormatContext *s) {
+    XCBGrabContext *c = s->priv_data;
+    if(c->shmid != -1) {
+      shmctl(c->shmid, IPC_RMID, 0);
+      c->shmid == -1;
+    }
+}
+
 static int allocate_shm(AVFormatContext *s)
 {
     XCBGrabContext *c = s->priv_data;
@@ -230,7 +239,8 @@ static int allocate_shm(AVFormatContext *s)

     if (c->buffer)
         return 0;
-    id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0777);
+
+    id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0666);
     if (id == -1) {
         char errbuf[1024];
         int err = AVERROR(errno);
@@ -239,15 +249,20 @@ static int allocate_shm(AVFormatContext *s)
                size, errbuf);
         return err;
     }
+
     xcb_shm_attach(c->conn, c->segment, id, 0);
     data = shmat(id, NULL, 0);
-    shmctl(id, IPC_RMID, 0);
-    if ((intptr_t)data == -1 || !data)
-        return AVERROR(errno);
+
+    if ((intptr_t)data == -1 || !data) {
+      shmctl(id, IPC_RMID, 0);
+      return AVERROR(errno);
+    }
     c->buffer = data;
+    c->shmid = id;
     return 0;
 }

+
 static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
 {
     XCBGrabContext *c = s->priv_data;
@@ -268,6 +283,8 @@ static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)

     xcb_flush(c->conn);

+    rm_shmid(s);
+
     if (e) {
         av_log(s, AV_LOG_ERROR,
                "Cannot get the image data "
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 801 bytes
Desc: not available
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20190722/9c95cc36/attachment.sig>