[FFmpeg-devel] [PATCH v2 1/2] avformat/file: Add a specialized url_check callback for pipe protocol

[Neil Roberts]

Using file_check for the pipe protocol doesn't make sense. For example,
for a URL like “pipe:0” it would end up checking whether the “pipe:0”
file exists. This patch instead makes it check the access modes on the
corresponding file descriptor using fcntl on *nix and DuplicateHandle on
Windows.

v2: Use DuplicateHandle on Windows to check whether the duplicated
    handle can have the corresponding access flags.

Signed-off-by: Neil Roberts <bpeeluk at yahoo.co.uk>
---
 libavformat/file.c | 74 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 71 insertions(+), 3 deletions(-)

diff --git a/libavformat/file.c b/libavformat/file.c
index 98c9e81bcb..b3f7bc9282 100644
--- a/libavformat/file.c
+++ b/libavformat/file.c
@@ -375,9 +375,8 @@ const URLProtocol ff_file_protocol = {
 
 #if CONFIG_PIPE_PROTOCOL
 
-static int pipe_open(URLContext *h, const char *filename, int flags)
+static int pipe_get_fd(const char *filename, int flags)
 {
-    FileContext *c = h->priv_data;
     int fd;
     char *final;
     av_strstart(filename, "pipe:", &filename);
@@ -390,6 +389,14 @@ static int pipe_open(URLContext *h, const char *filename, int flags)
             fd = 0;
         }
     }
+
+    return fd;
+}
+
+static int pipe_open(URLContext *h, const char *filename, int flags)
+{
+    FileContext *c = h->priv_data;
+    int fd = pipe_get_fd(filename, flags);
 #if HAVE_SETMODE
     setmode(fd, O_BINARY);
 #endif
@@ -398,13 +405,74 @@ static int pipe_open(URLContext *h, const char *filename, int flags)
     return 0;
 }
 
+static int pipe_check(URLContext *h, int mask)
+{
+    int fd = pipe_get_fd(h->filename, mask);
+    int access = 0;
+
+#ifdef _WIN32
+
+    HANDLE pipe_handle = (HANDLE) _get_osfhandle(fd);
+    HANDLE tmp_handle;
+
+    if (pipe_handle == INVALID_HANDLE_VALUE)
+        return AVERROR(errno);
+
+    if (mask & AVIO_FLAG_READ &&
+        DuplicateHandle(GetCurrentProcess(), /* hSourceProcessHandle */
+                        pipe_handle,
+                        GetCurrentProcess(), /* hTargetProcessHandle */
+                        &tmp_handle,
+                        FILE_READ_DATA,
+                        FALSE, /* bInheritHandle */
+                        0 /* options */)) {
+        access |= AVIO_FLAG_READ;
+        CloseHandle(tmp_handle);
+    }
+
+    if (mask & AVIO_FLAG_WRITE &&
+        DuplicateHandle(GetCurrentProcess(), /* hSourceProcessHandle */
+                        pipe_handle,
+                        GetCurrentProcess(), /* hTargetProcessHandle */
+                        &tmp_handle,
+                        FILE_WRITE_DATA,
+                        FALSE, /* bInheritHandle */
+                        0 /* options */)) {
+        access |= AVIO_FLAG_WRITE;
+        CloseHandle(tmp_handle);
+    }
+
+#else /* _WIN32 */
+
+    int status_flags = fcntl(fd, F_GETFL);
+
+    if (status_flags == -1)
+        return AVERROR(errno);
+
+    switch (status_flags & O_ACCMODE) {
+    case O_RDONLY:
+        access = AVIO_FLAG_READ;
+        break;
+    case O_WRONLY:
+        access = AVIO_FLAG_WRITE;
+        break;
+    case O_RDWR:
+        access = AVIO_FLAG_READ | AVIO_FLAG_WRITE;
+        break;
+    }
+
+#endif /* _WIN32 */
+
+    return access & mask;
+}
+
 const URLProtocol ff_pipe_protocol = {
     .name                = "pipe",
     .url_open            = pipe_open,
     .url_read            = file_read,
     .url_write           = file_write,
     .url_get_file_handle = file_get_handle,
-    .url_check           = file_check,
+    .url_check           = pipe_check,
     .priv_data_size      = sizeof(FileContext),
     .priv_data_class     = &pipe_class,
     .default_whitelist   = "crypto,data"
-- 
2.37.2