[FFmpeg-cvslog] win32: Make ff_win32_open more robust

[Reimar Döffinger]

ffmpeg | branch: master | Reimar Döffinger <Reimar.Doeffinger at gmx.de> | Sun Mar 10 20:03:19 2013 +0100| [ad04025987e5c926ae5bde8947a44b3f1346e64c] | committer: Martin Storsjö

win32: Make ff_win32_open more robust

- Make MultiByteToWideChar fail when it encounters invalid encoding.
  Without this, invalid characters might just be skipped
- When MultiByteToWideChar fails, assume the file name is in CP_ACP
  and open it via normal open function, even when the file will be
  written
- When malloc fails return error instead of crashing

Signed-off-by: Martin Storsjö <martin at martin.st>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ad04025987e5c926ae5bde8947a44b3f1346e64c
---

 libavformat/os_support.c |   18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/libavformat/os_support.c b/libavformat/os_support.c
index 1ecf43c..30581ed 100644
--- a/libavformat/os_support.c
+++ b/libavformat/os_support.c
@@ -33,6 +33,7 @@
 #include <io.h>
 #include <windows.h>
 #include <share.h>
+#include <errno.h>
 
 int ff_win32_open(const char *filename_utf8, int oflag, int pmode)
 {
@@ -41,20 +42,25 @@ int ff_win32_open(const char *filename_utf8, int oflag, int pmode)
     wchar_t *filename_w;
 
     /* convert UTF-8 to wide chars */
-    num_chars = MultiByteToWideChar(CP_UTF8, 0, filename_utf8, -1, NULL, 0);
+    num_chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, filename_utf8, -1, NULL, 0);
     if (num_chars <= 0)
-        return -1;
+        goto fallback;
     filename_w = av_mallocz(sizeof(wchar_t) * num_chars);
+    if (!filename_w) {
+        errno = ENOMEM;
+        return -1;
+    }
     MultiByteToWideChar(CP_UTF8, 0, filename_utf8, -1, filename_w, num_chars);
 
     fd = _wsopen(filename_w, oflag, SH_DENYNO, pmode);
     av_freep(&filename_w);
 
-    /* filename maybe be in CP_ACP */
-    if (fd == -1 && !(oflag & O_CREAT))
-        return _sopen(filename_utf8, oflag, SH_DENYNO, pmode);
+    if (fd != -1 || (oflag & O_CREAT))
+        return fd;
 
-    return fd;
+fallback:
+    /* filename may be be in CP_ACP */
+    return _sopen(filename_utf8, oflag, SH_DENYNO, pmode);
 }
 #endif