[FFmpeg-cvslog] lavu/mem: Make other alloc functions more similar to av_malloc().

[Carl Eugen Hoyos]

ffmpeg | branch: master | Carl Eugen Hoyos <ceffmpeg at gmail.com> | Sun Apr 12 00:36:30 2020 +0200| [b7d9507bb8c4d1b8bf99158d6859a5b2ecd73298] | committer: Carl Eugen Hoyos

lavu/mem: Make other alloc functions more similar to av_malloc().

Do not limit the array allocation functions and av_calloc() to allocations
of INT_MAX, instead depend on max_alloc_size like av_malloc().

Allows a workaround for ticket #7140.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=b7d9507bb8c4d1b8bf99158d6859a5b2ecd73298
---

 libavutil/mem.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/libavutil/mem.c b/libavutil/mem.c
index 88fe09b179..a29c224264 100644
--- a/libavutil/mem.c
+++ b/libavutil/mem.c
@@ -183,23 +183,26 @@ int av_reallocp(void *ptr, size_t size)
 
 void *av_malloc_array(size_t nmemb, size_t size)
 {
-    if (!size || nmemb >= INT_MAX / size)
+    size_t result;
+    if (av_size_mult(nmemb, size, &result) < 0)
         return NULL;
-    return av_malloc(nmemb * size);
+    return av_malloc(result);
 }
 
 void *av_mallocz_array(size_t nmemb, size_t size)
 {
-    if (!size || nmemb >= INT_MAX / size)
+    size_t result;
+    if (av_size_mult(nmemb, size, &result) < 0)
         return NULL;
-    return av_mallocz(nmemb * size);
+    return av_mallocz(result);
 }
 
 void *av_realloc_array(void *ptr, size_t nmemb, size_t size)
 {
-    if (!size || nmemb >= INT_MAX / size)
+    size_t result;
+    if (av_size_mult(nmemb, size, &result) < 0)
         return NULL;
-    return av_realloc(ptr, nmemb * size);
+    return av_realloc(ptr, result);
 }
 
 int av_reallocp_array(void *ptr, size_t nmemb, size_t size)
@@ -243,9 +246,10 @@ void *av_mallocz(size_t size)
 
 void *av_calloc(size_t nmemb, size_t size)
 {
-    if (size <= 0 || nmemb >= INT_MAX / size)
+    size_t result;
+    if (av_size_mult(nmemb, size, &result) < 0)
         return NULL;
-    return av_mallocz(nmemb * size);
+    return av_mallocz(result);
 }
 
 char *av_strdup(const char *s)