[FFmpeg-devel] [PATCH] swscale/utils: Add missing check for av_malloc

[Jiasheng Jiang]

As the potential failure of the memory allocation, the return
value of the av_malloc() could be NULL and be dereferenced on.
Therefore it should be better to check it and return error if fails.
Also, the callers of the ff_shuffle_filter_coefficients() should deal
with the return value.

Fixes: f900a19fa9 ("libswscale: Adds ff_hscale8to15_4_avx2 and ff_hscale8to15_X4_avx2 for all filter sizes.")
Signed-off-by: Jiasheng Jiang <jiasheng at iscas.ac.cn>
---
 libswscale/swscale_internal.h |  2 +-
 libswscale/utils.c            | 13 ++++++++++---
 tests/checkasm/sw_scale.c     |  3 ++-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 3a78d95ba6..26d28d42e6 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -1144,5 +1144,5 @@ void ff_sws_slice_worker(void *priv, int jobnr, int threadnr,
 #define MAX_LINES_AHEAD 4
 
 //shuffle filter and filterPos for hyScale and hcScale filters in avx2
-void ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
+int ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
 #endif /* SWSCALE_SWSCALE_INTERNAL_H */
diff --git a/libswscale/utils.c b/libswscale/utils.c
index c5ea8853d5..7754a03e00 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -278,7 +278,7 @@ static const FormatEntry format_entries[] = {
     [AV_PIX_FMT_P416LE]      = { 1, 1 },
 };
 
-void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSize, int16_t *filter, int dstW){
+int ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSize, int16_t *filter, int dstW){
 #if ARCH_X86_64
     int i, j, k, l;
     int cpu_flags = av_get_cpu_flags();
@@ -292,6 +292,9 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSiz
                     }
                     if (filterSize > 4){
                         int16_t *tmp2 = av_malloc(dstW * filterSize * 2);
+                        if (!tmp2)
+                            return AVERROR(ENOMEM);
+
                         memcpy(tmp2, filter, dstW * filterSize * 2);
                         for (i = 0; i < dstW; i += 16){//pixel
                             for (k = 0; k < filterSize / 4; ++k){//fcoeff
@@ -311,6 +314,8 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSiz
         }
     }
 #endif
+
+    return 0;
 }
 
 int sws_isSupportedInput(enum AVPixelFormat pix_fmt)
@@ -1836,7 +1841,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                            get_local_pos(c, 0, 0, 0),
                            get_local_pos(c, 0, 0, 0))) < 0)
                 goto fail;
-            ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW);
+            if ((ret = ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW)) < 0)
+                goto fail;
             if ((ret = initFilter(&c->hChrFilter, &c->hChrFilterPos,
                            &c->hChrFilterSize, c->chrXInc,
                            c->chrSrcW, c->chrDstW, filterAlign, 1 << 14,
@@ -1846,7 +1852,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                            get_local_pos(c, c->chrSrcHSubSample, c->src_h_chr_pos, 0),
                            get_local_pos(c, c->chrDstHSubSample, c->dst_h_chr_pos, 0))) < 0)
                 goto fail;
-            ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW);
+            if ((ret = ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW)) < 0)
+                goto fail;
         }
     } // initialize horizontal stuff
 
diff --git a/tests/checkasm/sw_scale.c b/tests/checkasm/sw_scale.c
index 3c0a083b42..0cb0ac4a4a 100644
--- a/tests/checkasm/sw_scale.c
+++ b/tests/checkasm/sw_scale.c
@@ -218,7 +218,8 @@ static void check_hscale(void)
             ff_sws_init_scale(ctx);
             memcpy(filterAvx2, filter, sizeof(uint16_t) * (SRC_PIXELS * MAX_FILTER_WIDTH + MAX_FILTER_WIDTH));
             if ((cpu_flags & AV_CPU_FLAG_AVX2) && !(cpu_flags & AV_CPU_FLAG_SLOW_GATHER))
-                ff_shuffle_filter_coefficients(ctx, filterPosAvx, width, filterAvx2, SRC_PIXELS);
+                if (ff_shuffle_filter_coefficients(ctx, filterPosAvx, width, filterAvx2, SRC_PIXELS) < 0)
+                    fail();
 
             if (check_func(ctx->hcScale, "hscale_%d_to_%d_width%d", ctx->srcBpc, ctx->dstBpc + 1, width)) {
                 memset(dst0, 0, SRC_PIXELS * sizeof(dst0[0]));
-- 
2.25.1