[FFmpeg-devel] [PATCH 4/5] libswscale: Propagate error codes from ff_shuffle_filter_coefficients

[Alan Kelly]

---
 libswscale/swscale_internal.h |  2 +-
 libswscale/utils.c            | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 3a78d95ba6..26d28d42e6 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -1144,5 +1144,5 @@ void ff_sws_slice_worker(void *priv, int jobnr, int threadnr,
 #define MAX_LINES_AHEAD 4
 
 //shuffle filter and filterPos for hyScale and hcScale filters in avx2
-void ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
+int ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
 #endif /* SWSCALE_SWSCALE_INTERNAL_H */
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 31c365fcee..1f8705a417 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -278,7 +278,7 @@ static const FormatEntry format_entries[] = {
     [AV_PIX_FMT_P416LE]      = { 1, 1 },
 };
 
-void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos,
+int ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos,
                                    int filterSize, int16_t *filter,
                                    int dstW)
 {
@@ -286,13 +286,13 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos,
     int i, j, k;
     int cpu_flags = av_get_cpu_flags();
     if (!filter)
-        return;
+        return 0;
     if (EXTERNAL_AVX2_FAST(cpu_flags) && !(cpu_flags & AV_CPU_FLAG_SLOW_GATHER)) {
         if ((c->srcBpc == 8) && (c->dstBpc <= 14)) {
            int16_t *filterCopy = NULL;
            if (filterSize > 4) {
                if (!FF_ALLOC_TYPED_ARRAY(filterCopy, dstW * filterSize))
-                   return;
+                   return AVERROR(ENOMEM);
                memcpy(filterCopy, filter, dstW * filterSize * sizeof(int16_t));
            }
            // Do not swap filterPos for pixels which won't be processed by
@@ -333,7 +333,7 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos,
         }
     }
 #endif
-    return;
+    return 0;
 }
 
 int sws_isSupportedInput(enum AVPixelFormat pix_fmt)
@@ -1859,7 +1859,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                            get_local_pos(c, 0, 0, 0),
                            get_local_pos(c, 0, 0, 0))) < 0)
                 goto fail;
-            ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW);
+            if ((ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW)) < 0)
+                goto nomem;
             if ((ret = initFilter(&c->hChrFilter, &c->hChrFilterPos,
                            &c->hChrFilterSize, c->chrXInc,
                            c->chrSrcW, c->chrDstW, filterAlign, 1 << 14,
@@ -1869,7 +1870,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                            get_local_pos(c, c->chrSrcHSubSample, c->src_h_chr_pos, 0),
                            get_local_pos(c, c->chrDstHSubSample, c->dst_h_chr_pos, 0))) < 0)
                 goto fail;
-            ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW);
+            if ((ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW)) < 0)
+                goto nomem;
         }
     } // initialize horizontal stuff
 
-- 
2.35.0.263.gb82422642f-goog