[FFmpeg-devel] [PATCH] avfilter/vf_scale: validate values before converting to integer

[Kacper Michajłow]

Fixes the conversion of double values to integer, which may be out of
the representable range.

Also, bail out on overflow check instead of printing an error only.

Found by OSS-Fuzz.

Signed-off-by: Kacper Michajłow <kasper93 at gmail.com>
---
 libavfilter/vf_scale.c | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index ae7356fd7b..66bb81dd1f 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -537,7 +537,6 @@ static int scale_eval_dimensions(AVFilterContext *ctx)
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
     const AVPixFmtDescriptor *out_desc = av_pix_fmt_desc_get(outlink->format);
     char *expr;
-    int eval_w, eval_h;
     int ret;
     double res;
     const AVPixFmtDescriptor *main_desc;
@@ -588,26 +587,20 @@ static int scale_eval_dimensions(AVFilterContext *ctx)
     }
 
     res = av_expr_eval(scale->w_pexpr, scale->var_values, NULL);
-    eval_w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = (int) res == 0 ? inlink->w : (int) res;
-
-    res = av_expr_eval(scale->h_pexpr, scale->var_values, NULL);
-    if (isnan(res)) {
-        expr = scale->h_expr;
+    if (isnan(res) || res < INT_MIN || res > INT_MAX) {
+        expr = scale->w_expr;
         ret = AVERROR(EINVAL);
         goto fail;
     }
-    eval_h = scale->var_values[VAR_OUT_H] = scale->var_values[VAR_OH] = (int) res == 0 ? inlink->h : (int) res;
+    scale->w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = res == 0 ? inlink->w : res;
 
-    res = av_expr_eval(scale->w_pexpr, scale->var_values, NULL);
-    if (isnan(res)) {
-        expr = scale->w_expr;
+    res = av_expr_eval(scale->h_pexpr, scale->var_values, NULL);
+    if (isnan(res) || res < INT_MIN || res > INT_MAX) {
+        expr = scale->h_expr;
         ret = AVERROR(EINVAL);
         goto fail;
     }
-    eval_w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = (int) res == 0 ? inlink->w : (int) res;
-
-    scale->w = eval_w;
-    scale->h = eval_h;
+    scale->h = scale->var_values[VAR_OUT_H] = scale->var_values[VAR_OH] = res == 0 ? inlink->h : res;
 
     return 0;
 
@@ -642,11 +635,15 @@ static int config_props(AVFilterLink *outlink)
                                scale->force_original_aspect_ratio,
                                scale->force_divisible_by);
 
-    if (outlink->w > INT_MAX ||
-        outlink->h > INT_MAX ||
-        (outlink->h * inlink->w) > INT_MAX ||
-        (outlink->w * inlink->h) > INT_MAX)
+    if (outlink->w <= 0 ||
+        outlink->h <= 0 ||
+        outlink->h > INT_MAX / inlink->w ||
+        outlink->w > INT_MAX / inlink->h)
+    {
         av_log(ctx, AV_LOG_ERROR, "Rescaled value for width or height is too big.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
 
     /* TODO: make algorithm configurable */
 
-- 
2.43.0