[FFmpeg-cvslog] r20557 - trunk/libavcodec/dnxhdenc.c

[bcoudurier]

Author: bcoudurier
Date: Fri Nov 20 01:13:34 2009
New Revision: 20557

Log:
avoid integer overflow in dnxhd encoder, fixes #1557

Modified:
   trunk/libavcodec/dnxhdenc.c

Modified: trunk/libavcodec/dnxhdenc.c
==============================================================================
--- trunk/libavcodec/dnxhdenc.c	Fri Nov 20 00:28:21 2009	(r20556)
+++ trunk/libavcodec/dnxhdenc.c	Fri Nov 20 01:13:34 2009	(r20557)
@@ -574,9 +574,11 @@ static int dnxhd_encode_rdo(AVCodecConte
             last_higher = FFMAX(lambda, last_higher);
             if (last_lower != INT_MAX)
                 lambda = (lambda+last_lower)>>1;
+            else if ((int64_t)lambda + up_step > INT_MAX)
+                return -1;
             else
                 lambda += up_step;
-            up_step *= 5;
+            up_step = FFMIN((int64_t)up_step*5, INT_MAX);
             down_step = 1<<LAMBDA_FRAC_BITS;
         }
     }