[FFmpeg-cvslog] avutil/mathematics: return INT64_MIN (=AV_NOPTS_VALUE) from av_rescale_rnd () for overflows

[Michael Niedermayer]

ffmpeg | branch: release/2.8 | Michael Niedermayer <michael at niedermayer.cc> | Tue Dec  1 13:32:31 2015 +0100| [e04b039b1528f4c7df5c2b93865651bfea168a19] | committer: Michael Niedermayer

avutil/mathematics: return INT64_MIN (=AV_NOPTS_VALUE) from av_rescale_rnd() for overflows

Fixes integer overflow
Fixes: mozilla bug 1229167

Found-by: Tyson Smith
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit f03c2ceec174877e03bb302f5971fbe9ffbe4856)

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e04b039b1528f4c7df5c2b93865651bfea168a19
---

 libavutil/mathematics.c |   13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libavutil/mathematics.c b/libavutil/mathematics.c
index b1ffd65..4d8467b 100644
--- a/libavutil/mathematics.c
+++ b/libavutil/mathematics.c
@@ -77,7 +77,7 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
     }
 
     if (a < 0)
-        return -av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1));
+        return -(uint64_t)av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1));
 
     if (rnd == AV_ROUND_NEAR_INF)
         r = c / 2;
@@ -87,8 +87,13 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
     if (b <= INT_MAX && c <= INT_MAX) {
         if (a <= INT_MAX)
             return (a * b + r) / c;
-        else
-            return a / c * b + (a % c * b + r) / c;
+        else {
+            int64_t ad = a / c;
+            int64_t a2 = (a % c * b + r) / c;
+            if (ad >= INT32_MAX && ad > (INT64_MAX - a2) / b)
+                return INT64_MIN;
+            return ad * b + a2;
+        }
     } else {
 #if 1
         uint64_t a0  = a & 0xFFFFFFFF;
@@ -112,6 +117,8 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
                 t1++;
             }
         }
+        if (t1 > INT64_MAX)
+            return INT64_MIN;
         return t1;
     }
 #else