[FFmpeg-cvslog] tiff: fix overflows when calling av_reduce

[Andreas Cadhalpun]

ffmpeg | branch: master | Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com> | Tue Dec 13 00:43:21 2016 +0100| [ed412d285078c167a3a5326bcb16b2169b488943] | committer: Andreas Cadhalpun

tiff: fix overflows when calling av_reduce

The arguments of av_reduce are signed, so the cast to uint64_t is misleading.

Reviewed-by: Michael Niedermayer <michael at niedermayer.cc>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ed412d285078c167a3a5326bcb16b2169b488943
---

 libavcodec/tiff.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c
index 4721e94..7ccda51 100644
--- a/libavcodec/tiff.c
+++ b/libavcodec/tiff.c
@@ -772,9 +772,18 @@ static void set_sar(TiffContext *s, unsigned tag, unsigned num, unsigned den)
     int offset = tag == TIFF_YRES ? 2 : 0;
     s->res[offset++] = num;
     s->res[offset]   = den;
-    if (s->res[0] && s->res[1] && s->res[2] && s->res[3])
+    if (s->res[0] && s->res[1] && s->res[2] && s->res[3]) {
+        uint64_t num = s->res[2] * (uint64_t)s->res[1];
+        uint64_t den = s->res[0] * (uint64_t)s->res[3];
+        if (num > INT64_MAX || den > INT64_MAX) {
+            num = num >> 1;
+            den = den >> 1;
+        }
         av_reduce(&s->avctx->sample_aspect_ratio.num, &s->avctx->sample_aspect_ratio.den,
-                  s->res[2] * (uint64_t)s->res[1], s->res[0] * (uint64_t)s->res[3], INT32_MAX);
+                  num, den, INT32_MAX);
+        if (!s->avctx->sample_aspect_ratio.den)
+            s->avctx->sample_aspect_ratio = (AVRational) {0, 1};
+    }
 }
 
 static int tiff_decode_tag(TiffContext *s, AVFrame *frame)