[FFmpeg-cvslog] avutil/softfloat: Fix overflows in shifts in av_cmp_sf() and av_gt_sf()

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sun Nov  8 14:13:42 2015 +0100| [cee3c9d29aceec8cddd829acd6dfb56dc5f60322] | committer: Michael Niedermayer

avutil/softfloat: Fix overflows in shifts in av_cmp_sf() and av_gt_sf()

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=cee3c9d29aceec8cddd829acd6dfb56dc5f60322
---

 libavutil/softfloat.h |   12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/libavutil/softfloat.h b/libavutil/softfloat.h
index e5bfbbd..ae11bdb 100644
--- a/libavutil/softfloat.h
+++ b/libavutil/softfloat.h
@@ -119,15 +119,19 @@ static inline av_const SoftFloat av_div_sf(SoftFloat a, SoftFloat b){
 
 static inline av_const int av_cmp_sf(SoftFloat a, SoftFloat b){
     int t= a.exp - b.exp;
-    if(t<0) return (a.mant >> (-t)) -  b.mant      ;
-    else    return  a.mant          - (b.mant >> t);
+    if      (t <-31) return                  -  b.mant      ;
+    else if (t <  0) return (a.mant >> (-t)) -  b.mant      ;
+    else if (t < 32) return  a.mant          - (b.mant >> t);
+    else             return  a.mant                         ;
 }
 
 static inline av_const int av_gt_sf(SoftFloat a, SoftFloat b)
 {
     int t= a.exp - b.exp;
-    if(t<0) return (a.mant >> (-t)) >  b.mant      ;
-    else    return  a.mant          > (b.mant >> t);
+    if      (t <-31) return 0;
+    else if (t <  0) return (a.mant >> (-t)) >  b.mant      ;
+    else if (t < 32) return  a.mant          > (b.mant >> t);
+    else             return 1;
 }
 
 static inline av_const SoftFloat av_add_sf(SoftFloat a, SoftFloat b){