[FFmpeg-devel] [PATCHv2] lavc: avoid rounding errors in float constants

[remi at remlab.net]

From: Rémi Denis-Courmont <remi at remlab.net>

INT_MAX is (typically) a value with 31 significant bits but float can
only represent 23 significant bits, leading to a rounding error.

This substitutes the actual rounded value as an unsigned int,
to avoid a clang warning while not overflowing signed int:

 warning: implicit conversion from 'int' to 'float' changes value from
  2147483647 to 2147483648 [-Wimplicit-const-int-float-conversion]
---
 libavcodec/aaccoder.c | 2 +-
 libavcodec/imc.c      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libavcodec/aaccoder.c b/libavcodec/aaccoder.c
index e3b6b2f02c..4eab81c43f 100644
--- a/libavcodec/aaccoder.c
+++ b/libavcodec/aaccoder.c
@@ -531,7 +531,7 @@ static void search_for_quantizers_anmr(AVCodecContext *avctx, AACEncContext *s,
             int nz = 0;
 
             bandaddr[idx] = w * 16 + g;
-            qmin = INT_MAX;
+            qmin = 1U << 31;
             qmax = 0.0f;
             for (w2 = 0; w2 < sce->ics.group_len[w]; w2++) {
                 FFPsyBand *band = &s->psy.ch[s->cur_channel].psy_bands[(w+w2)*16+g];
diff --git a/libavcodec/imc.c b/libavcodec/imc.c
index 92f9980ded..afdb93636a 100644
--- a/libavcodec/imc.c
+++ b/libavcodec/imc.c
@@ -917,7 +917,7 @@ static int imc_decode_block(AVCodecContext *avctx, IMCContext *q, int ch)
                                        chctx->flcoeffs1, chctx->flcoeffs2);
 
     for(i=0; i<BANDS; i++) {
-        if(chctx->flcoeffs1[i] > INT_MAX) {
+        if(chctx->flcoeffs1[i] > (1U << 31)) {
             av_log(avctx, AV_LOG_ERROR, "scalefactor out of range\n");
             return AVERROR_INVALIDDATA;
         }
-- 
2.37.2