[FFmpeg-cvslog] mp3dec: adjust IMDCT scaling to avoid overflows

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Thu Aug  2 04:21:53 2012 +0200| [571572fcddc16ebe3d60054ae5a2db05800c1d6d] | committer: Michael Niedermayer

mp3dec: adjust IMDCT scaling to avoid overflows

Fixes ticket268

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=571572fcddc16ebe3d60054ae5a2db05800c1d6d
---

 libavcodec/mpegaudio.h             |    2 ++
 libavcodec/mpegaudio_tablegen.h    |    4 ++--
 libavcodec/mpegaudiodsp_template.c |    2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libavcodec/mpegaudio.h b/libavcodec/mpegaudio.h
index b829cd3..b880b7a 100644
--- a/libavcodec/mpegaudio.h
+++ b/libavcodec/mpegaudio.h
@@ -52,6 +52,8 @@
 #define WFRAC_BITS  16   /* fractional bits for window */
 #endif
 
+#define IMDCT_SCALAR 1.759
+
 #define FRAC_ONE    (1 << FRAC_BITS)
 
 #define FIX(a)   ((int)((a) * FRAC_ONE))
diff --git a/libavcodec/mpegaudio_tablegen.h b/libavcodec/mpegaudio_tablegen.h
index 291e40b..a8756fa 100644
--- a/libavcodec/mpegaudio_tablegen.h
+++ b/libavcodec/mpegaudio_tablegen.h
@@ -47,7 +47,7 @@ static void mpegaudio_tableinit(void)
         double value = i / 4;
         double f, fm;
         int e, m;
-        f  = value * cbrtf(value) * pow(2, (i & 3) * 0.25);
+        f  = value / IMDCT_SCALAR * cbrtf(value) * pow(2, (i & 3) * 0.25);
         fm = frexp(f, &e);
         m  = (uint32_t)(fm * (1LL << 31) + 0.5);
         e += FRAC_BITS - 31 + 5 - 100;
@@ -58,7 +58,7 @@ static void mpegaudio_tableinit(void)
     }
     for (exponent = 0; exponent < 512; exponent++) {
         for (value = 0; value < 16; value++) {
-            double f = (double)value * cbrtf(value) * pow(2, (exponent - 400) * 0.25 + FRAC_BITS + 5);
+            double f = (double)value * cbrtf(value) * pow(2, (exponent - 400) * 0.25 + FRAC_BITS + 5) / IMDCT_SCALAR;
             expval_table_fixed[exponent][value] = llrint(f);
             expval_table_float[exponent][value] = f;
         }
diff --git a/libavcodec/mpegaudiodsp_template.c b/libavcodec/mpegaudiodsp_template.c
index 8df2ff3..5d00467 100644
--- a/libavcodec/mpegaudiodsp_template.c
+++ b/libavcodec/mpegaudiodsp_template.c
@@ -243,7 +243,7 @@ void RENAME(ff_init_mpadsp_tabs)(void)
                 else if (i <  18) d = 1;
             }
             //merge last stage of imdct into the window coefficients
-            d *= 0.5 / cos(M_PI * (2 * i + 19) / 72);
+            d *= 0.5 * IMDCT_SCALAR / cos(M_PI * (2 * i + 19) / 72);
 
             if (j == 2)
                 RENAME(ff_mdct_win)[j][i/3] = FIXHR((d / (1<<5)));