[FFmpeg-cvslog] avcodec/aacdec_template: Fix undefined integer overflow in apply_tns()

[Michael Niedermayer]

ffmpeg | branch: release/3.1 | Michael Niedermayer <michael at niedermayer.cc> | Fri Jul 14 00:45:29 2017 +0200| [a2c3c9bc6038b03af917c7d840ca56d062574ba2] | committer: Michael Niedermayer

avcodec/aacdec_template: Fix undefined integer overflow in apply_tns()

Fixes: runtime error: signed integer overflow: -2147483648 - 1202286525 cannot be represented in type 'int'
Fixes: 2071/clusterfuzz-testcase-minimized-6036414271586304

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 0ef8f03133a0bd83c74200a8cf30982c0f574016)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a2c3c9bc6038b03af917c7d840ca56d062574ba2
---

 libavcodec/aac_defines.h     | 2 ++
 libavcodec/aacdec_template.c | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/libavcodec/aac_defines.h b/libavcodec/aac_defines.h
index 3c79a8a4a1..438d78a7aa 100644
--- a/libavcodec/aac_defines.h
+++ b/libavcodec/aac_defines.h
@@ -35,6 +35,7 @@
 #define AAC_RENAME(x)       x ## _fixed
 #define AAC_RENAME_32(x)    x ## _fixed_32
 typedef int                 INTFLOAT;
+typedef unsigned            UINTFLOAT;  ///< Equivalent to INTFLOAT, Used as temporal cast to avoid undefined sign overflow operations.
 typedef int64_t             INT64FLOAT;
 typedef int16_t             SHORTFLOAT;
 typedef SoftFloat           AAC_FLOAT;
@@ -83,6 +84,7 @@ typedef int                 AAC_SIGNE;
 #define AAC_RENAME(x)       x
 #define AAC_RENAME_32(x)    x
 typedef float               INTFLOAT;
+typedef float               UINTFLOAT;
 typedef float               INT64FLOAT;
 typedef float               SHORTFLOAT;
 typedef float               AAC_FLOAT;
diff --git a/libavcodec/aacdec_template.c b/libavcodec/aacdec_template.c
index a37032a025..996ab638ad 100644
--- a/libavcodec/aacdec_template.c
+++ b/libavcodec/aacdec_template.c
@@ -2363,7 +2363,7 @@ static int decode_extension_payload(AACContext *ac, GetBitContext *gb, int cnt,
  * @param   decode  1 if tool is used normally, 0 if tool is used in LTP.
  * @param   coef    spectral coefficients
  */
-static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
+static void apply_tns(INTFLOAT coef_param[1024], TemporalNoiseShaping *tns,
                       IndividualChannelStream *ics, int decode)
 {
     const int mmm = FFMIN(ics->tns_max_bands, ics->max_sfb);
@@ -2371,6 +2371,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
     int bottom, top, order, start, end, size, inc;
     INTFLOAT lpc[TNS_MAX_ORDER];
     INTFLOAT tmp[TNS_MAX_ORDER+1];
+    UINTFLOAT *coef = coef_param;
 
     for (w = 0; w < ics->num_windows; w++) {
         bottom = ics->num_swb;
@@ -2400,7 +2401,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
                 // ar filter
                 for (m = 0; m < size; m++, start += inc)
                     for (i = 1; i <= FFMIN(m, order); i++)
-                        coef[start] -= AAC_MUL26(coef[start - i * inc], lpc[i - 1]);
+                        coef[start] -= AAC_MUL26((INTFLOAT)coef[start - i * inc], lpc[i - 1]);
             } else {
                 // ma filter
                 for (m = 0; m < size; m++, start += inc) {