[FFmpeg-cvslog] avcodec/apedec: Fix 48khz 24bit below insane level

[Michael Niedermayer]

ffmpeg | branch: release/5.1 | Michael Niedermayer <michael at niedermayer.cc> | Fri Aug 25 16:59:15 2023 +0200| [e61e0ea8c6a57327669737fd4ffe0164fde2f06b] | committer: Michael Niedermayer

avcodec/apedec: Fix 48khz 24bit below insane level

Fixes: Ticket9816
Fixes: vlc.ape and APE_48K_24bit_2CH_02_01.ape

Regression since: ed0001482a74b60f3d5bc5cd7e304c9d65b2fcd5.

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 80ad0e2198df4e2961928d8304da58df6db77ec4)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e61e0ea8c6a57327669737fd4ffe0164fde2f06b
---

 libavcodec/apedec.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index 81953df221..8aca605343 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -1170,7 +1170,8 @@ static void predictor_decode_mono_3930(APEContext *ctx, int count)
 static av_always_inline int predictor_update_filter(APEPredictor64 *p,
                                                     const int decoded, const int filter,
                                                     const int delayA,  const int delayB,
-                                                    const int adaptA,  const int adaptB)
+                                                    const int adaptA,  const int adaptB,
+                                                    int compression_level)
 {
     int64_t predictionA, predictionB;
     int32_t sign;
@@ -1198,7 +1199,13 @@ static av_always_inline int predictor_update_filter(APEPredictor64 *p,
                   p->buf[delayB - 3] * p->coeffsB[filter][3] +
                   p->buf[delayB - 4] * p->coeffsB[filter][4];
 
-    p->lastA[filter] = decoded + ((int64_t)((uint64_t)predictionA + (predictionB >> 1)) >> 10);
+    if (compression_level < COMPRESSION_LEVEL_INSANE) {
+        predictionA = (int32_t)predictionA;
+        predictionB = (int32_t)predictionB;
+        p->lastA[filter] = decoded + ((int32_t)(predictionA + (predictionB >> 1)) >> 10);
+    } else {
+        p->lastA[filter] = decoded + ((int64_t)((uint64_t)predictionA + (predictionB >> 1)) >> 10);
+    }
     p->filterA[filter] = p->lastA[filter] + ((int64_t)(p->filterA[filter] * 31ULL) >> 5);
 
     sign = APESIGN(decoded);
@@ -1226,10 +1233,12 @@ static void predictor_decode_stereo_3950(APEContext *ctx, int count)
     while (count--) {
         /* Predictor Y */
         *decoded0 = predictor_update_filter(p, *decoded0, 0, YDELAYA, YDELAYB,
-                                            YADAPTCOEFFSA, YADAPTCOEFFSB);
+                                            YADAPTCOEFFSA, YADAPTCOEFFSB,
+                                            ctx->compression_level);
         decoded0++;
         *decoded1 = predictor_update_filter(p, *decoded1, 1, XDELAYA, XDELAYB,
-                                            XADAPTCOEFFSA, XADAPTCOEFFSB);
+                                            XADAPTCOEFFSA, XADAPTCOEFFSB,
+                                            ctx->compression_level);
         decoded1++;
 
         /* Combined */