[FFmpeg-cvslog] avcodec/j2kenc: Merge dwt_norm into lambda

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Wed Jun 19 20:58:05 2024 +0200| [a84fbd747119841942c67d2f55cc796ab25cd245] | committer: Michael Niedermayer

avcodec/j2kenc: Merge dwt_norm into lambda

This moves computations out of a loop

This may help with UB in vsynth*-jpeg2000-yuva444p16

Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long'
Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a84fbd747119841942c67d2f55cc796ab25cd245
---

 libavcodec/j2kenc.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 8cf82f7216..91e66d8104 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
     }
 }
 
-static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm)
+static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda)
 {
     int passno, res = 0;
     for (passno = 0; passno < cblk->npasses; passno++){
@@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm)
         dd = cblk->passes[passno].disto
            - (res ? cblk->passes[res-1].disto : 0);
 
-        if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda)
+        if (dd  >= dr * lambda)
             res = passno+1;
     }
     return res;
@@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
                     Jpeg2000Band *band = reslevel->band + bandno;
                     Jpeg2000Prec *prec = band->prec + precno;
 
+                    int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15;
+                    int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm);
                     for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){
                         Jpeg2000Cblk *cblk = prec->cblk + cblkno;
 
-                        cblk->ninclpasses = getcut(cblk, s->lambda,
-                                (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15);
+                        cblk->ninclpasses = getcut(cblk, lambda_prime);
                         cblk->layers[0].data_start = cblk->data;
                         cblk->layers[0].cum_passes = cblk->ninclpasses;
                         cblk->layers[0].npasses = cblk->ninclpasses;