[FFmpeg-soc] [soc]: r1188 - in jpeg2000: j2kdec.c j2kenc.c

k.nowosad subversion at mplayerhq.hu
Sat Aug 25 18:16:51 CEST 2007


Author: k.nowosad
Date: Sat Aug 25 18:16:51 2007
New Revision: 1188

Log:
added buffer overflow checks


Modified:
   jpeg2000/j2kdec.c
   jpeg2000/j2kenc.c

Modified: jpeg2000/j2kdec.c
==============================================================================
--- jpeg2000/j2kdec.c	(original)
+++ jpeg2000/j2kdec.c	Sat Aug 25 18:16:51 2007
@@ -73,6 +73,8 @@ typedef struct {
 static int get_bits(J2kDecoderContext *s, int n)
 {
     int res = 0;
+    if (s->buf_end - s->buf < (n - s->bit_index >> 8))
+        return AVERROR(EINVAL);
     while (--n >= 0){
         res <<= 1;
         if (s->bit_index == 0){
@@ -178,11 +180,14 @@ static int tag_tree_decode(J2kDecoderCon
         if (curval < stack[sp]->val)
             curval = stack[sp]->val;
         while (curval < threshold){
-            if (get_bits(s, 1)){
+            int ret;
+            if ((ret = get_bits(s, 1)) > 0){
                 stack[sp]->vis++;
                 break;
-            } else
+            } else if (!ret)
                 curval++;
+            else
+                return ret;
         }
         stack[sp]->val = curval;
         sp--;
@@ -196,6 +201,9 @@ static int get_siz(J2kDecoderContext *s)
 {
     int i;
 
+    if (s->buf_end - s->buf < 36)
+        return AVERROR(EINVAL);
+
                         bytestream_get_be16(&s->buf); // Rsiz (skipped)
              s->width = bytestream_get_be32(&s->buf); // width
             s->height = bytestream_get_be32(&s->buf); // height
@@ -208,6 +216,9 @@ static int get_siz(J2kDecoderContext *s)
      s->tile_offset_y = bytestream_get_be32(&s->buf); // YT0Siz
        s->ncomponents = bytestream_get_be16(&s->buf); // CSiz
 
+    if (s->buf_end - s->buf < 2 * s->ncomponents)
+        return AVERROR(EINVAL);
+
     for (i = 0; i < s->ncomponents; i++){ // Ssiz_i XRsiz_i, YRsiz_i
         uint8_t x = bytestream_get_byte(&s->buf);
         s->cbps[i] = (x & 0x7f) + 1;
@@ -256,6 +267,8 @@ static int get_siz(J2kDecoderContext *s)
 /** get common part for COD and COC segments */
 static int get_cox(J2kDecoderContext *s, J2kCodingStyle *c)
 {
+    if (s->buf_end - s->buf < 5)
+        return AVERROR(EINVAL);
           c->nreslevels = bytestream_get_byte(&s->buf) + 1; // num of resolution levels - 1
      c->log2_cblk_width = bytestream_get_byte(&s->buf) + 2; // cblk width
     c->log2_cblk_height = bytestream_get_byte(&s->buf) + 2; // cblk height
@@ -274,6 +287,9 @@ static int get_cod(J2kDecoderContext *s,
     J2kCodingStyle tmp;
     int compno;
 
+    if (s->buf_end - s->buf < 5)
+        return AVERROR(EINVAL);
+
     tmp.log2_prec_width  =
     tmp.log2_prec_height = 15;
 
@@ -298,7 +314,12 @@ static int get_cod(J2kDecoderContext *s,
 /** get coding parameters for a component in the whole image on a particular tile */
 static int get_coc(J2kDecoderContext *s, J2kCodingStyle *c, uint8_t *properties)
 {
-    int compno = bytestream_get_byte(&s->buf);
+    int compno;
+
+    if (s->buf_end - s->buf < 2)
+        return AVERROR(EINVAL);
+
+    compno = bytestream_get_byte(&s->buf);
 
     c += compno;
     c->csty = bytestream_get_byte(&s->buf);
@@ -312,6 +333,10 @@ static int get_coc(J2kDecoderContext *s,
 static int get_qcx(J2kDecoderContext *s, int n, J2kQuantStyle *q)
 {
     int i, x;
+
+    if (s->buf_end - s->buf < 1)
+        return AVERROR(EINVAL);
+
     x = bytestream_get_byte(&s->buf); // Sqcd
 
     q->nguardbits = x >> 5;
@@ -319,10 +344,14 @@ static int get_qcx(J2kDecoderContext *s,
 
     if (q->quantsty == J2K_QSTY_NONE){
         n -= 3;
+        if (s->buf_end - s->buf < n)
+            return AVERROR(EINVAL);
         for (i = 0; i < n; i++)
             q->expn[i] = bytestream_get_byte(&s->buf) >> 3;
     }
     else if (q->quantsty == J2K_QSTY_SI){
+        if (s->buf_end - s->buf < 2)
+            return AVERROR(EINVAL);
         x = bytestream_get_be16(&s->buf);
         q->expn[0] = x >> 11;
         q->mant[0] = x & 0x7ff;
@@ -334,6 +363,8 @@ static int get_qcx(J2kDecoderContext *s,
     }
     else{
         n = (n - 3) >> 1;
+        if (s->buf_end - s->buf < n)
+            return AVERROR(EINVAL);
         for (i = 0; i < n; i++){
             x = bytestream_get_be16(&s->buf);
             q->expn[i] = x >> 11;
@@ -360,7 +391,12 @@ static int get_qcd(J2kDecoderContext *s,
 /** get quantization paramteres for a component in the whole image on in a particular tile */
 static int get_qcc(J2kDecoderContext *s, int n, J2kQuantStyle *q, uint8_t *properties)
 {
-    int compno = bytestream_get_byte(&s->buf);
+    int compno;
+
+    if (s->buf_end - s->buf < 1)
+        return AVERROR(EINVAL);
+
+    compno = bytestream_get_byte(&s->buf);
     properties[compno] |= HAD_QCC;
     return get_qcx(s, n-1, q+compno);
 }
@@ -368,6 +404,9 @@ static int get_qcc(J2kDecoderContext *s,
 /** get start of tile segment */
 static uint8_t get_sot(J2kDecoderContext *s)
 {
+    if (s->buf_end - s->buf < 4)
+        return AVERROR(EINVAL);
+
     s->curtileno = bytestream_get_be16(&s->buf); ///< Isot
 
     s->buf += 4; ///< Psot (ignored)
@@ -419,28 +458,35 @@ static int getnpasses(J2kDecoderContext 
     if (!get_bits(s, 1))
         return 2;
     if ((num = get_bits(s, 2)) != 3)
-        return 3 + num;
+        return num < 0 ? num : 3 + num;
     if ((num = get_bits(s, 5)) != 31)
-        return 6 + num;
-    return 37 + get_bits(s, 7);
+        return num < 0 ? num : 6 + num;
+    num = get_bits(s, 7);
+    return num < 0 ? num : 37 + num;
 }
 
 static int getlblockinc(J2kDecoderContext *s)
 {
-    int res = 0;
-    while (get_bits(s, 1))
+    int res = 0, ret;
+    while (ret = get_bits(s, 1)){
+        if (ret < 0)
+            return ret;
         res++;
+    }
     return res;
 }
 
 static int decode_packet(J2kDecoderContext *s, J2kResLevel *rlevel, int precno, int layno, uint8_t *expn, int numgbits)
 {
-    int bandno, cblkny, cblknx, cblkno;
+    int bandno, cblkny, cblknx, cblkno, ret;
 
-    if (!get_bits(s, 1)){
+    if (!(ret = get_bits(s, 1))){
         j2k_flush(s);
         return 0;
     }
+    else if (ret < 0)
+        return ret;
+
     for (bandno = 0; bandno < rlevel->nbands; bandno++){
         J2kBand *band = rlevel->band + bandno;
         J2kPrec *prec = band->prec + precno;
@@ -456,15 +502,20 @@ static int decode_packet(J2kDecoderConte
                 else{
                     incl = tag_tree_decode(s, prec->cblkincl + pos, layno+1) == layno;
                 }
-                if (!incl){
+                if (!incl)
                     continue;
-                }
+                else if (incl < 0)
+                    return incl;
+
                 if (!cblk->npasses)
                     cblk->nonzerobits = expn[bandno] + numgbits - 1 - tag_tree_decode(s, prec->zerobits + pos, 100);
-                newpasses = getnpasses(s);
-                llen = getlblockinc(s);
+                if ((newpasses = getnpasses(s)) < 0)
+                    return newpasses;
+                if ((llen = getlblockinc(s)) < 0)
+                    return llen;
                 cblk->lblock += llen;
-                cblk->lengthinc = get_bits(s, av_log2(newpasses) + cblk->lblock);
+                if ((cblk->lengthinc = get_bits(s, av_log2(newpasses) + cblk->lblock)) < 0)
+                    return cblk->lengthinc;
                 cblk->npasses += newpasses;
             }
     }
@@ -476,6 +527,8 @@ static int decode_packet(J2kDecoderConte
             int xi;
             for (xi = band->prec[precno].xi0; xi < band->prec[precno].xi1; xi++){
                 J2kCblk *cblk = band->cblk + yi * cblknw + xi;
+                if (s->buf_end - s->buf < cblk->lengthinc)
+                    return AVERROR(EINVAL);
                 bytestream_get_buffer(&s->buf, cblk->data, cblk->lengthinc);
                 cblk->length += cblk->lengthinc;
                 cblk->lengthinc = 0;
@@ -786,8 +839,14 @@ static int decode_codestream(J2kDecoderC
     uint8_t *properties = s->properties;
 
     for (;;){
-        int marker = bytestream_get_be16(&s->buf), len, ret = 0;
-        uint8_t *oldbuf = s->buf;
+        int marker, len, ret = 0;
+        uint8_t *oldbuf;
+
+        if (s->buf_end - s->buf < 2)
+            return AVERROR(EINVAL);
+
+        marker = bytestream_get_be16(&s->buf);
+        oldbuf = s->buf;
 
         if (marker == J2K_SOD){
             J2kTile *tile = s->tile + s->curtileno;
@@ -800,6 +859,8 @@ static int decode_codestream(J2kDecoderC
         if (marker == J2K_EOC)
             break;
 
+        if (s->buf_end - s->buf < 2)
+            return AVERROR(EINVAL);
         len = bytestream_get_be16(&s->buf);
         switch(marker){
             case J2K_SIZ:
@@ -852,6 +913,8 @@ static int decode_frame(AVCodecContext *
 
     ff_j2k_init_tier1_luts();
 
+    if (s->buf_end - s->buf < 2)
+        return AVERROR(EINVAL);
     if (bytestream_get_be16(&s->buf) != J2K_SOC){
         av_log(avctx, AV_LOG_ERROR, "SOC marker not present\n");
         return -1;

Modified: jpeg2000/j2kenc.c
==============================================================================
--- jpeg2000/j2kenc.c	(original)
+++ jpeg2000/j2kenc.c	Sat Aug 25 18:16:51 2007
@@ -229,10 +229,13 @@ static void tag_tree_update(J2kTgtNode *
     }
 }
 
-static void put_siz(J2kEncoderContext *s)
+static int put_siz(J2kEncoderContext *s)
 {
     int i;
 
+    if (s->buf_end - s->buf < 40 + 3 * s->ncomponents)
+        return -1;
+
     bytestream_put_be16(&s->buf, J2K_SIZ);
     bytestream_put_be16(&s->buf, 38 + 3 * s->ncomponents); // Lsiz
     bytestream_put_be16(&s->buf, 0); // Rsiz
@@ -252,12 +255,16 @@ static void put_siz(J2kEncoderContext *s
         bytestream_put_byte(&s->buf, 1);
         bytestream_put_byte(&s->buf, 1);
     }
+    return 0;
 }
 
-static void put_cod(J2kEncoderContext *s)
+static int put_cod(J2kEncoderContext *s)
 {
     J2kCodingStyle *codsty = &s->codsty;
 
+    if (s->buf_end - s->buf < 14)
+        return -1;
+
     bytestream_put_be16(&s->buf, J2K_COD);
     bytestream_put_be16(&s->buf, 12); // Lcod
     bytestream_put_byte(&s->buf, 0);  // Scod
@@ -271,24 +278,33 @@ static void put_cod(J2kEncoderContext *s
     bytestream_put_byte(&s->buf, codsty->log2_cblk_height-2); // cblk height
     bytestream_put_byte(&s->buf, 0); // cblk style
     bytestream_put_byte(&s->buf, 1); // transformation
+    return 0;
 }
 
-static void put_qcd(J2kEncoderContext *s, int compno)
+static int put_qcd(J2kEncoderContext *s, int compno)
 {
     int i;
     J2kCodingStyle *codsty = &s->codsty;
     J2kQuantStyle  *qntsty = &s->qntsty;
 
+    if (s->buf_end - s->buf < 6 + 3*(codsty->nreslevels - 1))
+        return -1;
+
     bytestream_put_be16(&s->buf, J2K_QCD);
     bytestream_put_be16(&s->buf, 4+3*(codsty->nreslevels-1));  // LQcd
     bytestream_put_byte(&s->buf, qntsty->nguardbits << 5);  // Sqcd
     for (i = 0; i < codsty->nreslevels * 3 - 2; i++)
         bytestream_put_byte(&s->buf, qntsty->expn[i] << 3);
+    return 0;
 }
 
 static uint8_t *put_sot(J2kEncoderContext *s, int tileno)
 {
     uint8_t *psotptr;
+
+    if (s->buf_end - s->buf < 12)
+        return -1;
+
     bytestream_put_be16(&s->buf, J2K_SOT);
     bytestream_put_be16(&s->buf, 10); // Lsot
     bytestream_put_be16(&s->buf, tileno); // Isot
@@ -567,7 +583,7 @@ static void putnumpasses(J2kEncoderConte
 }
 
 
-static void encode_packet(J2kEncoderContext *s, J2kResLevel *rlevel, int precno,
+static int encode_packet(J2kEncoderContext *s, J2kResLevel *rlevel, int precno,
                           uint8_t *expn, int numgbits)
 {
     int bandno, empty = 1;
@@ -614,6 +630,9 @@ static void encode_packet(J2kEncoderCont
                 int pad = 0, llen, length;
                 J2kCblk *cblk = band->cblk + yi * cblknw + xi;
 
+                if (s->buf_end - s->buf < 20) // approximately
+                    return -1;
+
                 // inclusion information
                 tag_tree_code(s, prec->cblkincl + pos, 1);
                 if (!cblk->ninclpasses)
@@ -645,16 +664,20 @@ static void encode_packet(J2kEncoderCont
             int xi;
             for (xi = prec->xi0; xi < prec->xi1; xi++){
                 J2kCblk *cblk = band->cblk + yi * cblknw + xi;
-                if (cblk->ninclpasses)
+                if (cblk->ninclpasses){
+                    if (s->buf_end - s->buf < cblk->passes[cblk->ninclpasses-1].rate)
+                        return -1;
                     bytestream_put_buffer(&s->buf, cblk->data, cblk->passes[cblk->ninclpasses-1].rate);
+                }
             }
         }
     }
+    return 0;
 }
 
-static void encode_packets(J2kEncoderContext *s, J2kTile *tile, int tileno)
+static int encode_packets(J2kEncoderContext *s, J2kTile *tile, int tileno)
 {
-    int compno, reslevelno;
+    int compno, reslevelno, ret;
     J2kCodingStyle *codsty = &s->codsty;
     J2kQuantStyle  *qntsty = &s->qntsty;
 
@@ -665,12 +688,14 @@ static void encode_packets(J2kEncoderCon
             int precno;
             J2kResLevel *reslevel = s->tile[tileno].comp[compno].reslevel + reslevelno;
             for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++){
-                encode_packet(s, reslevel, precno, qntsty->expn + (reslevelno ? 3*reslevelno-2 : 0),
-                              qntsty->nguardbits);
+                if (ret = encode_packet(s, reslevel, precno, qntsty->expn + (reslevelno ? 3*reslevelno-2 : 0),
+                              qntsty->nguardbits))
+                    return ret;
             }
         }
     }
     av_log(s->avctx, AV_LOG_DEBUG, "after tier2\n");
+    return 0;
 }
 
 static int getcut(J2kCblk *cblk, int64_t lambda, int dwt_norm)
@@ -721,16 +746,17 @@ static void truncpasses(J2kEncoderContex
     }
 }
 
-static void encode_tile(J2kEncoderContext *s, J2kTile *tile, int tileno)
+static int encode_tile(J2kEncoderContext *s, J2kTile *tile, int tileno)
 {
-    int compno, reslevelno, bandno;
+    int compno, reslevelno, bandno, ret;
     J2kT1Context t1;
     J2kCodingStyle *codsty = &s->codsty;
     for (compno = 0; compno < s->ncomponents; compno++){
         J2kComponent *comp = s->tile[tileno].comp + compno;
 
         av_log(s->avctx, AV_LOG_DEBUG,"dwt\n");
-        ff_dwt_encode(&comp->dwt, comp->data);
+        if (ret = ff_dwt_encode(&comp->dwt, comp->data))
+            return ret;
         av_log(s->avctx, AV_LOG_DEBUG,"after dwt -> tier1\n");
 
         for (reslevelno = 0; reslevelno < codsty->nreslevels; reslevelno++){
@@ -779,8 +805,10 @@ static void encode_tile(J2kEncoderContex
 
     av_log(s->avctx, AV_LOG_DEBUG, "rate control\n");
     truncpasses(s, tile);
-    encode_packets(s, tile, tileno);
+    if (ret = encode_packets(s, tile, tileno))
+        return ret;
     av_log(s->avctx, AV_LOG_DEBUG, "after rate control\n");
+    return 0;
 }
 
 void cleanup(J2kEncoderContext *s)
@@ -873,18 +901,29 @@ static int encode_frame(AVCodecContext *
     init_luts();
     av_log(s->avctx, AV_LOG_DEBUG, "after init\n");
 
+    if (s->buf_end - s->buf < 2)
+        return -1;
     bytestream_put_be16(&s->buf, J2K_SOC);
-    put_siz(s);
-    put_cod(s);
-    put_qcd(s, 0);
+    if (ret = put_siz(s))
+        return ret;
+    if (ret = put_cod(s))
+        return ret;
+    if (ret = put_qcd(s, 0))
+        return ret;
 
     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
         uint8_t *psotptr;
-        psotptr = put_sot(s, tileno);
+        if ((psotptr = put_sot(s, tileno)) < 0)
+            return psotptr;
+        if (s->buf_end - s->buf < 2)
+            return -1;
         bytestream_put_be16(&s->buf, J2K_SOD);
-        encode_tile(s, s->tile + tileno, tileno);
+        if (ret = encode_tile(s, s->tile + tileno, tileno))
+            return ret;
         bytestream_put_be32(&psotptr, s->buf - psotptr + 6);
     }
+    if (s->buf_end - s->buf < 2)
+        return -1;
     bytestream_put_be16(&s->buf, J2K_EOC);
 
     cleanup(s);



More information about the FFmpeg-soc mailing list