[FFmpeg-cvslog] oma: check geob tag boundary

[Luca Barbato]

ffmpeg | branch: release/1.1 | Luca Barbato <lu_zero at gentoo.org> | Wed Apr 17 21:07:09 2013 +0200| [97e6099c0c702795a6c953b75920968fc256b6d2] | committer: Luca Barbato

oma: check geob tag boundary

Prevent read after buffer boundary on corrupted tag.

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable at libav.org
(cherry picked from commit 9d0b45ade864f3d2ccd8610149fe1fff53c4e937)
Signed-off-by: Luca Barbato <lu_zero at gentoo.org>

Conflicts:
	libavformat/omadec.c

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=97e6099c0c702795a6c953b75920968fc256b6d2
---

 libavformat/omadec.c |   34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/libavformat/omadec.c b/libavformat/omadec.c
index db599f5..07d8b2c 100644
--- a/libavformat/omadec.c
+++ b/libavformat/omadec.c
@@ -113,13 +113,18 @@ static int kset(AVFormatContext *s, const uint8_t *r_val, const uint8_t *n_val,
     return 0;
 }
 
-static int rprobe(AVFormatContext *s, uint8_t *enc_header, const uint8_t *r_val)
+#define OMA_RPROBE_M_VAL 48 + 1
+
+static int rprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
+                  const uint8_t *r_val)
 {
     OMAContext *oc = s->priv_data;
     unsigned int pos;
     struct AVDES av_des;
 
-    if (!enc_header || !r_val)
+    if (!enc_header || !r_val ||
+        size < OMA_ENC_HEADER_SIZE + oc->k_size + oc->e_size + oc->i_size ||
+        size < OMA_RPROBE_M_VAL)
         return -1;
 
     /* m_val */
@@ -140,35 +145,41 @@ static int rprobe(AVFormatContext *s, uint8_t *enc_header, const uint8_t *r_val)
     return memcmp(&enc_header[pos], oc->sm_val, 8) ? -1 : 0;
 }
 
-static int nprobe(AVFormatContext *s, uint8_t *enc_header, int size, const uint8_t *n_val)
+static int nprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
+                  const uint8_t *n_val)
 {
     OMAContext *oc = s->priv_data;
-    uint32_t pos, taglen, datalen;
+    uint64_t pos;
+    uint32_t taglen, datalen;
     struct AVDES av_des;
 
-    if (!enc_header || !n_val)
+    if (!enc_header || !n_val ||
+        size < OMA_ENC_HEADER_SIZE + oc->k_size + 4)
         return -1;
 
     pos = OMA_ENC_HEADER_SIZE + oc->k_size;
     if (!memcmp(&enc_header[pos], "EKB ", 4))
         pos += 32;
 
+    if (size < pos + 44)
+        return -1;
+
     if (AV_RB32(&enc_header[pos]) != oc->rid)
         av_log(s, AV_LOG_DEBUG, "Mismatching RID\n");
 
     taglen = AV_RB32(&enc_header[pos+32]);
     datalen = AV_RB32(&enc_header[pos+36]) >> 4;
 
-    if(taglen + (((uint64_t)datalen)<<4) + 44 > size)
-        return -1;
-
     pos += 44 + taglen;
 
+    if (datalen << 4 > size - pos)
+        return -1;
+
     av_des_init(&av_des, n_val, 192, 1);
     while (datalen-- > 0) {
         av_des_crypt(&av_des, oc->r_val, &enc_header[pos], 2, NULL, 1);
         kset(s, oc->r_val, NULL, 16);
-        if (!rprobe(s, enc_header, oc->r_val))
+        if (!rprobe(s, enc_header, size, oc->r_val))
             return 0;
         pos += 16;
     }
@@ -231,7 +242,7 @@ static int decrypt_init(AVFormatContext *s, ID3v2ExtraMeta *em, uint8_t *header)
         kset(s, s->key, s->key, s->keylen);
     }
     if (!memcmp(oc->r_val, (const uint8_t[8]){0}, 8) ||
-        rprobe(s, gdata, oc->r_val) < 0 &&
+        rprobe(s, gdata, geob->datasize, oc->r_val) < 0 &&
         nprobe(s, gdata, geob->datasize, oc->n_val) < 0) {
         int i;
         for (i = 0; i < FF_ARRAY_ELEMS(leaf_table); i += 2) {
@@ -239,7 +250,8 @@ static int decrypt_init(AVFormatContext *s, ID3v2ExtraMeta *em, uint8_t *header)
             AV_WL64(buf, leaf_table[i]);
             AV_WL64(&buf[8], leaf_table[i+1]);
             kset(s, buf, buf, 16);
-            if (!rprobe(s, gdata, oc->r_val) || !nprobe(s, gdata, geob->datasize, oc->n_val))
+            if (!rprobe(s, gdata, geob->datasize, oc->r_val) ||
+                !nprobe(s, gdata, geob->datasize, oc->n_val))
                 break;
         }
         if (i >= sizeof(leaf_table)) {