[FFmpeg-cvslog] rtmp: fix multiple broken overflow checks

[Xi Wang]

ffmpeg | branch: release/1.1 | Xi Wang <xi.wang at gmail.com> | Tue Jan 22 17:49:29 2013 -0500| [b54c155f5b18b0ce8e47e73d27f657e592f9072d] | committer: Michael Niedermayer

rtmp: fix multiple broken overflow checks

Sanity checks like `data + size >= data_end || data + size < data' are
broken, because `data + size < data' assumes pointer overflow, which is
undefined behavior in C.  Many compilers such as gcc/clang optimize such
checks away.

Use `size < 0 || size >= data_end - data' instead.

Signed-off-by: Xi Wang <xi.wang at gmail.com>
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>
(cherry picked from commit 902cfe2f74d777a7dc20ac68f2393b9f84b790c1)

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=b54c155f5b18b0ce8e47e73d27f657e592f9072d
---

 libavformat/rtmppkt.c |   12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libavformat/rtmppkt.c b/libavformat/rtmppkt.c
index 04724f7..078a832 100644
--- a/libavformat/rtmppkt.c
+++ b/libavformat/rtmppkt.c
@@ -356,11 +356,11 @@ int ff_amf_tag_size(const uint8_t *data, const uint8_t *data_end)
                 data++;
                 break;
             }
-            if (data + size >= data_end || data + size < data)
+            if (size < 0 || size >= data_end - data)
                 return -1;
             data += size;
             t = ff_amf_tag_size(data, data_end);
-            if (t < 0 || data + t >= data_end)
+            if (t < 0 || t >= data_end - data)
                 return -1;
             data += t;
         }
@@ -389,7 +389,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
         int size = bytestream_get_be16(&data);
         if (!size)
             break;
-        if (data + size >= data_end || data + size < data)
+        if (size < 0 || size >= data_end - data)
             return -1;
         data += size;
         if (size == namelen && !memcmp(data-size, name, namelen)) {
@@ -410,7 +410,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
             return 0;
         }
         len = ff_amf_tag_size(data, data_end);
-        if (len < 0 || data + len >= data_end || data + len < data)
+        if (len < 0 || len >= data_end - data)
             return -1;
         data += len;
     }
@@ -481,13 +481,13 @@ static void ff_amf_tag_contents(void *ctx, const uint8_t *data, const uint8_t *d
                 data++;
                 break;
             }
-            if (data + size >= data_end || data + size < data)
+            if (size < 0 || size >= data_end - data)
                 return;
             data += size;
             av_log(ctx, AV_LOG_DEBUG, "  %s: ", buf);
             ff_amf_tag_contents(ctx, data, data_end);
             t = ff_amf_tag_size(data, data_end);
-            if (t < 0 || data + t >= data_end)
+            if (t < 0 || t >= data_end - data)
                 return;
             data += t;
         }