[FFmpeg-cvslog] avcodec/cabac: support UNCHECKED_BITSTREAM_READER = 0

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Thu Oct 31 10:54:46 2013 +0100| [fa6fa2162b730336fc1d6ee0d547dcc81f4afbad] | committer: Michael Niedermayer

avcodec/cabac: support UNCHECKED_BITSTREAM_READER = 0

Fixes overreads in HEVC
Fixes Ticket3070
Also fixed remaining issues from Ticket3075 and Ticket3076

Some lines of code taken from  0c5f839693da2276c2da23400f67a67be4ea0af1:libavcodec/x86/cabac.h
and                            0c5f839693da2276c2da23400f67a67be4ea0af1:libavcodec/cabac_functions.h

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=fa6fa2162b730336fc1d6ee0d547dcc81f4afbad
---

 libavcodec/cabac_functions.h |   10 ++++++++--
 libavcodec/x86/cabac.h       |   23 +++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/libavcodec/cabac_functions.h b/libavcodec/cabac_functions.h
index b84258b..7e22064 100644
--- a/libavcodec/cabac_functions.h
+++ b/libavcodec/cabac_functions.h
@@ -49,7 +49,10 @@ static void refill(CABACContext *c){
         c->low+= c->bytestream[0]<<1;
 #endif
     c->low -= CABAC_MASK;
-    c->bytestream += CABAC_BITS / 8;
+#if !UNCHECKED_BITSTREAM_READER
+    if (c->bytestream < c->bytestream_end)
+#endif
+        c->bytestream += CABAC_BITS / 8;
 }
 
 static inline void renorm_cabac_decoder_once(CABACContext *c){
@@ -76,7 +79,10 @@ static void refill2(CABACContext *c){
 #endif
 
     c->low += x<<i;
-    c->bytestream += CABAC_BITS/8;
+#if !UNCHECKED_BITSTREAM_READER
+    if (c->bytestream < c->bytestream_end)
+#endif
+        c->bytestream += CABAC_BITS/8;
 }
 
 static av_always_inline int get_cabac_inline(CABACContext *c, uint8_t * const state){
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 7d431e5..558d287 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -36,6 +36,18 @@
 
 #if HAVE_INLINE_ASM
 
+#ifndef UNCHECKED_BITSTREAM_READER
+#define UNCHECKED_BITSTREAM_READER !CONFIG_SAFE_BITSTREAM_READER
+#endif
+
+#if UNCHECKED_BITSTREAM_READER
+#define END_CHECK(end) ""
+#else
+#define END_CHECK(end) \
+        "cmp    "end"       , %%"REG_c"                                 \n\t"\
+        "jge    1f                                                      \n\t"
+#endif
+
 #ifdef BROKEN_RELOCATIONS
 #define TABLES_ARG , "r"(tables)
 
@@ -80,7 +92,9 @@
         "test   "lowword"   , "lowword"                                 \n\t"\
         "jnz    2f                                                      \n\t"\
         "mov    "byte"      , %%"REG_c"                                 \n\t"\
+        END_CHECK(end)\
         "add"OPSIZE" $2     , "byte"                                    \n\t"\
+        "1:                                                             \n\t"\
         "movzwl (%%"REG_c") , "tmp"                                     \n\t"\
         "lea    -1("low")   , %%ecx                                     \n\t"\
         "xor    "low"       , %%ecx                                     \n\t"\
@@ -139,7 +153,9 @@
         "test   "lowword"   , "lowword"                                 \n\t"\
         " jnz   2f                                                      \n\t"\
         "mov    "byte"      , %%"REG_c"                                 \n\t"\
+        END_CHECK(end)\
         "add"OPSIZE" $2     , "byte"                                    \n\t"\
+        "1:                                                             \n\t"\
         "movzwl (%%"REG_c")     , "tmp"                                 \n\t"\
         "lea    -1("low")   , %%ecx                                     \n\t"\
         "xor    "low"       , %%ecx                                     \n\t"\
@@ -214,9 +230,16 @@ static av_always_inline int get_cabac_bypass_sign_x86(CABACContext *c, int val)
         "movzwl         (%1), %%edx     \n\t"
         "bswap         %%edx            \n\t"
         "shrl            $15, %%edx     \n\t"
+#if UNCHECKED_BITSTREAM_READER
         "add              $2, %1        \n\t"
         "addl          %%edx, %%eax     \n\t"
         "mov              %1, %c4(%2)   \n\t"
+#else
+        "addl          %%edx, %%eax     \n\t"
+        "cmp         %c5(%2), %1        \n\t"
+        "jge              1f            \n\t"
+        "add"OPSIZE"      $2, %c4(%2)   \n\t"
+#endif
         "1:                             \n\t"
         "movl          %%eax, %c3(%2)   \n\t"