[FFmpeg-cvslog] avcodec/x86/lossless_videoencdsp: Fix handling of small widths

[Michael Niedermayer]

ffmpeg | branch: release/3.3 | Michael Niedermayer <michael at niedermayer.cc> | Sat Sep 30 00:20:09 2017 +0200| [0a231e7dd32bdea4b2fc1c48040047986d1d4925] | committer: Michael Niedermayer

avcodec/x86/lossless_videoencdsp: Fix handling of small widths

Fixes out of array access
Fixes: crash-huf.avi

Regression since: 6b41b4414934cc930468ccd5db598dd6ef643987

This could also be fixed by adding checks in the C code that calls the dsp

Found-by: Zhibin Hu and 连一汉 <lianyihan at 360.cn>
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit df62b70de8aaa285168e72fe8f6e740843ca91fa)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=0a231e7dd32bdea4b2fc1c48040047986d1d4925
---

 libavcodec/x86/lossless_videoencdsp.asm | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/libavcodec/x86/lossless_videoencdsp.asm b/libavcodec/x86/lossless_videoencdsp.asm
index 63fd72174a..41ab23eae5 100644
--- a/libavcodec/x86/lossless_videoencdsp.asm
+++ b/libavcodec/x86/lossless_videoencdsp.asm
@@ -42,10 +42,11 @@ cglobal diff_bytes, 4,5,2, dst, src1, src2, w
 %define i t0q
 %endmacro
 
-; label to jump to if w < regsize
-%macro DIFF_BYTES_LOOP_PREP 1
+; labels to jump to if w < regsize and w < 0
+%macro DIFF_BYTES_LOOP_PREP 2
     mov                i, wq
     and                i, -2 * regsize
+        js            %2
         jz            %1
     add             dstq, i
     add            src1q, i
@@ -87,7 +88,7 @@ cglobal diff_bytes, 4,5,2, dst, src1, src2, w
 %if mmsize > 16
     ; fall back to narrower xmm
     %define regsize mmsize / 2
-    DIFF_BYTES_LOOP_PREP .setup_loop_gpr_aa
+    DIFF_BYTES_LOOP_PREP .setup_loop_gpr_aa, .end_aa
 .loop2_%1%2:
     DIFF_BYTES_LOOP_CORE %1, %2, xm0, xm1
     add                i, 2 * regsize
@@ -114,7 +115,7 @@ cglobal diff_bytes, 4,5,2, dst, src1, src2, w
 INIT_MMX mmx
 DIFF_BYTES_PROLOGUE
     %define regsize mmsize
-    DIFF_BYTES_LOOP_PREP .skip_main_aa
+    DIFF_BYTES_LOOP_PREP .skip_main_aa, .end_aa
     DIFF_BYTES_BODY    a, a
 %undef i
 %endif
@@ -122,7 +123,7 @@ DIFF_BYTES_PROLOGUE
 INIT_XMM sse2
 DIFF_BYTES_PROLOGUE
     %define regsize mmsize
-    DIFF_BYTES_LOOP_PREP .skip_main_aa
+    DIFF_BYTES_LOOP_PREP .skip_main_aa, .end_aa
     test            dstq, regsize - 1
         jnz     .loop_uu
     test           src1q, regsize - 1
@@ -138,7 +139,7 @@ DIFF_BYTES_PROLOGUE
     %define regsize mmsize
     ; Directly using unaligned SSE2 version is marginally faster than
     ; branching based on arguments.
-    DIFF_BYTES_LOOP_PREP .skip_main_uu
+    DIFF_BYTES_LOOP_PREP .skip_main_uu, .end_uu
     test            dstq, regsize - 1
         jnz     .loop_uu
     test           src1q, regsize - 1