[FFmpeg-cvslog] videodsp: don't overread edges in vfix3 emu_edge.

[Ronald S. Bultje]

ffmpeg | branch: release/2.3 | Ronald S. Bultje <rsbultje at gmail.com> | Fri Oct 23 11:11:53 2015 -0400| [f6006295c037e8ac2706874ee7b4b97bdda5c387] | committer: Carl Eugen Hoyos

videodsp: don't overread edges in vfix3 emu_edge.

Fixes trac ticket 3226. Also see Andreas' analysis in
https://bugs.debian.org/801745, which was very helpful.
(cherry picked from commit 52f84d82bdf1851ecfcc412c1719e5f6f3396209)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f6006295c037e8ac2706874ee7b4b97bdda5c387
---

 libavcodec/x86/videodsp.asm |   15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/libavcodec/x86/videodsp.asm b/libavcodec/x86/videodsp.asm
index 1ac0257..77189fa 100644
--- a/libavcodec/x86/videodsp.asm
+++ b/libavcodec/x86/videodsp.asm
@@ -185,8 +185,12 @@ hvar_fn
 %elif (%2-%%off) == 2
     mov            valw, [srcq+%2-2]
 %elifidn %1, body
-    mov            vald, [srcq+%2-3]
-%else
+    mov            valb, [srcq+%2-1]
+    sal            vald, 16
+    mov            valw, [srcq+%2-3]
+%elifidn %1, bottom
+    movd mm %+ %%mmx_idx, [srcq+%2-4]
+%else ; top
     movd mm %+ %%mmx_idx, [srcq+%2-3]
 %endif
 %endif ; (%2-%%off) >= 1
@@ -242,12 +246,15 @@ hvar_fn
     mov     [dstq+%2-2], valw
 %elifidn %1, body
     mov     [dstq+%2-3], valw
-    shr            vald, 16
+    sar            vald, 16
     mov     [dstq+%2-1], valb
 %else
     movd           vald, mm %+ %%mmx_idx
+%ifidn %1, bottom
+    sar            vald, 8
+%endif
     mov     [dstq+%2-3], valw
-    shr            vald, 16
+    sar            vald, 16
     mov     [dstq+%2-1], valb
 %endif
 %endif ; (%2-%%off) >= 1