[FFmpeg-cvslog] ff_emulated_edge_mc: fix integer anomalies, fix out of array reads

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Fri Nov 16 20:57:35 2012 +0100| [2207ea44fb4fad4d47646a789bc244e3e84c1726] | committer: Michael Niedermayer

ff_emulated_edge_mc: fix integer anomalies, fix out of array reads

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=2207ea44fb4fad4d47646a789bc244e3e84c1726
---

 libavcodec/dsputil_template.c |    6 ++++--
 libavcodec/x86/dsputil_mmx.c  |    6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/libavcodec/dsputil_template.c b/libavcodec/dsputil_template.c
index 0d6e8df..eacf59b 100644
--- a/libavcodec/dsputil_template.c
+++ b/libavcodec/dsputil_template.c
@@ -131,10 +131,12 @@ void FUNC(ff_emulated_edge_mc)(uint8_t *buf, const uint8_t *src, int linesize, i
     int start_y, start_x, end_y, end_x;
 
     if(src_y>= h){
-        src+= (h-1-src_y)*linesize;
+        src-= src_y*linesize;
+        src+= (h-1)*linesize;
         src_y=h-1;
     }else if(src_y<=-block_h){
-        src+= (1-block_h-src_y)*linesize;
+        src-= src_y*linesize;
+        src+= (1-block_h)*linesize;
         src_y=1-block_h;
     }
     if(src_x>= w){
diff --git a/libavcodec/x86/dsputil_mmx.c b/libavcodec/x86/dsputil_mmx.c
index ff338b0..2d9de06 100644
--- a/libavcodec/x86/dsputil_mmx.c
+++ b/libavcodec/x86/dsputil_mmx.c
@@ -1878,10 +1878,12 @@ static av_always_inline void emulated_edge_mc(uint8_t *buf, const uint8_t *src,
     int start_y, start_x, end_y, end_x, src_y_add = 0;
 
     if (src_y >= h) {
-        src_y_add = h - 1 - src_y;
+        src -= src_y*linesize;
+        src_y_add = h - 1;
         src_y     = h - 1;
     } else if (src_y <= -block_h) {
-        src_y_add = 1 - block_h - src_y;
+        src -= src_y*linesize;
+        src_y_add = 1 - block_h;
         src_y     = 1 - block_h;
     }
     if (src_x >= w) {