[FFmpeg-cvslog] r18999 - trunk/libavcodec/x86/h264dsp_mmx.c
astrange
subversion
Sun May 31 00:19:14 CEST 2009
Author: astrange
Date: Sun May 31 00:19:14 2009
New Revision: 18999
Log:
H264: Fix out of bounds reads in SSSE3 MC
Reading above src[-2] isn't safe, so move loads and palignr ahead
3 pixels to load starting at the first pixel actually used.
Fixes issue941.
Modified:
trunk/libavcodec/x86/h264dsp_mmx.c
Modified: trunk/libavcodec/x86/h264dsp_mmx.c
==============================================================================
--- trunk/libavcodec/x86/h264dsp_mmx.c Sat May 30 20:51:15 2009 (r18998)
+++ trunk/libavcodec/x86/h264dsp_mmx.c Sun May 31 00:19:14 2009 (r18999)
@@ -1497,8 +1497,8 @@ static av_noinline void OPNAME ## h264_q
"movdqa %6, %%xmm14 \n\t"\
"movdqa %7, %%xmm13 \n\t"\
"1: \n\t"\
- "lddqu 3(%0), %%xmm1 \n\t"\
- "lddqu -5(%0), %%xmm7 \n\t"\
+ "lddqu 6(%0), %%xmm1 \n\t"\
+ "lddqu -2(%0), %%xmm7 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm15, %%xmm1 \n\t"\
"punpcklbw %%xmm15, %%xmm0 \n\t"\
@@ -1509,20 +1509,20 @@ static av_noinline void OPNAME ## h264_q
"movdqa %%xmm0, %%xmm8 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm0, %%xmm9 \n\t"\
- "movdqa %%xmm1, %%xmm5 \n\t"\
- "movdqa %%xmm0, %%xmm10 \n\t"\
- "palignr $6, %%xmm0, %%xmm5 \n\t"\
- "palignr $6, %%xmm7, %%xmm10\n\t"\
- "palignr $8, %%xmm0, %%xmm4 \n\t"\
- "palignr $8, %%xmm7, %%xmm9 \n\t"\
- "palignr $10,%%xmm0, %%xmm3 \n\t"\
- "palignr $10,%%xmm7, %%xmm8 \n\t"\
- "paddw %%xmm1, %%xmm5 \n\t"\
- "paddw %%xmm0, %%xmm10 \n\t"\
- "palignr $12,%%xmm0, %%xmm2 \n\t"\
- "palignr $12,%%xmm7, %%xmm6 \n\t"\
- "palignr $14,%%xmm0, %%xmm1 \n\t"\
- "palignr $14,%%xmm7, %%xmm0 \n\t"\
+ "movdqa %%xmm0, %%xmm12 \n\t"\
+ "movdqa %%xmm1, %%xmm11 \n\t"\
+ "palignr $10,%%xmm0, %%xmm11\n\t"\
+ "palignr $10,%%xmm7, %%xmm12\n\t"\
+ "palignr $2, %%xmm0, %%xmm4 \n\t"\
+ "palignr $2, %%xmm7, %%xmm9 \n\t"\
+ "palignr $4, %%xmm0, %%xmm3 \n\t"\
+ "palignr $4, %%xmm7, %%xmm8 \n\t"\
+ "palignr $6, %%xmm0, %%xmm2 \n\t"\
+ "palignr $6, %%xmm7, %%xmm6 \n\t"\
+ "paddw %%xmm0 ,%%xmm11 \n\t"\
+ "palignr $8, %%xmm0, %%xmm1 \n\t"\
+ "palignr $8, %%xmm7, %%xmm0 \n\t"\
+ "paddw %%xmm12,%%xmm7 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm8, %%xmm6 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\
@@ -1531,13 +1531,13 @@ static av_noinline void OPNAME ## h264_q
"psllw $2, %%xmm6 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\
"psubw %%xmm0, %%xmm6 \n\t"\
- "paddw %%xmm13,%%xmm5 \n\t"\
- "paddw %%xmm13,%%xmm10 \n\t"\
+ "paddw %%xmm13,%%xmm11 \n\t"\
+ "paddw %%xmm13,%%xmm7 \n\t"\
"pmullw %%xmm14,%%xmm2 \n\t"\
"pmullw %%xmm14,%%xmm6 \n\t"\
"lddqu (%2), %%xmm3 \n\t"\
- "paddw %%xmm5, %%xmm2 \n\t"\
- "paddw %%xmm10,%%xmm6 \n\t"\
+ "paddw %%xmm11,%%xmm2 \n\t"\
+ "paddw %%xmm7, %%xmm6 \n\t"\
"psraw $5, %%xmm2 \n\t"\
"psraw $5, %%xmm6 \n\t"\
"packuswb %%xmm2,%%xmm6 \n\t"\
@@ -1577,7 +1577,7 @@ static av_noinline void OPNAME ## h264_q
);\
do{\
__asm__ volatile(\
- "lddqu -5(%0), %%xmm1 \n\t"\
+ "lddqu -2(%0), %%xmm1 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm7, %%xmm1 \n\t"\
"punpcklbw %%xmm7, %%xmm0 \n\t"\
@@ -1585,20 +1585,20 @@ static av_noinline void OPNAME ## h264_q
"movdqa %%xmm1, %%xmm3 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm1, %%xmm5 \n\t"\
- "palignr $6, %%xmm0, %%xmm5 \n\t"\
- "palignr $8, %%xmm0, %%xmm4 \n\t"\
- "palignr $10,%%xmm0, %%xmm3 \n\t"\
- "paddw %%xmm1, %%xmm5 \n\t"\
- "palignr $12,%%xmm0, %%xmm2 \n\t"\
- "palignr $14,%%xmm0, %%xmm1 \n\t"\
+ "palignr $2, %%xmm0, %%xmm4 \n\t"\
+ "palignr $4, %%xmm0, %%xmm3 \n\t"\
+ "palignr $6, %%xmm0, %%xmm2 \n\t"\
+ "palignr $8, %%xmm0, %%xmm1 \n\t"\
+ "palignr $10,%%xmm0, %%xmm5 \n\t"\
+ "paddw %%xmm5, %%xmm0 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\
"psllw $2, %%xmm2 \n\t"\
"movq (%2), %%xmm3 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\
- "paddw %5, %%xmm5 \n\t"\
+ "paddw %5, %%xmm0 \n\t"\
"pmullw %%xmm6, %%xmm2 \n\t"\
- "paddw %%xmm5, %%xmm2 \n\t"\
+ "paddw %%xmm0, %%xmm2 \n\t"\
"psraw $5, %%xmm2 \n\t"\
"packuswb %%xmm2, %%xmm2 \n\t"\
"pavgb %%xmm3, %%xmm2 \n\t"\
@@ -1621,7 +1621,7 @@ static av_noinline void OPNAME ## h264_q
"pxor %%xmm7, %%xmm7 \n\t"\
"movdqa %5, %%xmm6 \n\t"\
"1: \n\t"\
- "lddqu -5(%0), %%xmm1 \n\t"\
+ "lddqu -2(%0), %%xmm1 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm7, %%xmm1 \n\t"\
"punpcklbw %%xmm7, %%xmm0 \n\t"\
@@ -1629,19 +1629,19 @@ static av_noinline void OPNAME ## h264_q
"movdqa %%xmm1, %%xmm3 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm1, %%xmm5 \n\t"\
- "palignr $6, %%xmm0, %%xmm5 \n\t"\
- "palignr $8, %%xmm0, %%xmm4 \n\t"\
- "palignr $10,%%xmm0, %%xmm3 \n\t"\
- "paddw %%xmm1, %%xmm5 \n\t"\
- "palignr $12,%%xmm0, %%xmm2 \n\t"\
- "palignr $14,%%xmm0, %%xmm1 \n\t"\
+ "palignr $2, %%xmm0, %%xmm4 \n\t"\
+ "palignr $4, %%xmm0, %%xmm3 \n\t"\
+ "palignr $6, %%xmm0, %%xmm2 \n\t"\
+ "palignr $8, %%xmm0, %%xmm1 \n\t"\
+ "palignr $10,%%xmm0, %%xmm5 \n\t"\
+ "paddw %%xmm5, %%xmm0 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\
"psllw $2, %%xmm2 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\
- "paddw %6, %%xmm5 \n\t"\
+ "paddw %6, %%xmm0 \n\t"\
"pmullw %%xmm6, %%xmm2 \n\t"\
- "paddw %%xmm5, %%xmm2 \n\t"\
+ "paddw %%xmm0, %%xmm2 \n\t"\
"psraw $5, %%xmm2 \n\t"\
"packuswb %%xmm2, %%xmm2 \n\t"\
OP(%%xmm2, (%1), %%xmm4, q)\
More information about the ffmpeg-cvslog
mailing list