[FFmpeg-devel] [PATCH] Fix unaligned fill_rectangle in rv34.c

[Reimar Döffinger]

Hello,
rv34.c uses fill_rectangle with only 4 bytes alignment, which causes
crashes on 64 bit architectures without unaligned read support like
Sparc/Solaris.
This patch hacks rectangle.h to support this kind of use - this
seemed the simplest, even though rather hackish, way to me.
Patch:
Index: libavcodec/rectangle.h
===================================================================
--- libavcodec/rectangle.h      (revision 19671)
+++ libavcodec/rectangle.h      (working copy)
@@ -47,7 +47,11 @@
     w      *= size;
     stride *= size;
 
+#ifdef FILL_RECT_ALIGN4
+    assert((((long)vp)&(FFMIN(w, 4)-1)) == 0);
+#else
     assert((((long)vp)&(FFMIN(w, STRIDE_ALIGN)-1)) == 0);
+#endif
     assert((stride&(w-1))==0);
     if(w==2){
         const uint16_t v= size==4 ? val : val*0x0101;
@@ -67,7 +71,7 @@
         *(uint32_t*)(p + 3*stride)= v;
     }else if(w==8){
     //gcc can't optimize 64bit math on x86_32
-#if HAVE_FAST_64BIT
+#if HAVE_FAST_64BIT && (!defined(FILL_RECT_ALIGN4) || HAVE_FAST_UNALIGNED)
         const uint64_t v= val*0x0100000001ULL;
         *(uint64_t*)(p + 0*stride)= v;
         if(h==1) return;
Index: libavcodec/rv34.c
===================================================================
--- libavcodec/rv34.c   (revision 19671)
+++ libavcodec/rv34.c   (working copy)
@@ -29,6 +29,7 @@
 #include "mpegvideo.h"
 #include "golomb.h"
 #include "mathops.h"
+#define FILL_RECT_ALIGN4 1
 #include "rectangle.h"
 
 #include "rv34vlc.h"