[FFmpeg-devel] [PATCH] Fix nonsense non-mod16 AMV flipping code.

[Reimar Döffinger]

It is obviously nonsense since it produces wrong results
or even crashes (crashes should be encode-only though).
Fixes trac issue #1092.

Signed-off-by: Reimar Döffinger <Reimar.Doeffinger at gmx.de>
---
 libavcodec/mjpegdec.c      |    3 +--
 libavcodec/mjpegenc.c      |    2 +-
 libavcodec/mpegvideo_enc.c |    5 -----
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index a841384..fe54b45 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -972,8 +972,7 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
         s->coefs_finished[c] |= 1;
         if (s->flipped) {
             // picture should be flipped upside-down for this codec
-            int offset = (linesize[c] * (s->v_scount[i] *
-                         (8 * s->mb_height - ((s->height / s->v_max) & 7)) - 1));
+            int offset = linesize[c] * (s->v_scount[c] * s->height / s->v_max - 1);
             data[c]           += offset;
             reference_data[c] += offset;
             linesize[c]       *= -1;
diff --git a/libavcodec/mjpegenc.c b/libavcodec/mjpegenc.c
index 9326d4d..4788c47 100644
--- a/libavcodec/mjpegenc.c
+++ b/libavcodec/mjpegenc.c
@@ -466,7 +466,7 @@ static int amv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
 
     //picture should be flipped upside-down
     for(i=0; i < 3; i++) {
-        pic.data[i] += (pic.linesize[i] * (s->mjpeg_vsample[i] * (8 * s->mb_height -((s->height/V_MAX)&7)) - 1 ));
+        pic.data[i] += pic.linesize[i] * (s->mjpeg_vsample[i] * s->height / V_MAX - 1);
         pic.linesize[i] *= -1;
     }
     return ff_MPV_encode_picture(avctx, pkt, &pic, got_packet);
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 6d89cfe..bb4be28 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -1012,11 +1012,6 @@ static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg)
                 uint8_t *src = pic_arg->data[i];
                 uint8_t *dst = pic->data[i];
 
-                if(s->codec_id == CODEC_ID_AMV){
-                    av_assert0(!(s->avctx->flags & CODEC_FLAG_EMU_EDGE));
-                    h= ((s->height+15)/16*16)>>v_shift;
-                }
-
                 if (!s->avctx->rc_buffer_size)
                     dst += INPLACE_OFFSET;
 
-- 
1.7.10