[FFmpeg-devel] [PATCH]lavc/8bps: Fix 32bit output of 24bit video

[Carl Eugen Hoyos]

Hi!

Attached patch fixes a long-time regression.

Please comment, Carl Eugen
-------------- next part --------------
From 2b991a66c4bf14774210e2c7bfb0bb5f0e4f1b8f Mon Sep 17 00:00:00 2001
From: Carl Eugen Hoyos <cehoyos at ag.or.at>
Date: Sun, 25 Sep 2016 14:27:43 +0200
Subject: [PATCH] lavc/8bps: Fix 32bit output of 24bit video.

Regression since / partial revert of ba3bb53b
---
 libavcodec/8bps.c |   13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/libavcodec/8bps.c b/libavcodec/8bps.c
index 46344e0..49fd445 100644
--- a/libavcodec/8bps.c
+++ b/libavcodec/8bps.c
@@ -41,7 +41,7 @@
 
 
 static const enum AVPixelFormat pixfmt_rgb24[] = {
-    AV_PIX_FMT_BGR24, AV_PIX_FMT_RGB32, AV_PIX_FMT_NONE };
+    AV_PIX_FMT_BGR24, AV_PIX_FMT_0RGB32, AV_PIX_FMT_NONE };
 
 typedef struct EightBpsContext {
     AVCodecContext *avctx;
@@ -65,6 +65,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
     unsigned int dlen, p, row;
     const unsigned char *lp, *dp, *ep;
     unsigned char count;
+    unsigned int px_inc;
     unsigned int planes     = c->planes;
     unsigned char *planemap = c->planemap;
     int ret;
@@ -77,6 +78,8 @@ static int decode_frame(AVCodecContext *avctx, void *data,
     /* Set data pointer after line lengths */
     dp = encoded + planes * (height << 1);
 
+    px_inc = planes + (avctx->pix_fmt == AV_PIX_FMT_0RGB32);
+
     for (p = 0; p < planes; p++) {
         /* Lines length pointer for this plane */
         lp = encoded + p * (height << 1);
@@ -95,21 +98,21 @@ static int decode_frame(AVCodecContext *avctx, void *data,
                 if ((count = *dp++) <= 127) {
                     count++;
                     dlen -= count + 1;
-                    if (pixptr_end - pixptr < count * planes)
+                    if (pixptr_end - pixptr < count * px_inc)
                         break;
                     if (ep - dp < count)
                         return AVERROR_INVALIDDATA;
                     while (count--) {
                         *pixptr = *dp++;
-                        pixptr += planes;
+                        pixptr += px_inc;
                     }
                 } else {
                     count = 257 - count;
-                    if (pixptr_end - pixptr < count * planes)
+                    if (pixptr_end - pixptr < count * px_inc)
                         break;
                     while (count--) {
                         *pixptr = *dp;
-                        pixptr += planes;
+                        pixptr += px_inc;
                     }
                     dp++;
                     dlen -= 2;
-- 
1.7.10.4