[FFmpeg-devel] [PATCH]The ljpeg encoder does not support transparency

[Carl Eugen Hoyos]

Hi!

Users may think attached patch introduce a regression, but since the ljpeg 
encoder never supported encoding transparency, I consider it a fix.

An alternative would be to continue to claim to encode BGRA and add BGR0 and 
BGR24. Since there is indication that BGRA is actually possible (see ticket 
#896), and we just had a version bump, I prefer this solution.

(There is an orthogonal problem that the ljpeg encoder does not tell which 
colour spaces it supports.)

Please review, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/ljpegenc.c b/libavcodec/ljpegenc.c
index f8178d5..0fcb971 100644
--- a/libavcodec/ljpegenc.c
+++ b/libavcodec/ljpegenc.c
@@ -56,7 +56,7 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
 
     s->header_bits= put_bits_count(&s->pb);
 
-    if(avctx->pix_fmt == PIX_FMT_BGRA){
+    if(avctx->pix_fmt == PIX_FMT_BGR0 || avctx->pix_fmt == PIX_FMT_BGR24){
         int x, y, i;
         const int linesize= p->linesize[0];
         uint16_t (*buffer)[4]= (void *) s->rd_scratchpad;
@@ -79,9 +79,15 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
                 top[i]= left[i]= topleft[i]= buffer[0][i];
             }
             for(x = 0; x < width; x++) {
+                if(avctx->pix_fmt == PIX_FMT_BGR24){ 
+                    buffer[x][1] = ptr[3*x+0] - ptr[3*x+1] + 0x100;
+                    buffer[x][2] = ptr[3*x+2] - ptr[3*x+1] + 0x100;
+                    buffer[x][0] = (ptr[3*x+0] + 2*ptr[3*x+1] + ptr[3*x+2])>>2;
+                }else{
                 buffer[x][1] = ptr[4*x+0] - ptr[4*x+1] + 0x100;
                 buffer[x][2] = ptr[4*x+2] - ptr[4*x+1] + 0x100;
                 buffer[x][0] = (ptr[4*x+0] + 2*ptr[4*x+1] + ptr[4*x+2])>>2;
+                }
 
                 for(i=0;i<3;i++) {
                     int pred, diff;
diff --git a/libavcodec/mjpegenc.c b/libavcodec/mjpegenc.c
index 4e77e0d..a47b5dc 100644
--- a/libavcodec/mjpegenc.c
+++ b/libavcodec/mjpegenc.c
@@ -214,7 +214,7 @@ void ff_mjpeg_encode_picture_header(MpegEncContext *s)
     }
 
     put_bits(&s->pb, 16, 17);
-    if(lossless && s->avctx->pix_fmt == PIX_FMT_BGRA)
+    if(lossless && (s->avctx->pix_fmt == PIX_FMT_BGR0 || s->avctx->pix_fmt == PIX_FMT_BGR24))
         put_bits(&s->pb, 8, 9); /* 9 bits/component RCT */
     else
         put_bits(&s->pb, 8, 8); /* 8 bits/component */
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 4eaad5d..4d80e51 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -295,7 +295,8 @@ av_cold int MPV_encode_init(AVCodecContext *avctx)
         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
             avctx->pix_fmt != PIX_FMT_YUVJ444P &&
-            avctx->pix_fmt != PIX_FMT_BGRA     &&
+            avctx->pix_fmt != PIX_FMT_BGR0     &&
+            avctx->pix_fmt != PIX_FMT_BGR24    &&
             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
               avctx->pix_fmt != PIX_FMT_YUV422P &&
               avctx->pix_fmt != PIX_FMT_YUV444P) ||
@@ -625,7 +626,8 @@ av_cold int MPV_encode_init(AVCodecContext *avctx)
     case CODEC_ID_AMV:
         s->out_format = FMT_MJPEG;
         s->intra_only = 1; /* force intra only for jpeg */
-        if (avctx->codec->id == CODEC_ID_LJPEG && avctx->pix_fmt   == PIX_FMT_BGRA) {
+        if (avctx->codec->id == CODEC_ID_LJPEG
+            && (avctx->pix_fmt == PIX_FMT_BGR0 || s->avctx->pix_fmt == PIX_FMT_BGR24)) {
             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;