[FFmpeg-devel] [PATCH]Fix palette calculation in some decoders

Carl Eugen Hoyos cehoyos at ag.or.at
Fri Nov 11 22:44:18 CET 2011


Hi!

If I am not missing something, FFmpeg currently happily assumes complete 
transparency for most PAL8 codecs.

I did not find the palette code in vmnc.c, I am unsure if ff_cga_palette[] is 
correct (black is always transparent?).

Please comment, I did not test the ones with "AV_RB24()" and 
"bytestream_get_be24()".

Thank you, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/avs.c b/libavcodec/avs.c
index cf965b3..0317c79 100644
--- a/libavcodec/avs.c
+++ b/libavcodec/avs.c
@@ -85,8 +85,10 @@ avs_decode_frame(AVCodecContext * avctx,
         if (first >= 256 || last > 256 || buf_end - buf < 4 + 4 + 3 * (last - first))
             return AVERROR_INVALIDDATA;
         buf += 4;
-        for (i=first; i<last; i++, buf+=3)
+        for (i=first; i<last; i++, buf+=3) {
             pal[i] = (buf[0] << 18) | (buf[1] << 10) | (buf[2] << 2);
+            pal[i] |= 0xFF << 24 | (pal[i] >> 6) & 0x30303;
+	}
 
         sub_type = buf[0];
         type = buf[1];
diff --git a/libavcodec/bethsoftvideo.c b/libavcodec/bethsoftvideo.c
index 9491abd..b316535 100644
--- a/libavcodec/bethsoftvideo.c
+++ b/libavcodec/bethsoftvideo.c
@@ -52,7 +52,7 @@ static void set_palette(AVFrame * frame, const uint8_t * palette_buffer)
     uint32_t * palette = (uint32_t *)frame->data[1];
     int a;
     for(a = 0; a < 256; a++){
-        palette[a] = AV_RB24(&palette_buffer[a * 3]) * 4;
+        palette[a] = 0xFF << 24 | AV_RB24(&palette_buffer[a * 3]) * 4;
     }
     frame->palette_has_changed = 1;
 }
diff --git a/libavcodec/bfi.c b/libavcodec/bfi.c
index 8f4770e..25863cb 100644
--- a/libavcodec/bfi.c
+++ b/libavcodec/bfi.c
@@ -80,7 +80,7 @@ static int bfi_decode_frame(AVCodecContext * avctx, void *data,
         pal = (uint32_t *) bfi->frame.data[1];
         for (i = 0; i < avctx->extradata_size / 3; i++) {
             int shift = 16;
-            *pal = 0;
+            *pal = 0xFF << 24;
             for (j = 0; j < 3; j++, shift -= 8)
                 *pal +=
                     ((avctx->extradata[i * 3 + j] << 2) |
diff --git a/libavcodec/bmv.c b/libavcodec/bmv.c
index 86d1e91..e98d599 100644
--- a/libavcodec/bmv.c
+++ b/libavcodec/bmv.c
@@ -219,7 +219,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
             return AVERROR_INVALIDDATA;
         }
         for (i = 0; i < 256; i++)
-            c->pal[i] = bytestream_get_be24(&c->stream);
+            c->pal[i] = 0xFF << 24 | bytestream_get_be24(&c->stream);
     }
     if (type & BMV_SCROLL) {
         if (c->stream - pkt->data > pkt->size - 2) {
diff --git a/libavcodec/c93.c b/libavcodec/c93.c
index 06ce416..ad2dc0a 100644
--- a/libavcodec/c93.c
+++ b/libavcodec/c93.c
@@ -152,7 +152,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
         uint32_t *palette = (uint32_t *) newpic->data[1];
         const uint8_t *palbuf = buf + buf_size - 768 - 1;
         for (i = 0; i < 256; i++) {
-            palette[i] = bytestream_get_be24(&palbuf);
+            palette[i] = 0xFF << 24 | bytestream_get_be24(&palbuf);
         }
     } else {
         if (oldpic->data[1])
diff --git a/libavcodec/cdgraphics.c b/libavcodec/cdgraphics.c
index f7d9e5f..b87ca1d 100644
--- a/libavcodec/cdgraphics.c
+++ b/libavcodec/cdgraphics.c
@@ -126,7 +126,7 @@ static void cdg_load_palette(CDGraphicsContext *cc, uint8_t *data, int low)
         r = ((color >> 8) & 0x000F) * 17;
         g = ((color >> 4) & 0x000F) * 17;
         b = ((color     ) & 0x000F) * 17;
-        palette[i + array_offset] = r << 16 | g << 8 | b;
+        palette[i + array_offset] = 0xFF << 24 | r << 16 | g << 8 | b;
     }
     cc->frame.palette_has_changed = 1;
 }
diff --git a/libavcodec/dfa.c b/libavcodec/dfa.c
index f222a59..bce6260 100644
--- a/libavcodec/dfa.c
+++ b/libavcodec/dfa.c
@@ -339,7 +339,7 @@ static int dfa_decode_frame(AVCodecContext *avctx,
             tmp_buf = buf;
             for (i = 0; i < pal_elems; i++) {
                 s->pal[i] = bytestream_get_be24(&tmp_buf) << 2;
-                s->pal[i] |= (s->pal[i] >> 6) & 0x333;
+                s->pal[i] |= 0xFF << 24 | (s->pal[i] >> 6) & 0x30303;
             }
             s->pic.palette_has_changed = 1;
         } else if (chunk_type <= 9) {
diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c
index 53d4f90..80c9677 100644
--- a/libavcodec/dsicinav.c
+++ b/libavcodec/dsicinav.c
@@ -223,7 +223,7 @@ static int cinvideo_decode_frame(AVCodecContext *avctx,
         if (palette_colors_count > 256)
             return AVERROR_INVALIDDATA;
         for (i = 0; i < palette_colors_count; ++i) {
-            cin->palette[i] = bytestream_get_le24(&buf);
+            cin->palette[i] = 0xFF << 24 | bytestream_get_le24(&buf);
             bitmap_frame_size -= 3;
         }
     } else {
diff --git a/libavcodec/dxa.c b/libavcodec/dxa.c
index b3bdae5..f32fdd3 100644
--- a/libavcodec/dxa.c
+++ b/libavcodec/dxa.c
@@ -209,7 +209,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
             r = *buf++;
             g = *buf++;
             b = *buf++;
-            c->pal[i] = (r << 16) | (g << 8) | b;
+            c->pal[i] = 0xFF << 24 | r << 16 | g << 8 | b;
         }
         pc = 1;
         buf_size -= 768+4;
diff --git a/libavcodec/eacmv.c b/libavcodec/eacmv.c
index e05a7a9..8a8d6a9 100644
--- a/libavcodec/eacmv.c
+++ b/libavcodec/eacmv.c
@@ -141,7 +141,7 @@ static void cmv_process_header(CmvContext *s, const uint8_t *buf, const uint8_t
 
     buf += 16;
     for (i=pal_start; i<pal_start+pal_count && i<AVPALETTE_COUNT && buf_end - buf >= 3; i++) {
-        s->palette[i] = AV_RB24(buf);
+        s->palette[i] = 0xFF << 24 | AV_RB24(buf);
         buf += 3;
     }
 }
diff --git a/libavcodec/eatgv.c b/libavcodec/eatgv.c
index 5559dc9..91ae94c 100644
--- a/libavcodec/eatgv.c
+++ b/libavcodec/eatgv.c
@@ -278,7 +278,7 @@ static int tgv_decode_frame(AVCodecContext *avctx,
         pal_count = AV_RL16(&buf[6]);
         buf += 12;
         for(i=0; i<pal_count && i<AVPALETTE_COUNT && buf_end - buf >= 3; i++) {
-            s->palette[i] = AV_RB24(buf);
+            s->palette[i] = 0xFF << 24 | AV_RB24(buf);
             buf += 3;
         }
     }
diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
index 2374d54..e5367d0 100644
--- a/libavcodec/flicvideo.c
+++ b/libavcodec/flicvideo.c
@@ -238,7 +238,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
                     r = buf[stream_ptr++] << color_shift;
                     g = buf[stream_ptr++] << color_shift;
                     b = buf[stream_ptr++] << color_shift;
-                    entry = (r << 16) | (g << 8) | b;
+                    entry = 0xFF << 24 | r << 16 | g << 8 | b;
                     if (s->palette[palette_ptr] != entry)
                         s->new_palette = 1;
                     s->palette[palette_ptr++] = entry;
diff --git a/libavcodec/kmvc.c b/libavcodec/kmvc.c
index 32d9a34..175b84c 100644
--- a/libavcodec/kmvc.c
+++ b/libavcodec/kmvc.c
@@ -251,7 +251,7 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *data_size, AVPa
     if (buf[0] == 127) {
         buf += 3;
         for (i = 0; i < 127; i++) {
-            ctx->pal[i + (header & 0x81)] = AV_RB24(buf);
+            ctx->pal[i + (header & 0x81)] = 0xFF << 24 | AV_RB24(buf);
             buf += 4;
         }
         buf -= 127 * 4 + 3;
diff --git a/libavcodec/mmvideo.c b/libavcodec/mmvideo.c
index 183b298..ff7d100 100644
--- a/libavcodec/mmvideo.c
+++ b/libavcodec/mmvideo.c
@@ -69,7 +69,7 @@ static void mm_decode_pal(MmContext *s, const uint8_t *buf, const uint8_t *buf_e
     int i;
     buf += 4;
     for (i=0; i<128 && buf+2<buf_end; i++) {
-        s->palette[i] = AV_RB24(buf);
+        s->palette[i] = 0xFF << 24 | AV_RB24(buf);
         s->palette[i+128] = s->palette[i]<<2;
         buf += 3;
     }
diff --git a/libavcodec/qdrw.c b/libavcodec/qdrw.c
index bbc0be0..54deec5 100644
--- a/libavcodec/qdrw.c
+++ b/libavcodec/qdrw.c
@@ -90,7 +90,7 @@ static int decode_frame(AVCodecContext *avctx,
         buf++;
         b = *buf++;
         buf++;
-        pal[idx] = (r << 16) | (g << 8) | b;
+        pal[idx] = 0xFF << 24 | r << 16 | g << 8 | b;
     }
     p->palette_has_changed = 1;
 
diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c
index 7d1cf16..0090bf3 100644
--- a/libavcodec/rl2.c
+++ b/libavcodec/rl2.c
@@ -152,7 +152,7 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
 
     /** initialize palette */
     for(i=0;i<AVPALETTE_COUNT;i++)
-        s->palette[i] = AV_RB24(&avctx->extradata[6 + i * 3]);
+        s->palette[i] = 0xFF << 24 | AV_RB24(&avctx->extradata[6 + i * 3]);
 
     /** decode background frame if present */
     back_size = avctx->extradata_size - EXTRADATA1_SIZE;
diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index 8345b7f..19bfab4 100644
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -386,7 +386,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
 
     buf++;
     for(i = 0; i < 256; i++)
-        *pal++ = bytestream_get_be24(&buf);
+        *pal++ = 0xFF << 24 | bytestream_get_be24(&buf);
     buf_size -= 769;
 
     last_reset(smk->mmap_tbl, smk->mmap_last);
diff --git a/libavcodec/sunrast.c b/libavcodec/sunrast.c
index 36dfa98..0f63739 100644
--- a/libavcodec/sunrast.c
+++ b/libavcodec/sunrast.c
@@ -135,7 +135,7 @@ static int sunrast_decode_frame(AVCodecContext *avctx, void *data,
 
         ptr = p->data[1];
         for (x=0; x<len; x++, ptr+=4)
-            *(uint32_t *)ptr = (buf[x]<<16) + (buf[len+x]<<8) + buf[len+len+x];
+            *(uint32_t *)ptr = (0xFF<<24) + (buf[x]<<16) + (buf[len+x]<<8) + buf[len+len+x];
     }
 
     buf += maplength;
diff --git a/libavcodec/tiertexseqv.c b/libavcodec/tiertexseqv.c
index 44ae10e..e8abcd9 100644
--- a/libavcodec/tiertexseqv.c
+++ b/libavcodec/tiertexseqv.c
@@ -178,7 +178,7 @@ static int seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int
         for (i = 0; i < 256; i++) {
             for (j = 0; j < 3; j++, data++)
                 c[j] = (*data << 2) | (*data >> 4);
-            palette[i] = AV_RB24(c);
+            palette[i] = 0xFF << 24 | AV_RB24(c);
         }
         seq->frame.palette_has_changed = 1;
     }
diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c
index 8d0bb98..1060a44 100644
--- a/libavcodec/tiff.c
+++ b/libavcodec/tiff.c
@@ -313,7 +313,7 @@ static int init_image(TiffContext *s)
             /* make default grayscale pal */
             pal = (uint32_t *) s->picture.data[1];
             for (i = 0; i < 1<<s->bpp; i++)
-                pal[i] = i * 255 / ((1<<s->bpp) - 1) * 0x010101;
+                pal[i] = 0xFF << 24 | i * 255 / ((1<<s->bpp) - 1) * 0x010101;
         }
     }
     return 0;
diff --git a/libavcodec/vb.c b/libavcodec/vb.c
index 956d656..d66c47b 100644
--- a/libavcodec/vb.c
+++ b/libavcodec/vb.c
@@ -73,7 +73,7 @@ static void vb_decode_palette(VBDecContext *c, int data_size)
         return;
     }
     for(i = start; i <= start + size; i++)
-        c->pal[i] = bytestream_get_be24(&c->stream);
+        c->pal[i] = 0xFF << 24 | bytestream_get_be24(&c->stream);
 }
 
 static inline int check_pixel(uint8_t *buf, uint8_t *start, uint8_t *end)
diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index 718a9b9..be256e1 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -264,7 +264,7 @@ static void vmd_decode(VmdVideoContext *s)
             r = *p++ * 4;
             g = *p++ * 4;
             b = *p++ * 4;
-            palette32[i] = (r << 16) | (g << 8) | (b);
+            palette32[i] = 0xFF << 24 | r << 16 | g << 8 | b;
         }
     }
     if (p < p_end) {
diff --git a/libavcodec/vqavideo.c b/libavcodec/vqavideo.c
index 8b57d79..5db8a22 100644
--- a/libavcodec/vqavideo.c
+++ b/libavcodec/vqavideo.c
@@ -402,7 +402,7 @@ static void vqa_decode_chunk(VqaContext *s)
             r = s->buf[cpl0_chunk++] * 4;
             g = s->buf[cpl0_chunk++] * 4;
             b = s->buf[cpl0_chunk++] * 4;
-            s->palette[i] = (r << 16) | (g << 8) | (b);
+            s->palette[i] = 0xFF << 24 | r << 16 | g << 8 | b;
         }
     }
 
diff --git a/libavcodec/yop.c b/libavcodec/yop.c
index 87a91f2..e5333db 100644
--- a/libavcodec/yop.c
+++ b/libavcodec/yop.c
@@ -217,10 +217,13 @@ static int yop_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     firstcolor   = s->first_color[is_odd_frame];
     palette      = (uint32_t *)s->frame.data[1];
 
-    for (i = 0; i < s->num_pal_colors; i++, s->srcptr += 3)
+    for (i = 0; i < s->num_pal_colors; i++, s->srcptr += 3) {
         palette[i + firstcolor] = (s->srcptr[0] << 18) |
                                   (s->srcptr[1] << 10) |
                                   (s->srcptr[2] << 2);
+        palette[i + firstcolor] |= 0xFF << 24 |
+                                   (palette[i + firstcolor] >> 6) & 0x30303;
+    }
 
     s->frame.palette_has_changed = 1;
 


More information about the ffmpeg-devel mailing list