[FFmpeg-devel] [PATCH]Fix palette calculation in some decoders
Carl Eugen Hoyos
cehoyos at ag.or.at
Fri Nov 11 22:44:18 CET 2011
Hi!
If I am not missing something, FFmpeg currently happily assumes complete
transparency for most PAL8 codecs.
I did not find the palette code in vmnc.c, I am unsure if ff_cga_palette[] is
correct (black is always transparent?).
Please comment, I did not test the ones with "AV_RB24()" and
"bytestream_get_be24()".
Thank you, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/avs.c b/libavcodec/avs.c
index cf965b3..0317c79 100644
--- a/libavcodec/avs.c
+++ b/libavcodec/avs.c
@@ -85,8 +85,10 @@ avs_decode_frame(AVCodecContext * avctx,
if (first >= 256 || last > 256 || buf_end - buf < 4 + 4 + 3 * (last - first))
return AVERROR_INVALIDDATA;
buf += 4;
- for (i=first; i<last; i++, buf+=3)
+ for (i=first; i<last; i++, buf+=3) {
pal[i] = (buf[0] << 18) | (buf[1] << 10) | (buf[2] << 2);
+ pal[i] |= 0xFF << 24 | (pal[i] >> 6) & 0x30303;
+ }
sub_type = buf[0];
type = buf[1];
diff --git a/libavcodec/bethsoftvideo.c b/libavcodec/bethsoftvideo.c
index 9491abd..b316535 100644
--- a/libavcodec/bethsoftvideo.c
+++ b/libavcodec/bethsoftvideo.c
@@ -52,7 +52,7 @@ static void set_palette(AVFrame * frame, const uint8_t * palette_buffer)
uint32_t * palette = (uint32_t *)frame->data[1];
int a;
for(a = 0; a < 256; a++){
- palette[a] = AV_RB24(&palette_buffer[a * 3]) * 4;
+ palette[a] = 0xFF << 24 | AV_RB24(&palette_buffer[a * 3]) * 4;
}
frame->palette_has_changed = 1;
}
diff --git a/libavcodec/bfi.c b/libavcodec/bfi.c
index 8f4770e..25863cb 100644
--- a/libavcodec/bfi.c
+++ b/libavcodec/bfi.c
@@ -80,7 +80,7 @@ static int bfi_decode_frame(AVCodecContext * avctx, void *data,
pal = (uint32_t *) bfi->frame.data[1];
for (i = 0; i < avctx->extradata_size / 3; i++) {
int shift = 16;
- *pal = 0;
+ *pal = 0xFF << 24;
for (j = 0; j < 3; j++, shift -= 8)
*pal +=
((avctx->extradata[i * 3 + j] << 2) |
diff --git a/libavcodec/bmv.c b/libavcodec/bmv.c
index 86d1e91..e98d599 100644
--- a/libavcodec/bmv.c
+++ b/libavcodec/bmv.c
@@ -219,7 +219,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
return AVERROR_INVALIDDATA;
}
for (i = 0; i < 256; i++)
- c->pal[i] = bytestream_get_be24(&c->stream);
+ c->pal[i] = 0xFF << 24 | bytestream_get_be24(&c->stream);
}
if (type & BMV_SCROLL) {
if (c->stream - pkt->data > pkt->size - 2) {
diff --git a/libavcodec/c93.c b/libavcodec/c93.c
index 06ce416..ad2dc0a 100644
--- a/libavcodec/c93.c
+++ b/libavcodec/c93.c
@@ -152,7 +152,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
uint32_t *palette = (uint32_t *) newpic->data[1];
const uint8_t *palbuf = buf + buf_size - 768 - 1;
for (i = 0; i < 256; i++) {
- palette[i] = bytestream_get_be24(&palbuf);
+ palette[i] = 0xFF << 24 | bytestream_get_be24(&palbuf);
}
} else {
if (oldpic->data[1])
diff --git a/libavcodec/cdgraphics.c b/libavcodec/cdgraphics.c
index f7d9e5f..b87ca1d 100644
--- a/libavcodec/cdgraphics.c
+++ b/libavcodec/cdgraphics.c
@@ -126,7 +126,7 @@ static void cdg_load_palette(CDGraphicsContext *cc, uint8_t *data, int low)
r = ((color >> 8) & 0x000F) * 17;
g = ((color >> 4) & 0x000F) * 17;
b = ((color ) & 0x000F) * 17;
- palette[i + array_offset] = r << 16 | g << 8 | b;
+ palette[i + array_offset] = 0xFF << 24 | r << 16 | g << 8 | b;
}
cc->frame.palette_has_changed = 1;
}
diff --git a/libavcodec/dfa.c b/libavcodec/dfa.c
index f222a59..bce6260 100644
--- a/libavcodec/dfa.c
+++ b/libavcodec/dfa.c
@@ -339,7 +339,7 @@ static int dfa_decode_frame(AVCodecContext *avctx,
tmp_buf = buf;
for (i = 0; i < pal_elems; i++) {
s->pal[i] = bytestream_get_be24(&tmp_buf) << 2;
- s->pal[i] |= (s->pal[i] >> 6) & 0x333;
+ s->pal[i] |= 0xFF << 24 | (s->pal[i] >> 6) & 0x30303;
}
s->pic.palette_has_changed = 1;
} else if (chunk_type <= 9) {
diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c
index 53d4f90..80c9677 100644
--- a/libavcodec/dsicinav.c
+++ b/libavcodec/dsicinav.c
@@ -223,7 +223,7 @@ static int cinvideo_decode_frame(AVCodecContext *avctx,
if (palette_colors_count > 256)
return AVERROR_INVALIDDATA;
for (i = 0; i < palette_colors_count; ++i) {
- cin->palette[i] = bytestream_get_le24(&buf);
+ cin->palette[i] = 0xFF << 24 | bytestream_get_le24(&buf);
bitmap_frame_size -= 3;
}
} else {
diff --git a/libavcodec/dxa.c b/libavcodec/dxa.c
index b3bdae5..f32fdd3 100644
--- a/libavcodec/dxa.c
+++ b/libavcodec/dxa.c
@@ -209,7 +209,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
r = *buf++;
g = *buf++;
b = *buf++;
- c->pal[i] = (r << 16) | (g << 8) | b;
+ c->pal[i] = 0xFF << 24 | r << 16 | g << 8 | b;
}
pc = 1;
buf_size -= 768+4;
diff --git a/libavcodec/eacmv.c b/libavcodec/eacmv.c
index e05a7a9..8a8d6a9 100644
--- a/libavcodec/eacmv.c
+++ b/libavcodec/eacmv.c
@@ -141,7 +141,7 @@ static void cmv_process_header(CmvContext *s, const uint8_t *buf, const uint8_t
buf += 16;
for (i=pal_start; i<pal_start+pal_count && i<AVPALETTE_COUNT && buf_end - buf >= 3; i++) {
- s->palette[i] = AV_RB24(buf);
+ s->palette[i] = 0xFF << 24 | AV_RB24(buf);
buf += 3;
}
}
diff --git a/libavcodec/eatgv.c b/libavcodec/eatgv.c
index 5559dc9..91ae94c 100644
--- a/libavcodec/eatgv.c
+++ b/libavcodec/eatgv.c
@@ -278,7 +278,7 @@ static int tgv_decode_frame(AVCodecContext *avctx,
pal_count = AV_RL16(&buf[6]);
buf += 12;
for(i=0; i<pal_count && i<AVPALETTE_COUNT && buf_end - buf >= 3; i++) {
- s->palette[i] = AV_RB24(buf);
+ s->palette[i] = 0xFF << 24 | AV_RB24(buf);
buf += 3;
}
}
diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
index 2374d54..e5367d0 100644
--- a/libavcodec/flicvideo.c
+++ b/libavcodec/flicvideo.c
@@ -238,7 +238,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
r = buf[stream_ptr++] << color_shift;
g = buf[stream_ptr++] << color_shift;
b = buf[stream_ptr++] << color_shift;
- entry = (r << 16) | (g << 8) | b;
+ entry = 0xFF << 24 | r << 16 | g << 8 | b;
if (s->palette[palette_ptr] != entry)
s->new_palette = 1;
s->palette[palette_ptr++] = entry;
diff --git a/libavcodec/kmvc.c b/libavcodec/kmvc.c
index 32d9a34..175b84c 100644
--- a/libavcodec/kmvc.c
+++ b/libavcodec/kmvc.c
@@ -251,7 +251,7 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *data_size, AVPa
if (buf[0] == 127) {
buf += 3;
for (i = 0; i < 127; i++) {
- ctx->pal[i + (header & 0x81)] = AV_RB24(buf);
+ ctx->pal[i + (header & 0x81)] = 0xFF << 24 | AV_RB24(buf);
buf += 4;
}
buf -= 127 * 4 + 3;
diff --git a/libavcodec/mmvideo.c b/libavcodec/mmvideo.c
index 183b298..ff7d100 100644
--- a/libavcodec/mmvideo.c
+++ b/libavcodec/mmvideo.c
@@ -69,7 +69,7 @@ static void mm_decode_pal(MmContext *s, const uint8_t *buf, const uint8_t *buf_e
int i;
buf += 4;
for (i=0; i<128 && buf+2<buf_end; i++) {
- s->palette[i] = AV_RB24(buf);
+ s->palette[i] = 0xFF << 24 | AV_RB24(buf);
s->palette[i+128] = s->palette[i]<<2;
buf += 3;
}
diff --git a/libavcodec/qdrw.c b/libavcodec/qdrw.c
index bbc0be0..54deec5 100644
--- a/libavcodec/qdrw.c
+++ b/libavcodec/qdrw.c
@@ -90,7 +90,7 @@ static int decode_frame(AVCodecContext *avctx,
buf++;
b = *buf++;
buf++;
- pal[idx] = (r << 16) | (g << 8) | b;
+ pal[idx] = 0xFF << 24 | r << 16 | g << 8 | b;
}
p->palette_has_changed = 1;
diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c
index 7d1cf16..0090bf3 100644
--- a/libavcodec/rl2.c
+++ b/libavcodec/rl2.c
@@ -152,7 +152,7 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
/** initialize palette */
for(i=0;i<AVPALETTE_COUNT;i++)
- s->palette[i] = AV_RB24(&avctx->extradata[6 + i * 3]);
+ s->palette[i] = 0xFF << 24 | AV_RB24(&avctx->extradata[6 + i * 3]);
/** decode background frame if present */
back_size = avctx->extradata_size - EXTRADATA1_SIZE;
diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index 8345b7f..19bfab4 100644
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -386,7 +386,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
buf++;
for(i = 0; i < 256; i++)
- *pal++ = bytestream_get_be24(&buf);
+ *pal++ = 0xFF << 24 | bytestream_get_be24(&buf);
buf_size -= 769;
last_reset(smk->mmap_tbl, smk->mmap_last);
diff --git a/libavcodec/sunrast.c b/libavcodec/sunrast.c
index 36dfa98..0f63739 100644
--- a/libavcodec/sunrast.c
+++ b/libavcodec/sunrast.c
@@ -135,7 +135,7 @@ static int sunrast_decode_frame(AVCodecContext *avctx, void *data,
ptr = p->data[1];
for (x=0; x<len; x++, ptr+=4)
- *(uint32_t *)ptr = (buf[x]<<16) + (buf[len+x]<<8) + buf[len+len+x];
+ *(uint32_t *)ptr = (0xFF<<24) + (buf[x]<<16) + (buf[len+x]<<8) + buf[len+len+x];
}
buf += maplength;
diff --git a/libavcodec/tiertexseqv.c b/libavcodec/tiertexseqv.c
index 44ae10e..e8abcd9 100644
--- a/libavcodec/tiertexseqv.c
+++ b/libavcodec/tiertexseqv.c
@@ -178,7 +178,7 @@ static int seqvideo_decode(SeqVideoContext *seq, const unsigned char *data, int
for (i = 0; i < 256; i++) {
for (j = 0; j < 3; j++, data++)
c[j] = (*data << 2) | (*data >> 4);
- palette[i] = AV_RB24(c);
+ palette[i] = 0xFF << 24 | AV_RB24(c);
}
seq->frame.palette_has_changed = 1;
}
diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c
index 8d0bb98..1060a44 100644
--- a/libavcodec/tiff.c
+++ b/libavcodec/tiff.c
@@ -313,7 +313,7 @@ static int init_image(TiffContext *s)
/* make default grayscale pal */
pal = (uint32_t *) s->picture.data[1];
for (i = 0; i < 1<<s->bpp; i++)
- pal[i] = i * 255 / ((1<<s->bpp) - 1) * 0x010101;
+ pal[i] = 0xFF << 24 | i * 255 / ((1<<s->bpp) - 1) * 0x010101;
}
}
return 0;
diff --git a/libavcodec/vb.c b/libavcodec/vb.c
index 956d656..d66c47b 100644
--- a/libavcodec/vb.c
+++ b/libavcodec/vb.c
@@ -73,7 +73,7 @@ static void vb_decode_palette(VBDecContext *c, int data_size)
return;
}
for(i = start; i <= start + size; i++)
- c->pal[i] = bytestream_get_be24(&c->stream);
+ c->pal[i] = 0xFF << 24 | bytestream_get_be24(&c->stream);
}
static inline int check_pixel(uint8_t *buf, uint8_t *start, uint8_t *end)
diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index 718a9b9..be256e1 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -264,7 +264,7 @@ static void vmd_decode(VmdVideoContext *s)
r = *p++ * 4;
g = *p++ * 4;
b = *p++ * 4;
- palette32[i] = (r << 16) | (g << 8) | (b);
+ palette32[i] = 0xFF << 24 | r << 16 | g << 8 | b;
}
}
if (p < p_end) {
diff --git a/libavcodec/vqavideo.c b/libavcodec/vqavideo.c
index 8b57d79..5db8a22 100644
--- a/libavcodec/vqavideo.c
+++ b/libavcodec/vqavideo.c
@@ -402,7 +402,7 @@ static void vqa_decode_chunk(VqaContext *s)
r = s->buf[cpl0_chunk++] * 4;
g = s->buf[cpl0_chunk++] * 4;
b = s->buf[cpl0_chunk++] * 4;
- s->palette[i] = (r << 16) | (g << 8) | (b);
+ s->palette[i] = 0xFF << 24 | r << 16 | g << 8 | b;
}
}
diff --git a/libavcodec/yop.c b/libavcodec/yop.c
index 87a91f2..e5333db 100644
--- a/libavcodec/yop.c
+++ b/libavcodec/yop.c
@@ -217,10 +217,13 @@ static int yop_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
firstcolor = s->first_color[is_odd_frame];
palette = (uint32_t *)s->frame.data[1];
- for (i = 0; i < s->num_pal_colors; i++, s->srcptr += 3)
+ for (i = 0; i < s->num_pal_colors; i++, s->srcptr += 3) {
palette[i + firstcolor] = (s->srcptr[0] << 18) |
(s->srcptr[1] << 10) |
(s->srcptr[2] << 2);
+ palette[i + firstcolor] |= 0xFF << 24 |
+ (palette[i + firstcolor] >> 6) & 0x30303;
+ }
s->frame.palette_has_changed = 1;
More information about the ffmpeg-devel
mailing list