[FFmpeg-devel] [PATCH] swscale/yuv2rgb: Increase YUV2RGB table headroom
Michael Niedermayer
michaelni at gmx.at
Thu Jan 14 03:50:31 CET 2016
From: Michael Niedermayer <michael at niedermayer.cc>
This makes SWS more robust
Fixes: 07650a772d98aa63b0fed6370dc89037/asan_heap-oob_27ddeaf_2657_2c81ff264dee5d9712cb3251fb9c3bbb.264
Fixes: out of array read
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
libswscale/swscale_internal.h | 2 +-
libswscale/yuv2rgb.c | 88 ++++++++++++++++++++---------------------
2 files changed, 45 insertions(+), 45 deletions(-)
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index a53fdc4..305db4a 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -39,7 +39,7 @@
#define STR(s) AV_TOSTRING(s) // AV_STRINGIFY is too long
-#define YUVRGB_TABLE_HEADROOM 256
+#define YUVRGB_TABLE_HEADROOM 512
#define MAX_FILTER_SIZE SWS_MAX_FILTER_SIZE
diff --git a/libswscale/yuv2rgb.c b/libswscale/yuv2rgb.c
index 1d682ba..723bec2 100644
--- a/libswscale/yuv2rgb.c
+++ b/libswscale/yuv2rgb.c
@@ -776,7 +776,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
uint16_t *y_table16;
uint32_t *y_table32;
int i, base, rbase, gbase, bbase, av_uninit(abase), needAlpha;
- const int yoffs = fullRange ? 384 : 326;
+ const int yoffs = fullRange ? 896 : 838;
int64_t crv = inv_table[0];
int64_t cbu = inv_table[1];
@@ -833,10 +833,10 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
return AVERROR(ENOMEM);
switch (bpp) {
case 1:
- ALLOC_YUV_TABLE(1024);
+ ALLOC_YUV_TABLE(2048);
y_table = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024 - 110; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048 - 110; i++) {
y_table[i + 110] = av_clip_uint8((yb + 0x8000) >> 16) >> 7;
yb += cy;
}
@@ -848,60 +848,60 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
rbase = isRgb ? 3 : 0;
gbase = 1;
bbase = isRgb ? 0 : 3;
- ALLOC_YUV_TABLE(1024 * 3);
+ ALLOC_YUV_TABLE(2048 * 3);
y_table = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024 - 110; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048 - 110; i++) {
int yval = av_clip_uint8((yb + 0x8000) >> 16);
y_table[i + 110] = (yval >> 7) << rbase;
- y_table[i + 37 + 1024] = ((yval + 43) / 85) << gbase;
- y_table[i + 110 + 2048] = (yval >> 7) << bbase;
+ y_table[i + 37 + 2048] = ((yval + 43) / 85) << gbase;
+ y_table[i + 110 + 4096] = (yval >> 7) << bbase;
yb += cy;
}
fill_table(c->table_rV, 1, crv, y_table + yoffs);
- fill_table(c->table_gU, 1, cgu, y_table + yoffs + 1024);
- fill_table(c->table_bU, 1, cbu, y_table + yoffs + 2048);
+ fill_table(c->table_gU, 1, cgu, y_table + yoffs + 2048);
+ fill_table(c->table_bU, 1, cbu, y_table + yoffs + 4096);
fill_gv_table(c->table_gV, 1, cgv);
break;
case 8:
rbase = isRgb ? 5 : 0;
gbase = isRgb ? 2 : 3;
bbase = isRgb ? 0 : 6;
- ALLOC_YUV_TABLE(1024 * 3);
+ ALLOC_YUV_TABLE(2048 * 3);
y_table = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024 - 38; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048 - 38; i++) {
int yval = av_clip_uint8((yb + 0x8000) >> 16);
y_table[i + 16] = ((yval + 18) / 36) << rbase;
- y_table[i + 16 + 1024] = ((yval + 18) / 36) << gbase;
- y_table[i + 37 + 2048] = ((yval + 43) / 85) << bbase;
+ y_table[i + 16 + 2048] = ((yval + 18) / 36) << gbase;
+ y_table[i + 37 + 4096] = ((yval + 43) / 85) << bbase;
yb += cy;
}
fill_table(c->table_rV, 1, crv, y_table + yoffs);
- fill_table(c->table_gU, 1, cgu, y_table + yoffs + 1024);
- fill_table(c->table_bU, 1, cbu, y_table + yoffs + 2048);
+ fill_table(c->table_gU, 1, cgu, y_table + yoffs + 2048);
+ fill_table(c->table_bU, 1, cbu, y_table + yoffs + 4096);
fill_gv_table(c->table_gV, 1, cgv);
break;
case 12:
rbase = isRgb ? 8 : 0;
gbase = 4;
bbase = isRgb ? 0 : 8;
- ALLOC_YUV_TABLE(1024 * 3 * 2);
+ ALLOC_YUV_TABLE(2048 * 3 * 2);
y_table16 = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048; i++) {
uint8_t yval = av_clip_uint8((yb + 0x8000) >> 16);
y_table16[i] = (yval >> 4) << rbase;
- y_table16[i + 1024] = (yval >> 4) << gbase;
- y_table16[i + 2048] = (yval >> 4) << bbase;
+ y_table16[i + 2048] = (yval >> 4) << gbase;
+ y_table16[i + 4096] = (yval >> 4) << bbase;
yb += cy;
}
if (isNotNe)
- for (i = 0; i < 1024 * 3; i++)
+ for (i = 0; i < 2048 * 3; i++)
y_table16[i] = av_bswap16(y_table16[i]);
fill_table(c->table_rV, 2, crv, y_table16 + yoffs);
- fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 1024);
- fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 2048);
+ fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 2048);
+ fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 4096);
fill_gv_table(c->table_gV, 2, cgv);
break;
case 15:
@@ -909,30 +909,30 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
rbase = isRgb ? bpp - 5 : 0;
gbase = 5;
bbase = isRgb ? 0 : (bpp - 5);
- ALLOC_YUV_TABLE(1024 * 3 * 2);
+ ALLOC_YUV_TABLE(2048 * 3 * 2);
y_table16 = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048; i++) {
uint8_t yval = av_clip_uint8((yb + 0x8000) >> 16);
y_table16[i] = (yval >> 3) << rbase;
- y_table16[i + 1024] = (yval >> (18 - bpp)) << gbase;
- y_table16[i + 2048] = (yval >> 3) << bbase;
+ y_table16[i + 2048] = (yval >> (18 - bpp)) << gbase;
+ y_table16[i + 4096] = (yval >> 3) << bbase;
yb += cy;
}
if (isNotNe)
- for (i = 0; i < 1024 * 3; i++)
+ for (i = 0; i < 2048 * 3; i++)
y_table16[i] = av_bswap16(y_table16[i]);
fill_table(c->table_rV, 2, crv, y_table16 + yoffs);
- fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 1024);
- fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 2048);
+ fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 2048);
+ fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 4096);
fill_gv_table(c->table_gV, 2, cgv);
break;
case 24:
case 48:
- ALLOC_YUV_TABLE(1024);
+ ALLOC_YUV_TABLE(2048);
y_table = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048; i++) {
y_table[i] = av_clip_uint8((yb + 0x8000) >> 16);
yb += cy;
}
@@ -951,20 +951,20 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
needAlpha = CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat);
if (!needAlpha)
abase = (base + 24) & 31;
- ALLOC_YUV_TABLE(1024 * 3 * 4);
+ ALLOC_YUV_TABLE(2048 * 3 * 4);
y_table32 = c->yuvTable;
- yb = -(384 << 16) - oy;
- for (i = 0; i < 1024; i++) {
+ yb = -(384 << 16) - 512*cy - oy;
+ for (i = 0; i < 2048; i++) {
unsigned yval = av_clip_uint8((yb + 0x8000) >> 16);
y_table32[i] = (yval << rbase) +
(needAlpha ? 0 : (255u << abase));
- y_table32[i + 1024] = yval << gbase;
- y_table32[i + 2048] = yval << bbase;
+ y_table32[i + 2048] = yval << gbase;
+ y_table32[i + 4096] = yval << bbase;
yb += cy;
}
fill_table(c->table_rV, 4, crv, y_table32 + yoffs);
- fill_table(c->table_gU, 4, cgu, y_table32 + yoffs + 1024);
- fill_table(c->table_bU, 4, cbu, y_table32 + yoffs + 2048);
+ fill_table(c->table_gU, 4, cgu, y_table32 + yoffs + 2048);
+ fill_table(c->table_bU, 4, cbu, y_table32 + yoffs + 4096);
fill_gv_table(c->table_gV, 4, cgv);
break;
default:
--
1.7.9.5
More information about the ffmpeg-devel
mailing list