[FFmpeg-devel] [PATCH] swscale/yuv2rgb: Increase YUV2RGB table headroom

Michael Niedermayer michaelni at gmx.at
Thu Jan 14 03:50:31 CET 2016


From: Michael Niedermayer <michael at niedermayer.cc>

This makes SWS more robust
Fixes: 07650a772d98aa63b0fed6370dc89037/asan_heap-oob_27ddeaf_2657_2c81ff264dee5d9712cb3251fb9c3bbb.264
Fixes: out of array read

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libswscale/swscale_internal.h |    2 +-
 libswscale/yuv2rgb.c          |   88 ++++++++++++++++++++---------------------
 2 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index a53fdc4..305db4a 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -39,7 +39,7 @@
 
 #define STR(s) AV_TOSTRING(s) // AV_STRINGIFY is too long
 
-#define YUVRGB_TABLE_HEADROOM 256
+#define YUVRGB_TABLE_HEADROOM 512
 
 #define MAX_FILTER_SIZE SWS_MAX_FILTER_SIZE
 
diff --git a/libswscale/yuv2rgb.c b/libswscale/yuv2rgb.c
index 1d682ba..723bec2 100644
--- a/libswscale/yuv2rgb.c
+++ b/libswscale/yuv2rgb.c
@@ -776,7 +776,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
     uint16_t *y_table16;
     uint32_t *y_table32;
     int i, base, rbase, gbase, bbase, av_uninit(abase), needAlpha;
-    const int yoffs = fullRange ? 384 : 326;
+    const int yoffs = fullRange ? 896 : 838;
 
     int64_t crv =  inv_table[0];
     int64_t cbu =  inv_table[1];
@@ -833,10 +833,10 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
             return AVERROR(ENOMEM);
     switch (bpp) {
     case 1:
-        ALLOC_YUV_TABLE(1024);
+        ALLOC_YUV_TABLE(2048);
         y_table     = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024 - 110; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048 - 110; i++) {
             y_table[i + 110]  = av_clip_uint8((yb + 0x8000) >> 16) >> 7;
             yb               += cy;
         }
@@ -848,60 +848,60 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? 3 : 0;
         gbase       = 1;
         bbase       = isRgb ? 0 : 3;
-        ALLOC_YUV_TABLE(1024 * 3);
+        ALLOC_YUV_TABLE(2048 * 3);
         y_table     = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024 - 110; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048 - 110; i++) {
             int yval                = av_clip_uint8((yb + 0x8000) >> 16);
             y_table[i + 110]        = (yval >> 7)        << rbase;
-            y_table[i +  37 + 1024] = ((yval + 43) / 85) << gbase;
-            y_table[i + 110 + 2048] = (yval >> 7)        << bbase;
+            y_table[i +  37 + 2048] = ((yval + 43) / 85) << gbase;
+            y_table[i + 110 + 4096] = (yval >> 7)        << bbase;
             yb += cy;
         }
         fill_table(c->table_rV, 1, crv, y_table + yoffs);
-        fill_table(c->table_gU, 1, cgu, y_table + yoffs + 1024);
-        fill_table(c->table_bU, 1, cbu, y_table + yoffs + 2048);
+        fill_table(c->table_gU, 1, cgu, y_table + yoffs + 2048);
+        fill_table(c->table_bU, 1, cbu, y_table + yoffs + 4096);
         fill_gv_table(c->table_gV, 1, cgv);
         break;
     case 8:
         rbase       = isRgb ? 5 : 0;
         gbase       = isRgb ? 2 : 3;
         bbase       = isRgb ? 0 : 6;
-        ALLOC_YUV_TABLE(1024 * 3);
+        ALLOC_YUV_TABLE(2048 * 3);
         y_table     = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024 - 38; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048 - 38; i++) {
             int yval               = av_clip_uint8((yb + 0x8000) >> 16);
             y_table[i + 16]        = ((yval + 18) / 36) << rbase;
-            y_table[i + 16 + 1024] = ((yval + 18) / 36) << gbase;
-            y_table[i + 37 + 2048] = ((yval + 43) / 85) << bbase;
+            y_table[i + 16 + 2048] = ((yval + 18) / 36) << gbase;
+            y_table[i + 37 + 4096] = ((yval + 43) / 85) << bbase;
             yb += cy;
         }
         fill_table(c->table_rV, 1, crv, y_table + yoffs);
-        fill_table(c->table_gU, 1, cgu, y_table + yoffs + 1024);
-        fill_table(c->table_bU, 1, cbu, y_table + yoffs + 2048);
+        fill_table(c->table_gU, 1, cgu, y_table + yoffs + 2048);
+        fill_table(c->table_bU, 1, cbu, y_table + yoffs + 4096);
         fill_gv_table(c->table_gV, 1, cgv);
         break;
     case 12:
         rbase       = isRgb ? 8 : 0;
         gbase       = 4;
         bbase       = isRgb ? 0 : 8;
-        ALLOC_YUV_TABLE(1024 * 3 * 2);
+        ALLOC_YUV_TABLE(2048 * 3 * 2);
         y_table16   = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048; i++) {
             uint8_t yval        = av_clip_uint8((yb + 0x8000) >> 16);
             y_table16[i]        = (yval >> 4) << rbase;
-            y_table16[i + 1024] = (yval >> 4) << gbase;
-            y_table16[i + 2048] = (yval >> 4) << bbase;
+            y_table16[i + 2048] = (yval >> 4) << gbase;
+            y_table16[i + 4096] = (yval >> 4) << bbase;
             yb += cy;
         }
         if (isNotNe)
-            for (i = 0; i < 1024 * 3; i++)
+            for (i = 0; i < 2048 * 3; i++)
                 y_table16[i] = av_bswap16(y_table16[i]);
         fill_table(c->table_rV, 2, crv, y_table16 + yoffs);
-        fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 1024);
-        fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 2048);
+        fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 2048);
+        fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 4096);
         fill_gv_table(c->table_gV, 2, cgv);
         break;
     case 15:
@@ -909,30 +909,30 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? bpp - 5 : 0;
         gbase       = 5;
         bbase       = isRgb ? 0 : (bpp - 5);
-        ALLOC_YUV_TABLE(1024 * 3 * 2);
+        ALLOC_YUV_TABLE(2048 * 3 * 2);
         y_table16   = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048; i++) {
             uint8_t yval        = av_clip_uint8((yb + 0x8000) >> 16);
             y_table16[i]        = (yval >> 3)          << rbase;
-            y_table16[i + 1024] = (yval >> (18 - bpp)) << gbase;
-            y_table16[i + 2048] = (yval >> 3)          << bbase;
+            y_table16[i + 2048] = (yval >> (18 - bpp)) << gbase;
+            y_table16[i + 4096] = (yval >> 3)          << bbase;
             yb += cy;
         }
         if (isNotNe)
-            for (i = 0; i < 1024 * 3; i++)
+            for (i = 0; i < 2048 * 3; i++)
                 y_table16[i] = av_bswap16(y_table16[i]);
         fill_table(c->table_rV, 2, crv, y_table16 + yoffs);
-        fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 1024);
-        fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 2048);
+        fill_table(c->table_gU, 2, cgu, y_table16 + yoffs + 2048);
+        fill_table(c->table_bU, 2, cbu, y_table16 + yoffs + 4096);
         fill_gv_table(c->table_gV, 2, cgv);
         break;
     case 24:
     case 48:
-        ALLOC_YUV_TABLE(1024);
+        ALLOC_YUV_TABLE(2048);
         y_table     = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048; i++) {
             y_table[i]  = av_clip_uint8((yb + 0x8000) >> 16);
             yb         += cy;
         }
@@ -951,20 +951,20 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         needAlpha = CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat);
         if (!needAlpha)
             abase = (base + 24) & 31;
-        ALLOC_YUV_TABLE(1024 * 3 * 4);
+        ALLOC_YUV_TABLE(2048 * 3 * 4);
         y_table32   = c->yuvTable;
-        yb = -(384 << 16) - oy;
-        for (i = 0; i < 1024; i++) {
+        yb = -(384 << 16) - 512*cy - oy;
+        for (i = 0; i < 2048; i++) {
             unsigned yval       = av_clip_uint8((yb + 0x8000) >> 16);
             y_table32[i]        = (yval << rbase) +
                                   (needAlpha ? 0 : (255u << abase));
-            y_table32[i + 1024] =  yval << gbase;
-            y_table32[i + 2048] =  yval << bbase;
+            y_table32[i + 2048] =  yval << gbase;
+            y_table32[i + 4096] =  yval << bbase;
             yb += cy;
         }
         fill_table(c->table_rV, 4, crv, y_table32 + yoffs);
-        fill_table(c->table_gU, 4, cgu, y_table32 + yoffs + 1024);
-        fill_table(c->table_bU, 4, cbu, y_table32 + yoffs + 2048);
+        fill_table(c->table_gU, 4, cgu, y_table32 + yoffs + 2048);
+        fill_table(c->table_bU, 4, cbu, y_table32 + yoffs + 4096);
         fill_gv_table(c->table_gV, 4, cgv);
         break;
     default:
-- 
1.7.9.5



More information about the ffmpeg-devel mailing list