[FFmpeg-devel] [PATCH] imgutils: generalize linesize computation for bitstream formats

[Stefano Sabatini]

Make it a subcase of the general algorithm used for the non-bitstream
case. Simplify, and make av_image_get_linesize() and
av_image_fill_linesizes() correctly return the right value when plane
!= 0.

All the supported bitstream formats have only one plane (containing
all the components), so the linesize of all the other planes should be
set to 0.

In particular fix a crash occurring with:
-vf format=monow,showinfo,format=showinfo.
---
 libavutil/imgutils.c |   17 +++++------------
 1 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c
index 8eefa4d..154fe87 100644
--- a/libavutil/imgutils.c
+++ b/libavutil/imgutils.c
@@ -48,14 +48,12 @@ int av_image_get_linesize(enum PixelFormat pix_fmt, int width, int plane)
     const AVPixFmtDescriptor *desc = &av_pix_fmt_descriptors[pix_fmt];
     int max_step     [4];       /* max pixel step for each plane */
     int max_step_comp[4];       /* the component for each plane which has the max pixel step */
-    int s;
-
-    if (desc->flags & PIX_FMT_BITSTREAM)
-        return (width * (desc->comp[0].step_minus1+1) + 7) >> 3;
+    int s, ls;
 
     av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
     s = (max_step_comp[plane] == 1 || max_step_comp[plane] == 2) ? desc->log2_chroma_w : 0;
-    return max_step[plane] * (((width + (1 << s) - 1)) >> s);
+    ls = max_step[plane] * (((width + (1 << s) - 1)) >> s);
+    return desc->flags & PIX_FMT_BITSTREAM ? (ls + 7) >> 3 : ls;
 }
 
 int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int width)
@@ -70,13 +68,6 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
     if ((unsigned)pix_fmt >= PIX_FMT_NB || desc->flags & PIX_FMT_HWACCEL)
         return AVERROR(EINVAL);
 
-    if (desc->flags & PIX_FMT_BITSTREAM) {
-        if (width > (INT_MAX -7) / (desc->comp[0].step_minus1+1))
-            return AVERROR(EINVAL);
-        linesizes[0] = (width * (desc->comp[0].step_minus1+1) + 7) >> 3;
-        return 0;
-    }
-
     av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
     for (i = 0; i < 4; i++) {
         int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0;
@@ -84,6 +75,8 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
         if (max_step[i] > INT_MAX / shifted_w)
             return AVERROR(EINVAL);
         linesizes[i] = max_step[i] * shifted_w;
+        if (desc->flags & PIX_FMT_BITSTREAM)
+            linesizes[i] = (linesizes[i] + 7) >> 3;
     }
 
     return 0;
-- 
1.7.2.3