[FFmpeg-devel] [PATCH v1 7/9] lavu/pix_fmt: consider bits per pixel in finding best pix fmt

[Fei Wang]

By using bits per pixel in finding best pix fmt will make the result
more accurate. For example, the src fmt is AV_PIX_FMT_YUV420P10LE and
dst fmt are AV_PIX_FMT_P012LE and AV_PIX_FMT_P010LE. In this case the
two dst fmts have the same scores and padded bits and number of
components and P012 will be chosen finally. So add the bits per pixel
as one of conditions will return P010 which should be the best choice
compare with src fmt.

Signed-off-by: Fei Wang <fei.w.wang at intel.com>
---
 libavutil/pixdesc.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 38297e2e83..354441e840 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -2890,6 +2890,7 @@ enum AVPixelFormat av_find_best_pix_fmt_of_2(enum AVPixelFormat dst_pix_fmt1, en
     int loss1, loss2, loss_mask;
     const AVPixFmtDescriptor *desc1 = av_pix_fmt_desc_get(dst_pix_fmt1);
     const AVPixFmtDescriptor *desc2 = av_pix_fmt_desc_get(dst_pix_fmt2);
+    const AVPixFmtDescriptor *src = av_pix_fmt_desc_get(src_pix_fmt);
     int score1, score2;
 
     if (!desc1) {
@@ -2905,8 +2906,11 @@ enum AVPixelFormat av_find_best_pix_fmt_of_2(enum AVPixelFormat dst_pix_fmt1, en
         score2 = get_pix_fmt_score(dst_pix_fmt2, src_pix_fmt, &loss2, loss_mask);
 
         if (score1 == score2) {
-            if(av_get_padded_bits_per_pixel(desc2) != av_get_padded_bits_per_pixel(desc1)) {
+            if (av_get_padded_bits_per_pixel(desc2) != av_get_padded_bits_per_pixel(desc1)) {
                 dst_pix_fmt = av_get_padded_bits_per_pixel(desc2) < av_get_padded_bits_per_pixel(desc1) ? dst_pix_fmt2 : dst_pix_fmt1;
+            } else if (av_get_bits_per_pixel(desc2) == av_get_bits_per_pixel(src) ||
+                       av_get_bits_per_pixel(desc1) == av_get_bits_per_pixel(src)) {
+                dst_pix_fmt = av_get_bits_per_pixel(desc2) == av_get_bits_per_pixel(src) ? dst_pix_fmt2 : dst_pix_fmt1;
             } else {
                 dst_pix_fmt = desc2->nb_components < desc1->nb_components ? dst_pix_fmt2 : dst_pix_fmt1;
             }
-- 
2.17.1