[FFmpeg-devel] [PATCH] avfilter/vf_chromakey: The chromakey filter preserves non-opaque alpha transparency.

[Gavin Smith]

This is to address trac ticket #8724.  The filter did not preserve alpha transparency. Items that were transparent in the input would appear black on the output or pixels that were semi-tranparent would appear opaque.
---
 libavfilter/vf_chromakey.c | 43 ++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/libavfilter/vf_chromakey.c b/libavfilter/vf_chromakey.c
index 4b1669d084..7206f44441 100644
--- a/libavfilter/vf_chromakey.c
+++ b/libavfilter/vf_chromakey.c
@@ -47,7 +47,7 @@ typedef struct ChromakeyContext {
                     int jobnr, int nb_jobs);
 } ChromakeyContext;
 
-static uint8_t do_chromakey_pixel(ChromakeyContext *ctx, uint8_t u[9], uint8_t v[9])
+static uint8_t do_chromakey_pixel(ChromakeyContext *ctx, uint8_t a, uint8_t u[9], uint8_t v[9])
 {
     double diff = 0.0;
     int du, dv, i;
@@ -62,13 +62,13 @@ static uint8_t do_chromakey_pixel(ChromakeyContext *ctx, uint8_t u[9], uint8_t v
     diff /= 9.0;
 
     if (ctx->blend > 0.0001) {
-        return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * 255.0;
+        return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * (float)a;
     } else {
-        return (diff > ctx->similarity) ? 255 : 0;
+        return (diff > ctx->similarity) ? a : 0;
     }
 }
 
-static uint16_t do_chromakey_pixel16(ChromakeyContext *ctx, uint16_t u[9], uint16_t v[9])
+static uint16_t do_chromakey_pixel16(ChromakeyContext *ctx, uint16_t a, uint16_t u[9], uint16_t v[9])
 {
     double max = ctx->max;
     double diff = 0.0;
@@ -84,9 +84,9 @@ static uint16_t do_chromakey_pixel16(ChromakeyContext *ctx, uint16_t u[9], uint1
     diff /= 9.0;
 
     if (ctx->blend > 0.0001) {
-        return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * max;
+        return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * (float)a;
     } else {
-        return (diff > ctx->similarity) ? max : 0;
+        return (diff > ctx->similarity) ? a : 0;
     }
 }
 
@@ -131,13 +131,17 @@ static int do_chromakey_slice(AVFilterContext *avctx, void *arg, int jobnr, int
 
     for (y = slice_start; y < slice_end; ++y) {
         for (x = 0; x < frame->width; ++x) {
-            for (yo = 0; yo < 3; ++yo) {
-                for (xo = 0; xo < 3; ++xo) {
-                    get_pixel_uv(frame, ctx->hsub_log2, ctx->vsub_log2, x + xo - 1, y + yo - 1, &u[yo * 3 + xo], &v[yo * 3 + xo]);
+            uint8_t *a = frame->data[3] + frame->linesize[3] * y;
+            const uint8_t ao = a[x];
+
+            if (ao != 0) {
+                for (yo = 0; yo < 3; ++yo) {
+                    for (xo = 0; xo < 3; ++xo) {
+                        get_pixel_uv(frame, ctx->hsub_log2, ctx->vsub_log2, x + xo - 1, y + yo - 1, &u[yo * 3 + xo], &v[yo * 3 + xo]);
+                    }
                 }
+                a[x] = do_chromakey_pixel(ctx, ao, u, v);
             }
-
-            frame->data[3][frame->linesize[3] * y + x] = do_chromakey_pixel(ctx, u, v);
         }
     }
 
@@ -163,15 +167,18 @@ static int do_chromakey16_slice(AVFilterContext *avctx, void *arg, int jobnr, in
 
     for (y = slice_start; y < slice_end; ++y) {
         for (x = 0; x < frame->width; ++x) {
-            uint16_t *dst = (uint16_t *)(frame->data[3] + frame->linesize[3] * y);
-
-            for (yo = 0; yo < 3; ++yo) {
-                for (xo = 0; xo < 3; ++xo) {
-                    get_pixel16_uv(frame, ctx->hsub_log2, ctx->vsub_log2, x + xo - 1, y + yo - 1, &u[yo * 3 + xo], &v[yo * 3 + xo]);
+            uint16_t *a = (uint16_t *)(frame->data[3] + frame->linesize[3] * y);
+            const uint16_t ao = a[x];
+
+            if (ao != 0) {
+                for (yo = 0; yo < 3; ++yo) {
+                    for (xo = 0; xo < 3; ++xo) {
+                        get_pixel16_uv(frame, ctx->hsub_log2, ctx->vsub_log2, x + xo - 1, y + yo - 1, &u[yo * 3 + xo], &v[yo * 3 + xo]);
+                    }
                 }
-            }
 
-            dst[x] = do_chromakey_pixel16(ctx, u, v);
+                a[x] = do_chromakey_pixel16(ctx, ao, u, v);
+            }
         }
     }
 
-- 
2.17.1