[FFmpeg-cvslog] avfilter/vf_colorkey: fix formula for calculation of difference

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sat May 16 19:45:53 2020 +0200| [8b0575d7638f40730c020ada41a573fff715c1a7] | committer: Paul B Mahol

avfilter/vf_colorkey: fix formula for calculation of difference

Also fixes colorhold filtering.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=8b0575d7638f40730c020ada41a573fff715c1a7
---

 libavfilter/vf_colorkey.c             |  2 +-
 tests/ref/fate/ffmpeg-filter_colorkey | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/libavfilter/vf_colorkey.c b/libavfilter/vf_colorkey.c
index 7af426a61e..0ac847c633 100644
--- a/libavfilter/vf_colorkey.c
+++ b/libavfilter/vf_colorkey.c
@@ -45,7 +45,7 @@ static uint8_t do_colorkey_pixel(ColorkeyContext *ctx, uint8_t r, uint8_t g, uin
     int dg = (int)g - ctx->colorkey_rgba[1];
     int db = (int)b - ctx->colorkey_rgba[2];
 
-    double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0));
+    double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0 * 3.0));
 
     if (ctx->blend > 0.0001) {
         return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * 255.0;
diff --git a/tests/ref/fate/ffmpeg-filter_colorkey b/tests/ref/fate/ffmpeg-filter_colorkey
index 490a0f9aa6..25c3fc0912 100644
--- a/tests/ref/fate/ffmpeg-filter_colorkey
+++ b/tests/ref/fate/ffmpeg-filter_colorkey
@@ -3,13 +3,13 @@
 #codec_id 0: rawvideo
 #dimensions 0: 720x576
 #sar 0: 0/1
-0,          0,          0,        1,   622080, 0x78efb628
-0,          1,          1,        1,   622080, 0x641f2564
-0,          2,          2,        1,   622080, 0x348f25c3
-0,          3,          3,        1,   622080, 0x6afc485a
-0,          4,          4,        1,   622080, 0xe949107f
-0,          5,          5,        1,   622080, 0x171716e5
-0,          6,          6,        1,   622080, 0x2985a01f
-0,          7,          7,        1,   622080, 0xc5ddabd7
-0,          8,          8,        1,   622080, 0xb4dd2b7f
-0,          9,          9,        1,   622080, 0x6e75ba82
+0,          0,          0,        1,   622080, 0x5775bb12
+0,          1,          1,        1,   622080, 0x84d8395a
+0,          2,          2,        1,   622080, 0x599d3a6b
+0,          3,          3,        1,   622080, 0x1df1e9dc
+0,          4,          4,        1,   622080, 0xf3255836
+0,          5,          5,        1,   622080, 0xf46bc26e
+0,          6,          6,        1,   622080, 0x76d491da
+0,          7,          7,        1,   622080, 0x2c9f3ca6
+0,          8,          8,        1,   622080, 0x5713e203
+0,          9,          9,        1,   622080, 0x47776493