[FFmpeg-cvslog] fate: add tests for psnr and ssim filter
Tobias Rapp
git at videolan.org
Wed Sep 27 17:16:32 EEST 2017
ffmpeg | branch: master | Tobias Rapp <t.rapp at noa-archive.com> | Tue Aug 29 14:45:13 2017 +0200| [bee01ee2ba2e62974447d5e8ea2afb27dbdb1e23] | committer: Tobias Rapp
fate: add tests for psnr and ssim filter
Metadata filter output is passed through an Awk script comparing floats
against reference values with specified "fuzz" tolerance to account for
architectural differences (e.g. x86-32 vs. x86-64).
Signed-off-by: Tobias Rapp <t.rapp at noa-archive.com>
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=bee01ee2ba2e62974447d5e8ea2afb27dbdb1e23
---
tests/fate-run.sh | 9 +++++
tests/fate/filter-video.mak | 14 ++++++++
tests/ref/fate/filter-refcmp-psnr-rgb | 45 ++++++++++++++++++++++++
tests/ref/fate/filter-refcmp-psnr-yuv | 45 ++++++++++++++++++++++++
tests/ref/fate/filter-refcmp-ssim-rgb | 30 ++++++++++++++++
tests/ref/fate/filter-refcmp-ssim-yuv | 30 ++++++++++++++++
tests/refcmp-metadata.awk | 64 +++++++++++++++++++++++++++++++++++
7 files changed, 237 insertions(+)
diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 9aa9a22395..c5480c7ede 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -234,6 +234,15 @@ lavftest(){
${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
}
+refcmp_metadata(){
+ refcmp=$1
+ pixfmt=$2
+ fuzz=${3:-0.001}
+ ffmpeg $FLAGS $ENC_OPTS \
+ -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
+ -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
+}
+
video_filter(){
filters=$1
shift
diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index d1e13414f6..78cd4711e6 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC
fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
+REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
+fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
+fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
+fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
+fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
+
FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
FATE_FFMPEG += $(FATE_FILTER-yes)
diff --git a/tests/ref/fate/filter-refcmp-psnr-rgb b/tests/ref/fate/filter-refcmp-psnr-rgb
new file mode 100644
index 0000000000..f06db575ac
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-rgb
@@ -0,0 +1,45 @@
+frame:0 pts:0 pts_time:0
+lavfi.psnr.mse.r=1381.80
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=896.00
+lavfi.psnr.psnr.g=18.61
+lavfi.psnr.mse.b=277.38
+lavfi.psnr.psnr.b=23.70
+lavfi.psnr.mse_avg=851.73
+lavfi.psnr.psnr_avg=18.83
+frame:1 pts:1 pts_time:1
+lavfi.psnr.mse.r=1380.37
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=975.91
+lavfi.psnr.psnr.g=18.24
+lavfi.psnr.mse.b=435.72
+lavfi.psnr.psnr.b=21.74
+lavfi.psnr.mse_avg=930.67
+lavfi.psnr.psnr_avg=18.44
+frame:2 pts:2 pts_time:2
+lavfi.psnr.mse.r=1403.20
+lavfi.psnr.psnr.r=16.66
+lavfi.psnr.mse.g=954.05
+lavfi.psnr.psnr.g=18.34
+lavfi.psnr.mse.b=494.22
+lavfi.psnr.psnr.b=21.19
+lavfi.psnr.mse_avg=950.49
+lavfi.psnr.psnr_avg=18.35
+frame:3 pts:3 pts_time:3
+lavfi.psnr.mse.r=1452.80
+lavfi.psnr.psnr.r=16.51
+lavfi.psnr.mse.g=1001.02
+lavfi.psnr.psnr.g=18.13
+lavfi.psnr.mse.b=557.39
+lavfi.psnr.psnr.b=20.67
+lavfi.psnr.mse_avg=1003.74
+lavfi.psnr.psnr_avg=18.11
+frame:4 pts:4 pts_time:4
+lavfi.psnr.mse.r=1401.25
+lavfi.psnr.psnr.r=16.67
+lavfi.psnr.mse.g=1009.80
+lavfi.psnr.psnr.g=18.09
+lavfi.psnr.mse.b=602.42
+lavfi.psnr.psnr.b=20.33
+lavfi.psnr.mse_avg=1004.49
+lavfi.psnr.psnr_avg=18.11
diff --git a/tests/ref/fate/filter-refcmp-psnr-yuv b/tests/ref/fate/filter-refcmp-psnr-yuv
new file mode 100644
index 0000000000..0e634ed0e4
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-yuv
@@ -0,0 +1,45 @@
+frame:0 pts:0 pts_time:0
+lavfi.psnr.mse.y=222.06
+lavfi.psnr.psnr.y=24.67
+lavfi.psnr.mse.u=339.38
+lavfi.psnr.psnr.u=22.82
+lavfi.psnr.mse.v=705.41
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=372.23
+lavfi.psnr.psnr_avg=22.42
+frame:1 pts:1 pts_time:1
+lavfi.psnr.mse.y=236.74
+lavfi.psnr.psnr.y=24.39
+lavfi.psnr.mse.u=416.17
+lavfi.psnr.psnr.u=21.94
+lavfi.psnr.mse.v=704.98
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=398.66
+lavfi.psnr.psnr_avg=22.12
+frame:2 pts:2 pts_time:2
+lavfi.psnr.mse.y=234.79
+lavfi.psnr.psnr.y=24.42
+lavfi.psnr.mse.u=435.72
+lavfi.psnr.psnr.u=21.74
+lavfi.psnr.mse.v=699.60
+lavfi.psnr.psnr.v=19.68
+lavfi.psnr.mse_avg=401.23
+lavfi.psnr.psnr_avg=22.10
+frame:3 pts:3 pts_time:3
+lavfi.psnr.mse.y=250.88
+lavfi.psnr.psnr.y=24.14
+lavfi.psnr.mse.u=479.73
+lavfi.psnr.psnr.u=21.32
+lavfi.psnr.mse.v=707.55
+lavfi.psnr.psnr.v=19.63
+lavfi.psnr.mse_avg=422.26
+lavfi.psnr.psnr_avg=21.88
+frame:4 pts:4 pts_time:4
+lavfi.psnr.mse.y=241.05
+lavfi.psnr.psnr.y=24.31
+lavfi.psnr.mse.u=505.04
+lavfi.psnr.psnr.u=21.10
+lavfi.psnr.mse.v=716.00
+lavfi.psnr.psnr.v=19.58
+lavfi.psnr.mse_avg=425.79
+lavfi.psnr.psnr_avg=21.84
diff --git a/tests/ref/fate/filter-refcmp-ssim-rgb b/tests/ref/fate/filter-refcmp-ssim-rgb
new file mode 100644
index 0000000000..8c23c60b37
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-rgb
@@ -0,0 +1,30 @@
+frame:0 pts:0 pts_time:0
+lavfi.ssim.R=0.72
+lavfi.ssim.G=0.76
+lavfi.ssim.B=0.89
+lavfi.ssim.All=0.79
+lavfi.ssim.dB=6.74
+frame:1 pts:1 pts_time:1
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.85
+lavfi.ssim.All=0.77
+lavfi.ssim.dB=6.31
+frame:2 pts:2 pts_time:2
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.75
+lavfi.ssim.B=0.84
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.29
+frame:3 pts:3 pts_time:3
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.73
+lavfi.ssim.B=0.83
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.11
+frame:4 pts:4 pts_time:4
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.80
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.05
diff --git a/tests/ref/fate/filter-refcmp-ssim-yuv b/tests/ref/fate/filter-refcmp-ssim-yuv
new file mode 100644
index 0000000000..5c8ffb9483
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-yuv
@@ -0,0 +1,30 @@
+frame:0 pts:0 pts_time:0
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.76
+lavfi.ssim.V=0.69
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.25
+frame:1 pts:1 pts_time:1
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.08
+frame:2 pts:2 pts_time:2
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.10
+frame:3 pts:3 pts_time:3
+lavfi.ssim.Y=0.79
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.94
+frame:4 pts:4 pts_time:4
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.97
diff --git a/tests/refcmp-metadata.awk b/tests/refcmp-metadata.awk
new file mode 100644
index 0000000000..fa21aad0e0
--- /dev/null
+++ b/tests/refcmp-metadata.awk
@@ -0,0 +1,64 @@
+# Compare metadata filter output containing float value strings to reference
+# output data. Returns the whole reference data if delta of each value is below
+# threshold, else returns the whole input data.
+
+function abs(val) {
+ return ((val < 0.0) ? -val : val);
+}
+
+function max(val1, val2) {
+ return ((val1 >= val2) ? val1 : val2);
+}
+
+function is_numeric_str(str) {
+ return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/);
+}
+
+BEGIN {
+ FS = "=";
+ # check for "fuzz" (threshold) program parameter, else use default
+ if (fuzz <= 0.0) {
+ fuzz = 0.1;
+ }
+ # check for "ref" (reference file) program parameter
+ if (ref) {
+ ref_nr = 0;
+ while ((getline line < ref) > 0) {
+ ref_nr++;
+ ref_lines[ref_nr] = line;
+ if (split(line, fields) == 2 && is_numeric_str(fields[2])) {
+ ref_keys[ref_nr] = fields[1];
+ ref_vals[ref_nr] = fields[2] + 0; # convert to number
+ }
+ }
+ close(ref);
+ }
+ delta_max = 0;
+ result = (ref ? 1 : 0);
+}
+
+{
+ cmp_lines[NR] = $0;
+ if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) {
+ val = $2 + 0; # convert to number
+ delta = abs((val / ref_vals[NR]) - 1);
+ delta_max = max(delta_max, delta);
+ result = result && ($1 == ref_keys[NR]) && (delta <= fuzz);
+ } else {
+ result = result && ($0 == ref_lines[NR]);
+ }
+}
+
+END {
+ if (result) {
+ for (i = 1; i <= ref_nr; i++)
+ print ref_lines[i];
+ } else {
+ for (i = 1; i <= NR; i++)
+ print cmp_lines[i];
+ if (NR != ref_nr)
+ print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr";
+ if (delta_max >= fuzz)
+ print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr";
+ }
+}
More information about the ffmpeg-cvslog
mailing list