[FFmpeg-devel] [PATCH] Added VMAF filter linking to Netflix's vmaf library
Ashish Singh
ashk43712 at gmail.com
Wed Jun 21 15:16:40 EEST 2017
This is VMAF filter which requires Netflix's vmaf library installed but currently there is no library implementation in the Netflix's vmaf. I will open a pull request soon to Netflix's vmaf for the library integration code and provide a link in this thread.
This uses pthreads (mutex/cond) where this filter runs in the main thread and Netflix's vmaf runs in a separate thread and it pulls the frames from libavfilter through mutex/cond(wait/signal) pairs. Library header is "libvmaf.h" which contains an API, compute_vmaf(..). There is a callback function inside this patch called read_frame() (passed as a parameter in compute_vmaf(..)) which gets called from inside Netflix's library to repeatedly update the frame data.
After installing the Netflix's vmaf library, do --enable-libvmaf at the time of configure. It can be run using "ffmpeg -i main -i ref -lavfi vmaf -f null -"
---
configure | 6 +
libavfilter/Makefile | 1 +
libavfilter/allfilters.c | 1 +
libavfilter/vf_vmaf.c | 353 +++++++++++++++++++++++++++++++++++++++++++++++
libavfilter/vmaf.h | 33 +++++
5 files changed, 394 insertions(+)
create mode 100644 libavfilter/vf_vmaf.c
create mode 100644 libavfilter/vmaf.h
diff --git a/configure b/configure
index 5ae5227..8487f06 100755
--- a/configure
+++ b/configure
@@ -259,6 +259,7 @@ External library support:
--enable-libtwolame enable MP2 encoding via libtwolame [no]
--enable-libv4l2 enable libv4l2/v4l-utils [no]
--enable-libvidstab enable video stabilization using vid.stab [no]
+ --enable-libvmaf enable vmaf filter via libvmaf [no]
--enable-libvo-amrwbenc enable AMR-WB encoding via libvo-amrwbenc [no]
--enable-libvorbis enable Vorbis en/decoding via libvorbis,
native implementation exists [no]
@@ -1569,6 +1570,7 @@ EXTERNAL_LIBRARY_LIST="
libtheora
libtwolame
libv4l2
+ libvmaf
libvorbis
libvpx
libwavpack
@@ -2878,6 +2880,7 @@ libspeex_encoder_deps="libspeex"
libspeex_encoder_select="audio_frame_queue"
libtheora_encoder_deps="libtheora"
libtwolame_encoder_deps="libtwolame"
+libvmaf_filter_deps="libvmaf"
libvo_amrwbenc_encoder_deps="libvo_amrwbenc"
libvorbis_decoder_deps="libvorbis"
libvorbis_encoder_deps="libvorbis"
@@ -5845,6 +5848,9 @@ enabled libtwolame && require libtwolame twolame.h twolame_init -ltwolame
die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }
enabled libv4l2 && require_pkg_config libv4l2 libv4l2.h v4l2_ioctl
enabled libvidstab && require_pkg_config "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit
+enabled libvmaf && { use_pkg_config libvmaf "libvmaf.h" compute_vmaf ||
+ { check_lib libvmaf "libvmaf.h" "compute_vmaf" -lvmaf -lstdc++ -lpthread -lm &&
+ warn "using libvmaf without pkg-config"; } }
enabled libvo_amrwbenc && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_init -lvo-amrwbenc
enabled libvorbis && require libvorbis vorbis/vorbisenc.h vorbis_info_init -lvorbisenc -lvorbis -logg
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index f7dfe8a..1c4bd56 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -314,6 +314,7 @@ OBJS-$(CONFIG_VFLIP_FILTER) += vf_vflip.o
OBJS-$(CONFIG_VIDSTABDETECT_FILTER) += vidstabutils.o vf_vidstabdetect.o
OBJS-$(CONFIG_VIDSTABTRANSFORM_FILTER) += vidstabutils.o vf_vidstabtransform.o
OBJS-$(CONFIG_VIGNETTE_FILTER) += vf_vignette.o
+OBJS-$(CONFIG_VMAF_FILTER) += vf_vmaf.o dualinput.o framesync.o
OBJS-$(CONFIG_VSTACK_FILTER) += vf_stack.o framesync.o
OBJS-$(CONFIG_W3FDIF_FILTER) += vf_w3fdif.o
OBJS-$(CONFIG_WAVEFORM_FILTER) += vf_waveform.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index cd35ae4..6894a6f 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -325,6 +325,7 @@ static void register_all(void)
REGISTER_FILTER(VIDSTABDETECT, vidstabdetect, vf);
REGISTER_FILTER(VIDSTABTRANSFORM, vidstabtransform, vf);
REGISTER_FILTER(VIGNETTE, vignette, vf);
+ REGISTER_FILTER(VMAF, vmaf, vf);
REGISTER_FILTER(VSTACK, vstack, vf);
REGISTER_FILTER(W3FDIF, w3fdif, vf);
REGISTER_FILTER(WAVEFORM, waveform, vf);
diff --git a/libavfilter/vf_vmaf.c b/libavfilter/vf_vmaf.c
new file mode 100644
index 0000000..a9c6e4d
--- /dev/null
+++ b/libavfilter/vf_vmaf.c
@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2017 Ronald S. Bultje <rsbultje at gmail.com>
+ * Copyright (c) 2017 Ashish Pratap Singh <ashk43712 at gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Caculate the VMAF between two input videos.
+ */
+
+#include <inttypes.h>
+#include <pthread.h>
+#include "libavutil/avstring.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "avfilter.h"
+#include "dualinput.h"
+#include "drawutils.h"
+#include "formats.h"
+#include "internal.h"
+#include "video.h"
+#include "vmaf.h"
+#include "libvmaf.h"
+
+typedef struct VMAFContext {
+ const AVClass *class;
+ FFDualInputContext dinput;
+ uint64_t nb_frames;
+ FILE *stats_file;
+ char *stats_file_str;
+ int stats_version;
+ int stats_header_written;
+ int stats_add_max;
+ pthread_t thread;
+ pthread_attr_t attr;
+ int nb_components;
+ double vmaf_sum;
+ VMAFDSPContext dsp;
+} VMAFContext;
+
+
+static char *format;
+static int width, height;
+pthread_mutex_t lock;
+pthread_cond_t cond;
+pthread_t main_thread_id;
+pthread_t vmaf_thread_id;
+int eof = 0;
+double vmaf_score;
+double curr_vmaf_score;
+AVFrame *gmain;
+AVFrame *gref;
+
+#define OFFSET(x) offsetof(VMAFContext, x)
+#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
+
+static const AVOption vmaf_options[] = {
+ {"stats_file", "Set file where to store per-frame difference information", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS },
+ {"f", "Set file where to store per-frame difference information", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS },
+ {"stats_version", "Set the format version for the stats file.", OFFSET(stats_version), AV_OPT_TYPE_INT, {.i64=1}, 1, 2, FLAGS },
+ {"output_max", "Add raw stats (max values) to the output log.", OFFSET(stats_add_max), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS},
+ { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(vmaf);
+
+static inline double get_vmaf(double vmaf_sum, uint64_t nb_frames)
+{
+ return vmaf_sum/nb_frames;
+}
+
+static void set_meta(AVDictionary **metadata, const char *key, float d)
+{
+ char value[128];
+ snprintf(value, sizeof(value), "%0.7f", d);
+ av_dict_set(metadata,key,value,0);
+}
+
+static int read_frame(float *ref_data, int *ref_stride, float *main_data, int *main_stride, double *score){
+ static int p = 0;
+ if(p == 0){
+ *ref_stride = gref->linesize[0];
+ *main_stride = gmain->linesize[0];
+ p = 1;
+ return 0;
+ }
+
+ if(eof == 1){
+ return 1;
+ }
+ pthread_mutex_lock(&lock);
+ while(gref == NULL){
+ pthread_cond_wait(&cond, &lock);
+ }
+
+ *ref_stride = gref->linesize[0];
+ *main_stride = gmain->linesize[0];
+
+ uint8_t *ptr = gref->data[0];
+ float *ptr1 = ref_data;
+
+ int i,j;
+ for(i=0;i<height;i++){
+ for(j=0;j<width;j++){
+ ptr1[j] = (float)ptr[j];
+ }
+ ptr += *ref_stride;
+ ptr1 += *ref_stride;
+ }
+
+ ptr = gmain->data[0];
+ ptr1 = main_data;
+
+ for(i=0;i<height;i++){
+ for(j=0;j<width;j++){
+ ptr1[j] = (float)ptr[j];
+ }
+ ptr += *main_stride;
+ ptr1 += *main_stride;
+ }
+
+ gref = NULL;
+ gmain = NULL;
+
+ pthread_cond_signal(&cond);
+ pthread_mutex_unlock(&lock);
+
+ return eof;
+}
+
+static AVFrame *do_vmaf(AVFilterContext *ctx, AVFrame *main, const AVFrame *ref)
+{
+ VMAFContext *s = ctx->priv;
+
+ AVDictionary **metadata = avpriv_frame_get_metadatap(main);
+
+ pthread_mutex_lock(&lock);
+ while(gref != NULL){
+ pthread_cond_wait(&cond, &lock);
+ }
+
+ gref = malloc(sizeof(AVFrame));
+ gmain = malloc(sizeof(AVFrame));
+
+ memcpy(gref, ref, sizeof(AVFrame));
+ memcpy(gmain, main, sizeof(AVFrame));
+
+ pthread_cond_signal(&cond);
+ pthread_mutex_unlock(&lock);
+
+ return main;
+}
+
+static void compute_vmaf_score()
+{
+ char *model_path = "/usr/local/share/model/vmaf_v0.6.1.pkl";
+
+ vmaf_score = compute_vmaf(format, width, height, read_frame, model_path);
+}
+
+static void *call_vmaf(void *t)
+{
+ int i;
+ long tid;
+ tid = (long)t;
+ compute_vmaf_score();
+ pthread_exit((void*) t);
+}
+
+static av_cold int init(AVFilterContext *ctx)
+{
+ VMAFContext *s = ctx->priv;
+
+ if (s->stats_file_str) {
+ if (s->stats_version < 2 && s->stats_add_max) {
+ av_log(ctx, AV_LOG_ERROR,
+ "stats_add_max was specified but stats_version < 2.\n" );
+ return AVERROR(EINVAL);
+ }
+ if (!strcmp(s->stats_file_str, "-")) {
+ s->stats_file = stdout;
+ } else {
+ s->stats_file = fopen(s->stats_file_str, "w");
+ if (!s->stats_file) {
+ int err = AVERROR(errno);
+ char buf[128];
+ av_strerror(err, buf, sizeof(buf));
+ av_log(ctx, AV_LOG_ERROR, "Could not open stats file %s: %s\n",
+ s->stats_file_str, buf);
+ return err;
+ }
+ }
+ }
+
+ s->dinput.process = do_vmaf;
+ return 0;
+}
+
+static int query_formats(AVFilterContext *ctx)
+{
+ static const enum AVPixelFormat pix_fmts[] = {
+ AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV420P,
+ AV_PIX_FMT_YUV444P10LE, AV_PIX_FMT_YUV422P10LE, AV_PIX_FMT_YUV420P10LE,
+ AV_PIX_FMT_NONE
+ };
+ VMAFContext *s = ctx->priv;
+
+ AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
+ if (!fmts_list)
+ return AVERROR(ENOMEM);
+ return ff_set_common_formats(ctx, fmts_list);
+}
+
+
+static int config_input_ref(AVFilterLink *inlink)
+{
+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+ AVFilterContext *ctx = inlink->dst;
+ VMAFContext *s = ctx->priv;
+ s->nb_components = desc->nb_components;
+ if (ctx->inputs[0]->w != ctx->inputs[1]->w ||
+ ctx->inputs[0]->h != ctx->inputs[1]->h) {
+ av_log(ctx, AV_LOG_ERROR, "Width and height of input videos must be same.\n");
+ return AVERROR(EINVAL);
+ }
+ if (ctx->inputs[0]->format != ctx->inputs[1]->format) {
+ av_log(ctx, AV_LOG_ERROR, "Inputs must be of same pixel format.\n");
+ return AVERROR(EINVAL);
+ }
+
+ format = av_get_pix_fmt_name(ctx->inputs[0]->format);
+ width = ctx->inputs[0]->w;
+ height = ctx->inputs[0]->h;
+
+ pthread_mutex_init(&lock, NULL);
+ pthread_cond_init (&cond, NULL);
+
+ pthread_attr_init(&s->attr);
+ int d = 5;
+
+ main_thread_id=pthread_self();
+
+ int rc = pthread_create(&s->thread, &s->attr, call_vmaf, (void *)d);
+ vmaf_thread_id = s->thread;
+
+ return 0;
+}
+
+
+static int config_output(AVFilterLink *outlink)
+{
+ AVFilterContext *ctx = outlink->src;
+ VMAFContext *s = ctx->priv;
+ AVFilterLink *mainlink = ctx->inputs[0];
+ int ret;
+
+ outlink->w = mainlink->w;
+ outlink->h = mainlink->h;
+ outlink->time_base = mainlink->time_base;
+ outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio;
+ outlink->frame_rate = mainlink->frame_rate;
+ if ((ret = ff_dualinput_init(ctx, &s->dinput)) < 0)
+ return ret;
+
+ return 0;
+}
+
+static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
+{
+ VMAFContext *s = inlink->dst->priv;
+ return ff_dualinput_filter_frame(&s->dinput, inlink, inpicref);
+}
+
+static int request_frame(AVFilterLink *outlink)
+{
+ VMAFContext *s = outlink->src->priv;
+ return ff_dualinput_request_frame(&s->dinput, outlink);
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+ VMAFContext *s = ctx->priv;
+
+ if (s->nb_frames > 0) {
+ av_log(ctx, AV_LOG_INFO, "VMAF average:%f\n",
+ get_vmaf(s->vmaf_sum, s->nb_frames));
+ }
+ ff_dualinput_uninit(&s->dinput);
+
+ if (s->stats_file && s->stats_file != stdout)
+ fclose(s->stats_file);
+
+ static int ptr = 0;
+ if(ptr == 1){
+ eof = 1;
+ pthread_join(vmaf_thread_id, NULL);
+ av_log(ctx, AV_LOG_INFO, "VMAF score: %f\n",vmaf_score);
+ }
+ ptr++;
+ return 0;
+}
+
+static const AVFilterPad vmaf_inputs[] = {
+ {
+ .name = "main",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .filter_frame = filter_frame,
+ },{
+ .name = "reference",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .filter_frame = filter_frame,
+ .config_props = config_input_ref,
+ },
+ { NULL }
+};
+
+static const AVFilterPad vmaf_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = config_output,
+ .request_frame = request_frame,
+ },
+ { NULL }
+};
+
+AVFilter ff_vf_vmaf = {
+ .name = "vmaf",
+ .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."),
+ .init = init,
+ .uninit = uninit,
+ .query_formats = query_formats,
+ .priv_size = sizeof(VMAFContext),
+ .priv_class = &vmaf_class,
+ .inputs = vmaf_inputs,
+ .outputs = vmaf_outputs,
+};
diff --git a/libavfilter/vmaf.h b/libavfilter/vmaf.h
new file mode 100644
index 0000000..2e7cd8a
--- /dev/null
+++ b/libavfilter/vmaf.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2015 Ronald S. Bultje <rsbultje at gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVFILTER_PSNR_H
+#define AVFILTER_PSNR_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct VMAFDSPContext {
+ uint64_t (*sse_line)(const uint8_t *buf, const uint8_t *ref, int w);
+} VMAFDSPContext;
+
+void ff_psnr_init_x86(VMAFDSPContext *dsp, int bpp);
+
+#endif /* AVFILTER_VMAF_H */
--
2.7.4
More information about the ffmpeg-devel
mailing list