[FFmpeg-devel] [PATCH] avfilter: add colorvd video filter
James Almer
jamrial at gmail.com
Sat Nov 13 14:22:12 EET 2021
On 11/13/2021 9:14 AM, Paul B Mahol wrote:
> Signed-off-by: Paul B Mahol <onemda at gmail.com>
> ---
> libavfilter/Makefile | 1 +
> libavfilter/allfilters.c | 1 +
> libavfilter/vf_colorvd.c | 259 +++++++++++++++++++++++++++++++++++++++
> 3 files changed, 261 insertions(+)
> create mode 100644 libavfilter/vf_colorvd.c
>
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 18e28bcc62..8c5e565ed1 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -215,6 +215,7 @@ OBJS-$(CONFIG_COLORLEVELS_FILTER) += vf_colorlevels.o
> OBJS-$(CONFIG_COLORMATRIX_FILTER) += vf_colormatrix.o
> OBJS-$(CONFIG_COLORSPACE_FILTER) += vf_colorspace.o colorspace.o colorspacedsp.o
> OBJS-$(CONFIG_COLORTEMPERATURE_FILTER) += vf_colortemperature.o
> +OBJS-$(CONFIG_COLORVD_FILTER) += vf_colorvd.o
> OBJS-$(CONFIG_CONVOLUTION_FILTER) += vf_convolution.o
> OBJS-$(CONFIG_CONVOLUTION_OPENCL_FILTER) += vf_convolution_opencl.o opencl.o \
> opencl/convolution.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 7ec13a15b2..d7556bc93f 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -204,6 +204,7 @@ extern const AVFilter ff_vf_colorlevels;
> extern const AVFilter ff_vf_colormatrix;
> extern const AVFilter ff_vf_colorspace;
> extern const AVFilter ff_vf_colortemperature;
> +extern const AVFilter ff_vf_colorvd;
> extern const AVFilter ff_vf_convolution;
> extern const AVFilter ff_vf_convolution_opencl;
> extern const AVFilter ff_vf_convolve;
> diff --git a/libavfilter/vf_colorvd.c b/libavfilter/vf_colorvd.c
> new file mode 100644
> index 0000000000..c5cec7b318
> --- /dev/null
> +++ b/libavfilter/vf_colorvd.c
> @@ -0,0 +1,259 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include <float.h>
> +
> +#include "libavutil/opt.h"
> +#include "libavutil/imgutils.h"
> +#include "avfilter.h"
> +#include "formats.h"
> +#include "internal.h"
> +#include "video.h"
> +
> +enum ColorVisualDeficiency {
> + PROTAN,
> + DEUTAN,
> + TRITAN,
> + NB_DEF
> +};
> +
> +static const float lrgb2lms[3][3] =
> +{
> + { 0.17886, 0.43997, 0.03597 },
> + { 0.03380, 0.27515, 0.03621 },
> + { 0.00031, 0.00192, 0.01528 },
> +};
> +
> +static const float lms2lrgb[3][3] =
> +{
> + { 8.00533, -12.88195, 11.68065 },
> + { -0.97821, 5.26945, -10.18300 },
> + { -0.04017, -0.39885, 66.48079 },
> +};
> +
> +typedef struct Brettel {
> + int element;
> + float projection[2][3];
> + float separation[3];
> +} Brettel;
> +
> +static Brettel brettel[NB_DEF] =
> +{
> + [PROTAN] = {
> + 0,
> + {
> + { 0.00000, 2.18394, -5.65554 },
> + { 0.00000, 2.16614, -5.30455 },
> + },
> + { 0.00000, 0.01751, -0.34516 }
> + },
> + [DEUTAN] = {
> + 1,
> + {
> + { 0.46165, 0.00000, 2.44885 },
> + { 0.45789, 0.00000, 2.58960 },
> + },
> + { -0.01751, 0.00000, 0.65480 }
> + },
> + [TRITAN] = {
> + 2,
> + {
> + { -0.00213, 0.05477, 0.00000 },
> + { -0.06195, 0.16826, 0.00000 },
> + },
> + { 0.34516, -0.65480, 0.00000 }
nit: f suffix on all these constants.
> + }
> +};
> +
> +typedef struct CVDContext {
> + const AVClass *class;
> +
> + int deficiency;
> + float severity;
> +
> + int (*do_slice)(AVFilterContext *s, void *arg,
> + int jobnr, int nb_jobs);
> +} CVDContext;
> +
> +static float apply_dot3(const float vector[3], const float input[3])
> +{
> + return vector[0] * input[0] + vector[1] * input[1] + vector[2] * input[2];
> +}
> +
> +static void apply_matrix(const float matrix[3][3], const float input[3], float output[3])
> +{
> + output[0] = matrix[0][0] * input[0] + matrix[0][1] * input[1] + matrix[0][2] * input[2];
> + output[1] = matrix[1][0] * input[0] + matrix[1][1] * input[1] + matrix[1][2] * input[2];
> + output[2] = matrix[2][0] * input[0] + matrix[2][1] * input[1] + matrix[2][2] * input[2];
> +}
> +
> +typedef struct ThreadData {
> + AVFrame *in, *out;
> +} ThreadData;
> +
> +static int cvd_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
> +{
> + CVDContext *s = ctx->priv;
> + ThreadData *td = arg;
> + AVFrame *in = td->in;
> + AVFrame *out = td->out;
> + const int width = in->width;
> + const int height = in->height;
> + const int slice_start = (height * jobnr) / nb_jobs;
> + const int slice_end = (height * (jobnr + 1)) / nb_jobs;
> + const int srlinesize = in->linesize[2] / 4;
> + const int sglinesize = in->linesize[0] / 4;
> + const int sblinesize = in->linesize[1] / 4;
> + const int drlinesize = out->linesize[2] / 4;
> + const int dglinesize = out->linesize[0] / 4;
> + const int dblinesize = out->linesize[1] / 4;
> + const float *sr = (const float *)in->data[2] + slice_start * srlinesize;
> + const float *sg = (const float *)in->data[0] + slice_start * sglinesize;
> + const float *sb = (const float *)in->data[1] + slice_start * sblinesize;
> + float *dr = (float *)out->data[2] + slice_start * drlinesize;
> + float *dg = (float *)out->data[0] + slice_start * dglinesize;
> + float *db = (float *)out->data[1] + slice_start * dblinesize;
> + const float severity = s->severity;
> + const float iseverity = 1.f - s->severity;
> + Brettel *cvd = &brettel[s->deficiency];
> + const int element = cvd->element;
> +
> + for (int y = slice_start; y < slice_end; y++) {
> + for (int x = 0; x < width; x++) {
> + float srgb[3], lms[3];
> + float *vector;
> + float projection;
> + float dot3;
> +
> + srgb[0] = sr[x];
> + srgb[1] = sg[x];
> + srgb[2] = sb[x];
> +
> + apply_matrix(lrgb2lms, srgb, lms);
> +
> + dot3 = apply_dot3(cvd->separation, lms);
> + vector = cvd->projection[dot3 > 0.f];
> + projection = apply_dot3(vector, lms);
> + lms[element] = projection * severity + lms[element] * iseverity;
> +
> + apply_matrix(lms2lrgb, lms, srgb);
> +
> + dr[x] = srgb[0];
> + dg[x] = srgb[1];
> + db[x] = srgb[2];
> + }
> +
> + sr += srlinesize;
> + sg += sglinesize;
> + sb += sblinesize;
> + dr += drlinesize;
> + dg += dglinesize;
> + db += dblinesize;
> + }
> +
> + return 0;
> +}
> +
> +static int filter_frame(AVFilterLink *inlink, AVFrame *in)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + AVFilterLink *outlink = ctx->outputs[0];
> + CVDContext *s = ctx->priv;
> + ThreadData td;
> + AVFrame *out;
> +
> + if (in->color_trc != AVCOL_TRC_LINEAR)
> + av_log(s, AV_LOG_WARNING, "Color Visual Deficiency filter works correctly only in linear light.\n");
> +
> + if (av_frame_is_writable(in)) {
> + out = in;
> + } else {
> + out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
> + if (!out) {
> + av_frame_free(&in);
> + return AVERROR(ENOMEM);
> + }
> + av_frame_copy_props(out, in);
> + }
> +
> + td.in = in;
> + td.out = out;
> + ff_filter_execute(ctx, s->do_slice, &td, NULL,
> + FFMIN(in->height, ff_filter_get_nb_threads(ctx)));
> +
> + if (in != out) {
> + av_image_copy_plane(out->data[3], out->linesize[3],
> + in->data[3], in->linesize[3], outlink->w * 4, outlink->h);
> + av_frame_free(&in);
> + }
> +
> + return ff_filter_frame(ctx->outputs[0], out);
> +}
> +
> +static av_cold int config_input(AVFilterLink *inlink)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + CVDContext *s = ctx->priv;
> +
> + s->do_slice = cvd_slice;
> +
> + return 0;
> +}
> +
> +static const AVFilterPad colorvd_inputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .filter_frame = filter_frame,
> + .config_props = config_input,
> + },
> +};
> +
> +static const AVFilterPad colorvd_outputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + },
> +};
> +
> +#define OFFSET(x) offsetof(CVDContext, x)
> +#define VF AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
> +#define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, VF, unit }
> +
> +static const AVOption colorvd_options[] = {
> + { "deficiency", "set the type of deficiency", OFFSET(deficiency), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_DEF-1, VF, "def" },
> + CONST("protan", "", 0, "def"),
> + CONST("deutan", "", 1, "def"),
> + CONST("tritan", "", 2, "def"),
> + { "severity", "set the severity of deficiency", OFFSET(severity), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0, 1, VF },
> + { NULL }
> +};
> +
> +AVFILTER_DEFINE_CLASS(colorvd);
> +
> +const AVFilter ff_vf_colorvd = {
> + .name = "colorvd",
> + .description = NULL_IF_CONFIG_SMALL("Simulate Color Visual Deficiencies in the video stream."),
Then maybe call the filter something more identifiable, like
colordeficiency.
Also, missing docs.
> + .priv_size = sizeof(CVDContext),
> + .priv_class = &colorvd_class,
> + FILTER_INPUTS(colorvd_inputs),
> + FILTER_OUTPUTS(colorvd_outputs),
> + FILTER_PIXFMTS(AV_PIX_FMT_GBRPF32, AV_PIX_FMT_GBRAPF32),
> + .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC | AVFILTER_FLAG_SLICE_THREADS,
> + .process_command = ff_filter_process_command,
> +};
>
More information about the ffmpeg-devel
mailing list