[FFmpeg-devel] [PATCH v3] avfilter: add OpenCL scale filter

Gabriel Machado gabriel_machado at live.com
Tue Mar 27 22:55:21 EEST 2018


> > +__kernel void bilinear(__write_only image2d_t dst,
> > +                       __read_only  image2d_t src)
> > +{
> > +    const sampler_t sampler = (CLK_NORMALIZED_COORDS_TRUE |
> > +                               CLK_ADDRESS_CLAMP_TO_EDGE |
> > +                               CLK_FILTER_LINEAR);
> > +
> > +    int2 coord = {get_global_id(0), get_global_id(1)};
> > +    int2 size = {get_global_size(0), get_global_size(1)};
> > +
> > +    float2 pos = (convert_float2(coord) + 0.5) / convert_float2(size);
> >

>
> Doesn't opencl have an option to use a sampler with non-normalized
> addressing mode?

Using CLK_NORMALIZED_COORDS_FALSE and mapping the output position to the input
position works just as fine:
float2 pos = (convert_float2(coord) + 0.5) * (src_size / dst_size);

Is that what you mean?


> You completely ignored what I said. This function doesn't have a license,
> you can't use it in a modified form. Either rewrite it or remove it.

Rewrote it based on the code in libswscale/utils.c

---
 configure                     |   1 +
 libavfilter/Makefile          |   1 +
 libavfilter/allfilters.c      |   1 +
 libavfilter/opencl/scale.cl   | 113 +++++++++++++++++
 libavfilter/opencl_source.h   |   1 +
 libavfilter/vf_scale_opencl.c | 284 ++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 401 insertions(+)
 create mode 100644 libavfilter/opencl/scale.cl
 create mode 100644 libavfilter/vf_scale_opencl.c

diff --git a/configure b/configure
index 5ccf3ce..4007ee8 100755
--- a/configure
+++ b/configure
@@ -2821,6 +2821,7 @@ v4l2_m2m_deps_any="linux_videodev2_h"
 
 hwupload_cuda_filter_deps="ffnvcodec"
 scale_npp_filter_deps="ffnvcodec libnpp"
+scale_opencl_filter_deps="opencl"
 scale_cuda_filter_deps="cuda_sdk"
 thumbnail_cuda_filter_deps="cuda_sdk"
 
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index a90ca30..6303cbd 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -302,6 +302,7 @@ OBJS-$(CONFIG_SAB_FILTER)                    += vf_sab.o
 OBJS-$(CONFIG_SCALE_FILTER)                  += vf_scale.o scale.o
 OBJS-$(CONFIG_SCALE_CUDA_FILTER)             += vf_scale_cuda.o vf_scale_cuda.ptx.o
 OBJS-$(CONFIG_SCALE_NPP_FILTER)              += vf_scale_npp.o scale.o
+OBJS-$(CONFIG_SCALE_OPENCL_FILTER)           += vf_scale_opencl.o opencl.o opencl/scale.o
 OBJS-$(CONFIG_SCALE_QSV_FILTER)              += vf_scale_qsv.o
 OBJS-$(CONFIG_SCALE_VAAPI_FILTER)            += vf_scale_vaapi.o scale.o vaapi_vpp.o
 OBJS-$(CONFIG_SCALE2REF_FILTER)              += vf_scale.o scale.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 1cf1340..3185b17 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -309,6 +309,7 @@ static void register_all(void)
     REGISTER_FILTER(SCALE,          scale,          vf);
     REGISTER_FILTER(SCALE_CUDA,     scale_cuda,     vf);
     REGISTER_FILTER(SCALE_NPP,      scale_npp,      vf);
+    REGISTER_FILTER(SCALE_OPENCL,   scale_opencl,   vf);
     REGISTER_FILTER(SCALE_QSV,      scale_qsv,      vf);
     REGISTER_FILTER(SCALE_VAAPI,    scale_vaapi,    vf);
     REGISTER_FILTER(SCALE2REF,      scale2ref,      vf);
diff --git a/libavfilter/opencl/scale.cl b/libavfilter/opencl/scale.cl
new file mode 100644
index 0000000..777344e
--- /dev/null
+++ b/libavfilter/opencl/scale.cl
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2018 Gabriel Machado
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+__kernel void neighbor(__write_only image2d_t dst,
+                       __read_only  image2d_t src)
+{
+    const sampler_t sampler = (CLK_NORMALIZED_COORDS_TRUE |
+                               CLK_ADDRESS_CLAMP_TO_EDGE |
+                               CLK_FILTER_NEAREST);
+
+    int2 coord = {get_global_id(0), get_global_id(1)};
+    int2 size = {get_global_size(0), get_global_size(1)};
+
+    float2 pos = (convert_float2(coord) + 0.5) / convert_float2(size);
+
+    float4 c = read_imagef(src, sampler, pos);
+    write_imagef(dst, coord, c);
+}
+
+__kernel void bilinear(__write_only image2d_t dst,
+                       __read_only  image2d_t src)
+{
+    const sampler_t sampler = (CLK_NORMALIZED_COORDS_TRUE |
+                               CLK_ADDRESS_CLAMP_TO_EDGE |
+                               CLK_FILTER_LINEAR);
+
+    int2 coord = {get_global_id(0), get_global_id(1)};
+    int2 size = {get_global_size(0), get_global_size(1)};
+
+    float2 pos = (convert_float2(coord) + 0.5) / convert_float2(size);
+
+    float4 c = read_imagef(src, sampler, pos);
+    write_imagef(dst, coord, c);
+}
+
+float netravali(float t, float B, float C)
+{
+    float d = fabs(t);
+    if (d > 2) {
+        return 0;
+    } else {
+        float dd  = d*d;
+        float ddd = d*dd;
+        if (d < 1) {
+            return ((12 -  9 * B - 6 * C) * ddd +
+                   (-18 + 12 * B + 6 * C) * dd  +
+                     (6 -  2 * B)) / 6;
+        } else {
+            return     ((-B -  6 * C) * ddd +
+                     (6 * B + 30 * C) * dd +
+                   (-12 * B - 48 * C) * d +
+                     (8 * B + 24 * C)) / 6;
+        }
+    }
+}
+
+float4 cubic(float4 c0, float4 c1, float4 c2, float4 c3, float t)
+{
+    float B = 0, C = 0.6; // libswscale default
+    float a = netravali(t + 1, B, C);
+    float b = netravali(t,     B, C);
+    float c = netravali(1 - t, B, C);
+    float d = netravali(2 - t, B, C);
+    return a*c0 + b*c1 + c*c2 + d*c3;
+}
+
+__kernel void bicubic(__write_only image2d_t dst,
+                      __read_only  image2d_t src)
+{
+    const sampler_t sampler = (CLK_NORMALIZED_COORDS_FALSE |
+                               CLK_ADDRESS_CLAMP_TO_EDGE |
+                               CLK_FILTER_NEAREST);
+
+    int2 dst_coord = {get_global_id(0), get_global_id(1)};
+
+    float2 dst_size = {get_global_size(0), get_global_size(1)};
+    float2 src_size = convert_float2(get_image_dim(src));
+
+    float2 uv = convert_float2(dst_coord) / dst_size;
+
+    float2 src_pos = uv * convert_float2(src_size) - 0.5;
+
+    float2 src_coordf;
+    float2 t = fract(src_pos, &src_coordf);
+    int2 src_coord = convert_int2(src_coordf);
+
+#define TEX(x,y) read_imagef(src, sampler, src_coord + (int2){x,y})
+    float4 col = cubic(cubic(TEX(-1,-1), TEX(0,-1), TEX(1,-1), TEX(2,-1), t.x),
+                       cubic(TEX(-1, 0), TEX(0, 0), TEX(1, 0), TEX(2, 0), t.x),
+                       cubic(TEX(-1, 1), TEX(0, 1), TEX(1, 1), TEX(2, 1), t.x),
+                       cubic(TEX(-1, 2), TEX(0, 2), TEX(1, 2), TEX(2, 2), t.x),
+                       t.y);
+#undef TEX
+
+    write_imagef(dst, dst_coord, col);
+}
diff --git a/libavfilter/opencl_source.h b/libavfilter/opencl_source.h
index 4bb9969..e3bb887 100644
--- a/libavfilter/opencl_source.h
+++ b/libavfilter/opencl_source.h
@@ -22,6 +22,7 @@
 extern const char *ff_opencl_source_avgblur;
 extern const char *ff_opencl_source_convolution;
 extern const char *ff_opencl_source_overlay;
+extern const char *ff_opencl_source_scale;
 extern const char *ff_opencl_source_unsharp;
 
 #endif /* AVFILTER_OPENCL_SOURCE_H */
diff --git a/libavfilter/vf_scale_opencl.c b/libavfilter/vf_scale_opencl.c
new file mode 100644
index 0000000..6482162
--- /dev/null
+++ b/libavfilter/vf_scale_opencl.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2018 Gabriel Machado
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/common.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/mem.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+
+#include "avfilter.h"
+#include "internal.h"
+#include "opencl.h"
+#include "opencl_source.h"
+#include "scale.h"
+#include "video.h"
+
+#define F_NEIGHBOR     0
+#define F_BILINEAR     1
+#define F_BICUBIC      2
+
+typedef struct ScaleOpenCLContext {
+    OpenCLFilterContext ocf;
+
+    int              initialised;
+    cl_kernel        kernel;
+    cl_command_queue command_queue;
+
+    char *w_expr;
+    char *h_expr;
+    int algorithm;
+} ScaleOpenCLContext;
+
+static const char *kernel_name[] = {
+    "neighbor",
+    "bilinear",
+    "bicubic"
+};
+
+static int scale_opencl_init(AVFilterContext *avctx)
+{
+    ScaleOpenCLContext *ctx = avctx->priv;
+    cl_int cle;
+    int err;
+
+    err = ff_opencl_filter_load_program(avctx, &ff_opencl_source_scale, 1);
+    if (err < 0)
+        goto fail;
+
+    ctx->command_queue = clCreateCommandQueue(ctx->ocf.hwctx->context,
+                                              ctx->ocf.hwctx->device_id,
+                                              0, &cle);
+    if (!ctx->command_queue) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create OpenCL "
+               "command queue: %d.\n", cle);
+        err = AVERROR(EIO);
+        goto fail;
+    }
+
+    ctx->kernel = clCreateKernel(ctx->ocf.program, kernel_name[ctx->algorithm], &cle);
+    if (!ctx->kernel) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create kernel: %d.\n", cle);
+        err = AVERROR(EIO);
+        goto fail;
+    }
+
+    ctx->initialised = 1;
+
+    return 0;
+
+fail:
+    if (ctx->command_queue)
+        clReleaseCommandQueue(ctx->command_queue);
+    if (ctx->kernel)
+        clReleaseKernel(ctx->kernel);
+    return err;
+}
+
+static int config_input_props(AVFilterLink *inlink)
+{
+    AVFilterContext  *avctx = inlink->dst;
+    AVFilterLink   *outlink = avctx->outputs[0];
+    ScaleOpenCLContext *ctx = avctx->priv;
+    int w, h;
+    int ret;
+
+    if ((ret = ff_scale_eval_dimensions(ctx,
+                                        ctx->w_expr, ctx->h_expr,
+                                        inlink, outlink,
+                                        &w, &h)) < 0)
+        return ret;
+
+    if (((int64_t)h * inlink->w) > INT_MAX  ||
+        ((int64_t)w * inlink->h) > INT_MAX)
+        av_log(ctx, AV_LOG_ERROR, "Rescaled value for width or height is too big.\n");
+
+    ctx->ocf.output_width  = w;
+    ctx->ocf.output_height = h;
+
+    return ff_opencl_filter_config_input(inlink);
+}
+
+static int scale_opencl_filter_frame(AVFilterLink *inlink, AVFrame *input)
+{
+    AVFilterContext     *avctx = inlink->dst;
+    AVFilterLink      *outlink = avctx->outputs[0];
+    ScaleOpenCLContext    *ctx = avctx->priv;
+    AVHWFramesContext *main_fc = (AVHWFramesContext *) inlink->hw_frames_ctx->data;
+    const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(main_fc->sw_format);
+    AVFrame *output = NULL;
+
+    cl_int cle;
+    size_t global_work[2];
+    cl_mem src, dst;
+    int err, p;
+    int x_subsample = 1 << pix_desc->log2_chroma_w;
+    int y_subsample = 1 << pix_desc->log2_chroma_h;
+
+    av_log(ctx, AV_LOG_DEBUG, "Filter input: %s, %ux%u (%"PRId64").\n",
+           av_get_pix_fmt_name(input->format),
+           input->width, input->height, input->pts);
+
+    if (!input->hw_frames_ctx)
+        return AVERROR(EINVAL);
+
+    if (!ctx->initialised) {
+        err = scale_opencl_init(avctx);
+        if (err < 0)
+            goto fail;
+    }
+
+    output = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+    if (!output) {
+        err = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    err = av_frame_copy_props(output, input);
+    if (err < 0)
+        goto fail;
+    output->width  = outlink->w;
+    output->height = outlink->h;
+
+    for (p = 0; p < FF_ARRAY_ELEMS(output->data); p++) {
+        src = (cl_mem) input->data[p];
+        dst = (cl_mem)output->data[p];
+
+        if (!dst)
+            break;
+
+        cle = clSetKernelArg(ctx->kernel, 0, sizeof(cl_mem), &dst);
+        if (cle != CL_SUCCESS) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to set kernel "
+                   "destination image argument: %d.\n", cle);
+            goto fail;
+        }
+        cle = clSetKernelArg(ctx->kernel, 1, sizeof(cl_mem), &src);
+        if (cle != CL_SUCCESS) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to set kernel "
+                   "source image argument: %d.\n", cle);
+            goto fail;
+        }
+
+        global_work[0] = output->width / (p ? x_subsample : 1);
+        global_work[1] = output->height / (p ? y_subsample : 1);
+
+        av_log(avctx, AV_LOG_DEBUG, "Run kernel on plane %d "
+               "(%"SIZE_SPECIFIER"x%"SIZE_SPECIFIER").\n",
+               p, global_work[0], global_work[1]);
+
+        cle = clEnqueueNDRangeKernel(ctx->command_queue, ctx->kernel, 2, NULL,
+                                     global_work, NULL, 0, NULL, NULL);
+        if (cle != CL_SUCCESS) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to enqueue kernel: %d.\n", cle);
+            err = AVERROR(EIO);
+            goto fail;
+        }
+    }
+
+    cle = clFinish(ctx->command_queue);
+    if (cle != CL_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to finish command queue: %d.\n", cle);
+        err = AVERROR(EIO);
+        goto fail;
+    }
+
+    av_frame_free(&input);
+
+    av_log(ctx, AV_LOG_DEBUG, "Filter output: %s, %ux%u (%"PRId64").\n",
+           av_get_pix_fmt_name(output->format),
+           output->width, output->height, output->pts);
+
+    return ff_filter_frame(outlink, output);
+
+fail:
+    clFinish(ctx->command_queue);
+    av_frame_free(&input);
+    av_frame_free(&output);
+    return err;
+}
+
+static av_cold void scale_opencl_uninit(AVFilterContext *avctx)
+{
+    ScaleOpenCLContext *ctx = avctx->priv;
+    cl_int cle;
+
+    if (ctx->kernel) {
+        cle = clReleaseKernel(ctx->kernel);
+        if (cle != CL_SUCCESS)
+            av_log(avctx, AV_LOG_ERROR, "Failed to release "
+                   "kernel: %d.\n", cle);
+    }
+
+    if (ctx->command_queue) {
+        cle = clReleaseCommandQueue(ctx->command_queue);
+        if (cle != CL_SUCCESS)
+            av_log(avctx, AV_LOG_ERROR, "Failed to release "
+                   "command queue: %d.\n", cle);
+    }
+
+    ff_opencl_filter_uninit(avctx);
+}
+
+#define OFFSET(x) offsetof(ScaleOpenCLContext, x)
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
+static const AVOption scale_opencl_options[] = {
+    { "w",             "Output video width",  OFFSET(w_expr),    AV_OPT_TYPE_STRING, { .str = "iw"       }, .flags = FLAGS },
+    { "h",             "Output video height", OFFSET(h_expr),    AV_OPT_TYPE_STRING, { .str = "ih"       }, .flags = FLAGS },
+    { "algorithm",     "Scaling algorithm",   OFFSET(algorithm), AV_OPT_TYPE_INT,    { .i64 = F_BICUBIC  }, INT_MIN, INT_MAX, FLAGS, "algorithm" },
+    { "bilinear",      "bilinear",            0,                 AV_OPT_TYPE_CONST,  { .i64 = F_BILINEAR }, 0, 0, FLAGS, "algorithm" },
+    { "bicubic",       "bicubic",             0,                 AV_OPT_TYPE_CONST,  { .i64 = F_BICUBIC  }, 0, 0, FLAGS, "algorithm" },
+    { "neighbor",      "nearest neighbor",    0,                 AV_OPT_TYPE_CONST,  { .i64 = F_NEIGHBOR }, 0, 0, FLAGS, "algorithm" },
+    { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(scale_opencl);
+
+static const AVFilterPad scale_opencl_inputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .filter_frame = &scale_opencl_filter_frame,
+        .config_props = &config_input_props,
+    },
+    { NULL }
+};
+
+static const AVFilterPad scale_opencl_outputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .config_props = &ff_opencl_filter_config_output,
+    },
+    { NULL }
+};
+
+AVFilter ff_vf_scale_opencl = {
+    .name           = "scale_opencl",
+    .description    = NULL_IF_CONFIG_SMALL("Scale the input video size."),
+    .priv_size      = sizeof(ScaleOpenCLContext),
+    .priv_class     = &scale_opencl_class,
+    .init           = &ff_opencl_filter_init,
+    .uninit         = &scale_opencl_uninit,
+    .query_formats  = &ff_opencl_filter_query_formats,
+    .inputs         = scale_opencl_inputs,
+    .outputs        = scale_opencl_outputs,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
+};
-- 
2.7.4






More information about the ffmpeg-devel mailing list