[FFmpeg-devel] [PATCH v2 11/13] scale_vulkan: add support for basic Debayering
Lynne
dev at lynne.ee
Sat Jul 12 21:51:20 EEST 2025
---
libavfilter/vf_scale_vulkan.c | 60 +++++++++++++++++--
libavfilter/vulkan/Makefile | 1 +
libavfilter/vulkan/debayer.comp | 102 ++++++++++++++++++++++++++++++++
3 files changed, 159 insertions(+), 4 deletions(-)
create mode 100644 libavfilter/vulkan/debayer.comp
diff --git a/libavfilter/vf_scale_vulkan.c b/libavfilter/vf_scale_vulkan.c
index 46b1476933..5cb09ac385 100644
--- a/libavfilter/vf_scale_vulkan.c
+++ b/libavfilter/vf_scale_vulkan.c
@@ -27,6 +27,8 @@
#include "colorspace.h"
#include "video.h"
+extern const char *ff_source_debayer_comp;
+
enum ScalerFunc {
F_BILINEAR = 0,
F_NEAREST,
@@ -34,6 +36,13 @@ enum ScalerFunc {
F_NB,
};
+enum DebayerFunc {
+ DB_BILINEAR = 0,
+ DB_BILINEAR_HQ,
+
+ DB_NB,
+};
+
typedef struct ScaleVulkanContext {
FFVulkanContext vkctx;
@@ -58,6 +67,7 @@ typedef struct ScaleVulkanContext {
enum ScalerFunc scaler;
enum AVColorRange out_range;
+ enum DebayerFunc debayer;
} ScaleVulkanContext;
static const char scale_bilinear[] = {
@@ -184,6 +194,25 @@ static int init_scale_shader(ScaleVulkanContext *s, FFVulkanShader *shd,
return 0;
}
+static int init_debayer_shader(ScaleVulkanContext *s, FFVulkanShader *shd,
+ FFVulkanDescriptorSetBinding *desc, AVFrame *in)
+{
+ GLSLD(ff_source_debayer_comp);
+
+ GLSLC(0, void main(void));
+ GLSLC(0, { );
+ if (s->debayer == DB_BILINEAR)
+ GLSLC(1, debayer_bilinear(););
+ else if (s->debayer == DB_BILINEAR_HQ)
+ GLSLC(1, debayer_bilinear_hq(););
+ GLSLC(0, } );
+
+ shd->lg_size[0] <<= 1;
+ shd->lg_size[1] <<= 1;
+
+ return 0;
+}
+
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
{
int err;
@@ -197,6 +226,7 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
FFVkSPIRVCompiler *spv;
FFVulkanDescriptorSetBinding *desc;
+ int debayer = s->vkctx.input_format == AV_PIX_FMT_BAYER_RGGB16;
int in_planes = av_pix_fmt_count_planes(s->vkctx.input_format);
switch (s->scaler) {
@@ -222,7 +252,10 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
}
RET(ff_vk_exec_pool_init(vkctx, s->qf, &s->e, s->qf->num*4, 0, 0, 0, NULL));
- RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, sampler_mode));
+
+ if (!debayer)
+ RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, sampler_mode));
+
RET(ff_vk_shader_init(vkctx, &s->shd, "scale",
VK_SHADER_STAGE_COMPUTE_BIT,
NULL, 0,
@@ -232,7 +265,13 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
desc = (FFVulkanDescriptorSetBinding []) {
{
.name = "input_img",
- .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+ .type = debayer ?
+ VK_DESCRIPTOR_TYPE_STORAGE_IMAGE :
+ VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+ .mem_layout = debayer ?
+ ff_vk_shader_rep_fmt(s->vkctx.input_format, FF_VK_REP_FLOAT) :
+ NULL,
+ .mem_quali = "readonly",
.dimensions = 2,
.elems = in_planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
@@ -263,7 +302,10 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
ff_vk_shader_add_push_const(&s->shd, 0, sizeof(s->opts),
VK_SHADER_STAGE_COMPUTE_BIT);
- err = init_scale_shader(s, shd, desc, in);
+ if (debayer)
+ err = init_debayer_shader(s, shd, desc, in);
+ else
+ err = init_scale_shader(s, shd, desc, in);
if (err < 0)
goto fail;
@@ -361,7 +403,14 @@ static int scale_vulkan_config_output(AVFilterLink *outlink)
s->vkctx.output_format = s->vkctx.input_format;
}
- if (s->vkctx.output_format != s->vkctx.input_format) {
+ if (s->vkctx.input_format == AV_PIX_FMT_BAYER_RGGB16) {
+ if (s->vkctx.output_format == s->vkctx.input_format) {
+ s->vkctx.output_format = AV_PIX_FMT_RGBA64;
+ } else if (!ff_vk_mt_is_np_rgb(s->vkctx.output_format)) {
+ av_log(avctx, AV_LOG_ERROR, "Unsupported output format for debayer\n");
+ return AVERROR(EINVAL);
+ }
+ } else if (s->vkctx.output_format != s->vkctx.input_format) {
if (!ff_vk_mt_is_np_rgb(s->vkctx.input_format)) {
av_log(avctx, AV_LOG_ERROR, "Unsupported input format for conversion\n");
return AVERROR(EINVAL);
@@ -406,6 +455,9 @@ static const AVOption scale_vulkan_options[] = {
{ "scaler", "Scaler function", OFFSET(scaler), AV_OPT_TYPE_INT, {.i64 = F_BILINEAR}, 0, F_NB, .flags = FLAGS, .unit = "scaler" },
{ "bilinear", "Bilinear interpolation (fastest)", 0, AV_OPT_TYPE_CONST, {.i64 = F_BILINEAR}, 0, 0, .flags = FLAGS, .unit = "scaler" },
{ "nearest", "Nearest (useful for pixel art)", 0, AV_OPT_TYPE_CONST, {.i64 = F_NEAREST}, 0, 0, .flags = FLAGS, .unit = "scaler" },
+ { "debayer", "Debayer algorithm to use", OFFSET(debayer), AV_OPT_TYPE_INT, {.i64 = DB_BILINEAR_HQ}, 0, DB_NB, .flags = FLAGS, .unit = "debayer" },
+ { "bilinear", "Bilinear debayering (fastest)", 0, AV_OPT_TYPE_CONST, {.i64 = DB_BILINEAR}, 0, 0, .flags = FLAGS, .unit = "debayer" },
+ { "bilinear_hq", "Bilinear debayering (high quality)", 0, AV_OPT_TYPE_CONST, {.i64 = DB_BILINEAR_HQ}, 0, 0, .flags = FLAGS, .unit = "debayer" },
{ "format", "Output video format (software format of hardware frames)", OFFSET(out_format_string), AV_OPT_TYPE_STRING, .flags = FLAGS },
{ "out_range", "Output colour range (from 0 to 2) (default 0)", OFFSET(out_range), AV_OPT_TYPE_INT, {.i64 = AVCOL_RANGE_UNSPECIFIED}, AVCOL_RANGE_UNSPECIFIED, AVCOL_RANGE_JPEG, .flags = FLAGS, .unit = "range" },
{ "full", "Full range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_JPEG }, 0, 0, FLAGS, .unit = "range" },
diff --git a/libavfilter/vulkan/Makefile b/libavfilter/vulkan/Makefile
index 573eee32c7..c77aaf4f6b 100644
--- a/libavfilter/vulkan/Makefile
+++ b/libavfilter/vulkan/Makefile
@@ -4,6 +4,7 @@ clean::
$(RM) $(GEN_CLEANSUFFIXES:%=libavfilter/vulkan/%)
OBJS-$(CONFIG_BWDIF_VULKAN_FILTER) += vulkan/bwdif.o
+OBJS-$(CONFIG_SCALE_VULKAN_FILTER) += vulkan/debayer.o
VULKAN = $(subst $(SRC_PATH)/,,$(wildcard $(SRC_PATH)/libavfilter/vulkan/*.comp))
.SECONDARY: $(VULKAN:.comp=.c)
diff --git a/libavfilter/vulkan/debayer.comp b/libavfilter/vulkan/debayer.comp
new file mode 100644
index 0000000000..c86c2f5eec
--- /dev/null
+++ b/libavfilter/vulkan/debayer.comp
@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright (c) 2025 Lynne <dev at lynne.ee>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#define LD(xo, yo) \
+ (imageLoad(input_img[0], pos + ivec2((xo), (yo))).r)
+
+void debayer_bilinear(void)
+{
+ ivec2 pos = ivec2(gl_GlobalInvocationID.xy) << 1;
+
+ /* R basis */
+ vec4 tl = vec4(LD(0, 0),
+ (LD(1, 0) + LD(-1, 0) + LD(0, 1) + LD(0, -1)) / 4.0f,
+ (LD(-1, -1) + LD(1, 1) + LD(-1, 1) + LD(1, -1)) / 4.0f,
+ 1.0f);
+ imageStore(output_img[0], pos, tl);
+
+ /* G1 basis */
+ vec4 tr = vec4((LD(2, 0) + LD(0, 0)) / 2.0f,
+ LD(1, 0),
+ (LD(1, 1) + LD(1, -1)) / 2.0f,
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(1, 0), tr);
+
+ /* G2 basis */
+ vec4 bl = vec4((LD(0, 2) + LD(0, 0)) / 2.0f,
+ LD(0, 1),
+ (LD(1, 1) + LD(-1, 1)) / 2.0f,
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(0, 1), bl);
+
+ /* B basis */
+ vec4 br = vec4((LD(0, 0) + LD(2, 2) + LD(0, 2) + LD(2, 0)) / 4.0f,
+ (LD(2, 1) + LD(0, 1) + LD(1, 2) + LD(1, 0)) / 4.0f,
+ LD(1, 1),
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(1, 1), br);
+}
+
+void debayer_bilinear_hq(void)
+{
+ ivec2 pos = ivec2(gl_GlobalInvocationID.xy) << 1;
+
+ /* R basis */
+ vec4 tl = vec4(LD(0, 0),
+ (4.0f*LD(0, 0) + 2.0f*(LD(0, -1) + LD(0, 1) + LD(-1, 0) + LD(1, 0)) -
+ (LD(0, -2) + LD(0, 2) + LD(-2, 0) + LD(2, 0))) / 8.0f,
+ (12.0f*LD(0, 0) + 4.0f*(LD(-1, -1) + LD(-1, 1) + LD(1, -1) + LD(1, 1)) -
+ 3.0f*(LD(0, -2) + LD(0, 2) + LD(-2, 0) + LD(2, 0))) / 16.0f,
+ 1.0f);
+ imageStore(output_img[0], pos, tl);
+
+ /* G1 basis */
+ vec4 tr = vec4((10.0f*LD(1, 0) + 8.0f*(LD(0, 0) + LD(2, 0)) -
+ 2.0f*(LD(0, -1) + LD(2, 1) + LD(0, 1) + LD(2, -1) + LD(-1, 0) + LD(3, 0)) +
+ LD(1, -2) + LD(1, 2)) / 16.0f,
+ LD(1, 0),
+ (10.0f*LD(1, 0) + 8.0f*(LD(1, -1) + LD(1, 1)) -
+ 2.0f*(LD(0, -1) + LD(0, 1) + LD(2, -1) + LD(2, 1) + LD(1, -2) + LD(1, 2)) +
+ LD(-1, 0) + LD(3, 0)) / 16.0f,
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(1, 0), tr);
+
+
+ /* G2 basis */
+ vec4 bl = vec4((10.0f*LD(0, 1) + 8.0f*(LD(0, 0) + LD(0, 2)) -
+ 2.0f*(LD(-1, 0) + LD(-1, 2) + LD(1, 0) + LD(1, 2) + LD(0, -1) + LD(0, 3)) +
+ LD(-2, 1) + LD(2, 1)) / 16.0f,
+ LD(0, 1),
+ (10.0f*LD(0, 1) + 8.0f*(LD(-1, 1) + LD(1, 1)) -
+ 2.0f*(LD(-1, 0) + LD(1, 2) + LD(-1, 2) + LD(1, 0) + LD(-2, 1) + LD(2, 1)) +
+ LD(0, -1) + LD(0, 3)) / 16.0f,
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(0, 1), bl);
+
+ /* B basis */
+ vec4 br = vec4((12.0f*LD(1, 1) + 4.0f*(LD(0, 0) + LD(0, 2) + LD(2, 0) + LD(2, 2)) -
+ 3.0f*(LD(1, -1) + LD(1, 3) + LD(-1, 1) + LD(3, 1))) / 16.0f,
+ (4.0f*LD(1, 1) + 2.0f*(LD(1, 0) + LD(1, 2) + LD(0, 1) + LD(2, 1)) -
+ (LD(1, -1) + LD(1, 3) + LD(-1, 1) + LD(3, 1))) / 8.0f,
+ LD(1, 1),
+ 1.0f);
+ imageStore(output_img[0], pos + ivec2(1, 1), br);
+}
--
2.50.0
More information about the ffmpeg-devel
mailing list