[FFmpeg-devel] [PATCH] libavfilter: Add multiple padding methods in FFmpeg dnn native mode.

xwmeng at pku.edu.cn xwmeng at pku.edu.cn
Wed May 8 12:33:56 EEST 2019




This patch is for the support of derain filter project in GSoC. It adds supports for the following operations: 




 (1) Conv padding method: "SAME", "VALID" and "SAME_CLAMP_TO_EDGE"




These operations are all needed in derain filter. As we discussed before, the "SAME_CLAMP_TO_EDGE" method is the same as dnn native padding method in the current implementation. And the sr model generation code should be changed if mutiple padding method supports added. So I sent a PR (https://github.com/HighVoltageRocknRoll/sr/pull/4)to the original sr repo(https://github.com/HighVoltageRocknRoll/sr).



From c0724bb304a6f4c3ca935cccda5b810e5c4eceb1 Mon Sep 17 00:00:00 2001
From: Xuewei Meng <xwmeng at pku.edu.cn>
Date: Wed, 8 May 2019 17:32:30 +0800
Subject: [PATCH] Add multiple padding method in dnn native

Signed-off-by: Xuewei Meng <xwmeng at pku.edu.cn>
---
 libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
 libavfilter/dnn_backend_native.h |  3 ++
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
index 70d857f5f2..b7c0508d91 100644
--- a/libavfilter/dnn_backend_native.c
+++ b/libavfilter/dnn_backend_native.c
@@ -59,6 +59,12 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, DNNDat
                 return DNN_ERROR;
             }
             cur_channels = conv_params->output_num;
+
+            if(conv_params->padding_method == VALID){
+                int pad_size = conv_params->kernel_size - 1;
+                cur_height -= pad_size;
+                cur_width -= pad_size;
+            }
             break;
         case DEPTH_TO_SPACE:
             depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
@@ -75,6 +81,10 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, DNNDat
         if (network->layers[layer].output){
             av_freep(&network->layers[layer].output);
         }
+
+        if(cur_height <= 0 || cur_width <= 0)
+            return DNN_ERROR; 
+        
         network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
         if (!network->layers[layer].output){
             return DNN_ERROR;
@@ -157,13 +167,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
                 ff_dnn_free_model_native(&model);
                 return NULL;
             }
+            conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
             conv_params->activation = (int32_t)avio_rl32(model_file_context);
             conv_params->input_num = (int32_t)avio_rl32(model_file_context);
             conv_params->output_num = (int32_t)avio_rl32(model_file_context);
             conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
             kernel_size = conv_params->input_num * conv_params->output_num *
                           conv_params->kernel_size * conv_params->kernel_size;
-            dnn_size += 16 + (kernel_size + conv_params->output_num << 2);
+            dnn_size += 20 + (kernel_size + conv_params->output_num << 2);
             if (dnn_size > file_size || conv_params->input_num <= 0 ||
                 conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
                 avio_closep(&model_file_context);
@@ -221,23 +232,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
 
 static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
 {
-    int y, x, n_filter, ch, kernel_y, kernel_x;
     int radius = conv_params->kernel_size >> 1;
     int src_linesize = width * conv_params->input_num;
     int filter_linesize = conv_params->kernel_size * conv_params->input_num;
     int filter_size = conv_params->kernel_size * filter_linesize;
+    int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0;
 
-    for (y = 0; y < height; ++y){
-        for (x = 0; x < width; ++x){
-            for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
+    for (int y = pad_size; y < height - pad_size; ++y){
+        for (int x = pad_size; x < width - pad_size; ++x){
+            for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
                 output[n_filter] = conv_params->biases[n_filter];
-                for (ch = 0; ch < conv_params->input_num; ++ch){
-                    for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
-                        for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
-                            output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
-                                                      CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] *
-                                                conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
-                                                                    kernel_x * conv_params->input_num + ch];
+
+                for (int ch = 0; ch < conv_params->input_num; ++ch){
+                    for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
+                        for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
+                            float input_pel;
+                            if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){
+                                int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height);
+                                int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width);
+                                input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+                            }else{
+                                int y_pos = y + kernel_y - radius;
+                                int x_pos = x + kernel_x - radius;
+                                input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : 
+                                                   input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+                            }
+
+
+                            output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
+                                                                                kernel_x * conv_params->input_num + ch];
                         }
                     }
                 }
@@ -308,6 +331,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
             conv_params = (ConvolutionalParams *)network->layers[layer].params;
             convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
             cur_channels = conv_params->output_num;
+            if(conv_params->padding_method == VALID){
+                int pad_size = conv_params->kernel_size - 1;
+                cur_height -= pad_size;
+                cur_width -= pad_size;
+            }
             break;
         case DEPTH_TO_SPACE:
             depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
index 51d4cac955..a609e09754 100644
--- a/libavfilter/dnn_backend_native.h
+++ b/libavfilter/dnn_backend_native.h
@@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
 
 typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
 
+typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNPaddingFunc;
+
 typedef struct Layer{
     DNNLayerType type;
     float *output;
@@ -43,6 +45,7 @@ typedef struct Layer{
 typedef struct ConvolutionalParams{
     int32_t input_num, output_num, kernel_size;
     DNNActivationFunc activation;
+    DNNPaddingFunc padding_method;
     float *kernel;
     float *biases;
 } ConvolutionalParams;
-- 
2.17.1






More information about the ffmpeg-devel mailing list