[FFmpeg-devel] [PATCH V3 1/2] dnn/native: add native support for avg_pool

Guo, Yejun yejun.guo at intel.com
Mon Aug 3 05:20:27 EEST 2020



> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of Ting Fu
> Sent: 2020年7月30日 18:03
> To: ffmpeg-devel at ffmpeg.org
> Subject: [FFmpeg-devel] [PATCH V3 1/2] dnn/native: add native support for
> avg_pool
> 
> Not support pooling strides in channel dimension now.
> It can be tested with the model generated with below python script:
> 
> import tensorflow as tf
> import numpy as np
> import imageio
> 
> in_img = imageio.imread('input_odd.jpg')
> in_img = in_img.astype(np.float32)/255.0
> in_data = in_img[np.newaxis, :]
> 
> x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in')
> x_pool = tf.nn.avg_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')

I don't see input/output channel is set in this function parameter, 
why we need them in struct AvgPoolParams?

> #please alter the params as needed
> y = tf.identity(x_pool, name='dnn_out')
> 
> sess=tf.Session()
> sess.run(tf.global_variables_initializer())
> 
> graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def,
> ['dnn_out'])
> tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False)
> 
> print("image_process.pb generated, please use \
> path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n")
> 
> output = sess.run(y, feed_dict={x: in_data})
> imageio.imsave("out.jpg", np.squeeze(output))
> 
> Signed-off-by: Ting Fu <ting.fu at intel.com>
> ---
>  libavfilter/dnn/Makefile                      |   1 +
>  libavfilter/dnn/dnn_backend_native.h          |   2 +
>  .../dnn/dnn_backend_native_layer_avgpool.c    | 147 ++++++++++++++++++
>  .../dnn/dnn_backend_native_layer_avgpool.h    |  35 +++++
>  .../dnn/dnn_backend_native_layer_conv2d.h     |   3 +-
>  libavfilter/dnn/dnn_backend_native_layers.c   |   2 +
>  tools/python/convert_from_tensorflow.py       |  39 ++++-
>  7 files changed, 226 insertions(+), 3 deletions(-)
>  create mode 100644 libavfilter/dnn/dnn_backend_native_layer_avgpool.c
>  create mode 100644 libavfilter/dnn/dnn_backend_native_layer_avgpool.h
> 
> diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile
> index d90137ec42..e0957073ee 100644
> --- a/libavfilter/dnn/Makefile
> +++ b/libavfilter/dnn/Makefile
> @@ -1,6 +1,7 @@
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_interface.o
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native.o
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native_layers.o
> +OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native_layer_avgpool.o
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native_layer_pad.o
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native_layer_conv2d.o
>  OBJS-$(CONFIG_DNN)                           +=
> dnn/dnn_backend_native_layer_depth2space.o
> diff --git a/libavfilter/dnn/dnn_backend_native.h
> b/libavfilter/dnn/dnn_backend_native.h
> index 62191ffe88..26e9a33387 100644
> --- a/libavfilter/dnn/dnn_backend_native.h
> +++ b/libavfilter/dnn/dnn_backend_native.h
> @@ -43,10 +43,12 @@ typedef enum {
>      DLT_MAXIMUM = 4,
>      DLT_MATH_BINARY = 5,
>      DLT_MATH_UNARY = 6,
> +    DLT_AVG_POOL = 7,
>      DLT_COUNT
>  } DNNLayerType;
> 
>  typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE =
> DOT_INPUT | DOT_OUTPUT} DNNOperandType;
> +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNPaddingParam;
> 
>  typedef struct Layer{
>      DNNLayerType type;
> diff --git a/libavfilter/dnn/dnn_backend_native_layer_avgpool.c
> b/libavfilter/dnn/dnn_backend_native_layer_avgpool.c
> new file mode 100644
> index 0000000000..a6ebb0db8f
> --- /dev/null
> +++ b/libavfilter/dnn/dnn_backend_native_layer_avgpool.c
> @@ -0,0 +1,147 @@
> +/*
> + * Copyright (c) 2020
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * DNN native backend implementation.
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "dnn_backend_native_layer_avgpool.h"
> +
> +int dnn_load_layer_avg_pool(Layer *layer, AVIOContext *model_file_context,
> int file_size, int operands_num)
> +{
> +    AvgPoolParams *avgpool_params;
> +    int dnn_size = 0;
> +    avgpool_params = av_malloc(sizeof(*avgpool_params));
> +    if(!avgpool_params)
> +        return 0;
> +
> +    avgpool_params->strides = (int32_t)avio_rl32(model_file_context);
> +    avgpool_params->padding_method =
> (int32_t)avio_rl32(model_file_context);
> +    avgpool_params->in_channels = (int32_t)avio_rl32(model_file_context);
> +    avgpool_params->out_channels = (int32_t)avio_rl32(model_file_context);
> +    avgpool_params->kernel_size = (int32_t)avio_rl32(model_file_context);
> +    dnn_size += 20;
> +
> +    if (dnn_size > file_size || avgpool_params->in_channels <= 0 ||
> +        avgpool_params->out_channels <= 0 ||
> avgpool_params->kernel_size <= 0 ||
> +        avgpool_params->strides <=0){
> +        av_freep(&avgpool_params);
> +        return 0;
> +    }
> +
> +    layer->params = avgpool_params;
> +    layer->input_operand_indexes[0] =
> (int32_t)avio_rl32(model_file_context);
> +    layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
> +    dnn_size += 8;
> +
> +    if (layer->input_operand_indexes[0] >= operands_num ||
> layer->output_operand_index >= operands_num) {
> +        return 0;
> +    }
> +    return dnn_size;
> +}
> +
> +int dnn_execute_layer_avg_pool(DnnOperand *operands, const int32_t
> *input_operand_indexes,
> +                             int32_t output_operand_index, const void
> *parameters)
> +{
> +    float *output;
> +    int height_end, width_end, height_radius, width_radius, output_height,
> output_width, kernel_area;
> +    int32_t input_operand_index = input_operand_indexes[0];
> +    int number = operands[input_operand_index].dims[0];
> +    int height = operands[input_operand_index].dims[1];
> +    int width = operands[input_operand_index].dims[2];
> +    int channel = operands[input_operand_index].dims[3];
> +    const float *input = operands[input_operand_index].data;
> +    const AvgPoolParams *avgpool_params = (const AvgPoolParams
> *)parameters;
> +
> +    int kernel_strides = avgpool_params->strides;
> +    int src_linesize = width * channel;
> +    DnnOperand *output_operand = &operands[output_operand_index];
> +
> +    /**
> +     * When padding_method = SAME, the tensorflow will only padding the
> hald number of 0 pxiels
> +     * except the remainders.
> +     * Eg: assuming the input height = 1080, the strides = 11, so the
> remainders = 1080 % 11 = 2
> +     *     and if ksize = 5: it will fill (5 - 2) >> 1 = 1 line before the first line of
> input image,
> +     *                       and 5 - 2 - 1 = 2 lines after the last line of
> input image.
> +     *     and if ksize = 7: it will fill (7 - 2) >> 1 = 2 lines before the first line
> of input image,
> +     *                       and 7 - 2 - 2 = 3 lines after the last line of
> input image.
> +     */
> +    if (avgpool_params->padding_method == SAME) {
> +        height_end = height;
> +        width_end = width;
> +        height_radius = avgpool_params->kernel_size - ((height - 1) %
> kernel_strides + 1);
> +        width_radius = avgpool_params->kernel_size - ((width - 1) %
> kernel_strides + 1);
> +        height_radius = height_radius < 0 ? 0 : height_radius >> 1;
> +        width_radius = width_radius < 0 ? 0 : width_radius >> 1;
> +        output_height = ceil(height / (kernel_strides * 1.0));
> +        output_width = ceil(width / (kernel_strides * 1.0));
> +    } else {
[] 
add an assert here, since avg_pool only accepts 'VALID' or 'SAME', while padding_method has three enum.
avassert0(padding_method==VALID)

> +        height_end = height - avgpool_params->kernel_size + 1;
> +        width_end = width - avgpool_params->kernel_size + 1;
> +        height_radius = 0;
> +        width_radius = 0;
> +        output_height = ceil((height - avgpool_params->kernel_size + 1) /
> (kernel_strides * 1.0));
> +        output_width = ceil((width - avgpool_params->kernel_size + 1) /
> (kernel_strides * 1.0));
> +    }
> +
> +    output_operand->dims[0] = number;
> +    output_operand->dims[1] = output_height;
> +    output_operand->dims[2] = output_width;
> +    // not support pooling in channel dimension now
> +    output_operand->dims[3] = channel;
> +    output_operand->data_type =
> operands[input_operand_index].data_type;
> +    output_operand->length =
> calculate_operand_data_length(output_operand);
> +    output_operand->data = av_realloc(output_operand->data,
> output_operand->length);
> +    if (!output_operand->data)
> +        return -1;
> +    output = output_operand->data;
> +
> +    av_assert0(channel == avgpool_params->in_channels);
> +    av_assert0(channel == avgpool_params->out_channels);
> +
> +    for (int y = 0; y < height_end; y += kernel_strides) {
> +        for (int x = 0; x < width_end; x += kernel_strides) {
> +            for (int n_channel = 0; n_channel < channel; ++n_channel) {
> +                output[n_channel] = 0.0;
> +                kernel_area = 0;
> +                for (int kernel_y = 0; kernel_y <
> avgpool_params->kernel_size; ++kernel_y) {
> +                    for (int kernel_x = 0; kernel_x <
> avgpool_params->kernel_size; ++kernel_x) {
> +                        float input_pel;
> +                        int y_pos = y + (kernel_y - height_radius);
> +                        int x_pos = x + (kernel_x - width_radius);
> +                        if (x_pos < 0 || x_pos >= width || y_pos < 0 ||
> y_pos >= height) {
> +                            input_pel = 0.0;
> +                        } else {
> +                            kernel_area++;
> +                            input_pel = input[y_pos * src_linesize +
> x_pos * channel + n_channel];
> +                        }
> +                        output[n_channel] += input_pel;
> +                    }
> +                }
> +                output[n_channel] /= kernel_area;
> +            }
> +            output += avgpool_params->out_channels;
> +        }
> +    }
> +
> +    return 0;
> +}
> diff --git a/libavfilter/dnn/dnn_backend_native_layer_avgpool.h
> b/libavfilter/dnn/dnn_backend_native_layer_avgpool.h
> new file mode 100644
> index 0000000000..0b37a8f64b
> --- /dev/null
> +++ b/libavfilter/dnn/dnn_backend_native_layer_avgpool.h
> @@ -0,0 +1,35 @@
> +/*
> + * Copyright (c) 2018 Sergey Lavrushkin
[] 
remove name here



More information about the ffmpeg-devel mailing list