[FFmpeg-devel] [PATCH V2 3/3] dnn: convert tf.pad to native model in python script, and load/execute it in the c code.

Pedro Arthur bygrandao at gmail.com
Mon Jul 29 18:36:31 EEST 2019


LGTM.
Pushed, thanks!

Em dom, 28 de jul de 2019 às 23:00, Guo, Yejun <yejun.guo at intel.com> escreveu:
>
> since tf.pad is enabled, the conv2d(valid) changes back to its original behavior.
>
> Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
> ---
>  libavfilter/dnn/dnn_backend_native.c    | 35 +++++++++++++++++++++++++++++++++
>  libavfilter/dnn/dnn_backend_native.h    |  2 +-
>  tools/python/convert_from_tensorflow.py | 23 +++++++++++++++++-----
>  3 files changed, 54 insertions(+), 6 deletions(-)
>
> diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
> index 82e900b..09c583b 100644
> --- a/libavfilter/dnn/dnn_backend_native.c
> +++ b/libavfilter/dnn/dnn_backend_native.c
> @@ -25,6 +25,7 @@
>
>  #include "dnn_backend_native.h"
>  #include "libavutil/avassert.h"
> +#include "dnn_backend_native_layer_pad.h"
>
>  static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
>  {
> @@ -32,6 +33,7 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
>      InputParams *input_params;
>      ConvolutionalParams *conv_params;
>      DepthToSpaceParams *depth_to_space_params;
> +    LayerPadParams *pad_params;
>      int cur_width, cur_height, cur_channels;
>      int32_t layer;
>
> @@ -77,6 +79,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
>              cur_height *= depth_to_space_params->block_size;
>              cur_width *= depth_to_space_params->block_size;
>              break;
> +        case MIRROR_PAD:
> +            pad_params = (LayerPadParams *)network->layers[layer].params;
> +            cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
> +            cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
> +            cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
> +            break;
>          default:
>              return DNN_ERROR;
>          }
> @@ -110,6 +118,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
>      DNNLayerType layer_type;
>      ConvolutionalParams *conv_params;
>      DepthToSpaceParams *depth_to_space_params;
> +    LayerPadParams *pad_params;
>
>      model = av_malloc(sizeof(DNNModel));
>      if (!model){
> @@ -207,6 +216,23 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
>              network->layers[layer].type = DEPTH_TO_SPACE;
>              network->layers[layer].params = depth_to_space_params;
>              break;
> +        case MIRROR_PAD:
> +            pad_params = av_malloc(sizeof(LayerPadParams));
> +            if (!pad_params){
> +                avio_closep(&model_file_context);
> +                ff_dnn_free_model_native(&model);
> +                return NULL;
> +            }
> +            pad_params->mode = (int32_t)avio_rl32(model_file_context);
> +            dnn_size += 4;
> +            for (i = 0; i < 4; ++i) {
> +                pad_params->paddings[i][0] = avio_rl32(model_file_context);
> +                pad_params->paddings[i][1] = avio_rl32(model_file_context);
> +                dnn_size += 8;
> +            }
> +            network->layers[layer].type = MIRROR_PAD;
> +            network->layers[layer].params = pad_params;
> +            break;
>          default:
>              avio_closep(&model_file_context);
>              ff_dnn_free_model_native(&model);
> @@ -314,6 +340,7 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
>      InputParams *input_params;
>      ConvolutionalParams *conv_params;
>      DepthToSpaceParams *depth_to_space_params;
> +    LayerPadParams *pad_params;
>
>      if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){
>          return DNN_ERROR;
> @@ -348,6 +375,14 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
>              cur_width *= depth_to_space_params->block_size;
>              cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
>              break;
> +        case MIRROR_PAD:
> +            pad_params = (LayerPadParams *)network->layers[layer].params;
> +            dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output,
> +                                  pad_params, 1, cur_height, cur_width, cur_channels);
> +            cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
> +            cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
> +            cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
> +            break;
>          case INPUT:
>              return DNN_ERROR;
>          }
> diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
> index 8ef1855..b6f9533 100644
> --- a/libavfilter/dnn/dnn_backend_native.h
> +++ b/libavfilter/dnn/dnn_backend_native.h
> @@ -30,7 +30,7 @@
>  #include "../dnn_interface.h"
>  #include "libavformat/avio.h"
>
> -typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
> +typedef enum {INPUT, CONV, DEPTH_TO_SPACE, MIRROR_PAD} DNNLayerType;
>
>  typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
>
> diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py
> index 37049e5..041c82c 100644
> --- a/tools/python/convert_from_tensorflow.py
> +++ b/tools/python/convert_from_tensorflow.py
> @@ -23,9 +23,6 @@ import sys, struct
>
>  __all__ = ['convert_from_tensorflow']
>
> -# as the first step to be compatible with vf_sr, it is not general.
> -# it will be refined step by step.
> -
>  class TFConverter:
>      def __init__(self, graph_def, nodes, outfile):
>          self.graph_def = graph_def
> @@ -36,9 +33,10 @@ class TFConverter:
>          self.name_node_dict = {}
>          self.edges = {}
>          self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'LeakyRelu':4}
> -        self.conv_paddings = {'VALID':2, 'SAME':1}
> +        self.conv_paddings = {'VALID':0, 'SAME':1}
>          self.converted_nodes = set()
> -        self.op2code = {'Conv2D':1, 'DepthToSpace':2}
> +        self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3}
> +        self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
>
>
>      def dump_for_tensorboard(self):
> @@ -101,6 +99,19 @@ class TFConverter:
>          self.converted_nodes.add(node.name)
>
>
> +    def dump_mirrorpad_to_file(self, node, f):
> +        assert(node.op == 'MirrorPad')
> +        self.layer_number = self.layer_number + 1
> +        mode = node.attr['mode'].s
> +        mode = self.mirrorpad_mode[mode.decode("utf-8")]
> +        np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f)
> +        pnode = self.name_node_dict[node.input[1]]
> +        self.converted_nodes.add(pnode.name)
> +        paddings = pnode.attr['value'].tensor.tensor_content
> +        f.write(paddings)
> +        self.converted_nodes.add(node.name)
> +
> +
>      def generate_layer_number(self):
>          # in current hard code implementation, the layer number is the first data written to the native model file
>          # it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility
> @@ -118,6 +129,8 @@ class TFConverter:
>                  self.dump_conv2d_to_file(node, f)
>              elif node.op == 'DepthToSpace':
>                  self.dump_depth2space_to_file(node, f)
> +            elif node.op == 'MirrorPad':
> +                self.dump_mirrorpad_to_file(node, f)
>
>
>      def dump_to_file(self):
> --
> 2.7.4
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".


More information about the ffmpeg-devel mailing list