[FFmpeg-cvslog] dnn/native: add native support for 'mul'

[Guo, Yejun]

ffmpeg | branch: master | Guo, Yejun <yejun.guo at intel.com> | Sat Apr 11 13:22:24 2020 +0800| [ef79408e975057e30d8f4afceaa89702fd5f27da] | committer: Guo, Yejun

dnn/native: add native support for 'mul'

it can be tested with model file generated from above python script:

import tensorflow as tf
import numpy as np
import imageio

in_img = imageio.imread('input.jpg')
in_img = in_img.astype(np.float32)/255.0
in_data = in_img[np.newaxis, :]

x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in')
z1 = 0.5 + 0.3 * x
z2 = z1 * 4
z3 = z2 - x - 2.0
y = tf.identity(z3, name='dnn_out')

sess=tf.Session()
sess.run(tf.global_variables_initializer())

graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out'])
tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False)

print("image_process.pb generated, please use \
path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n")

output = sess.run(y, feed_dict={x: in_data})
imageio.imsave("out.jpg", np.squeeze(output))

Signed-off-by: Guo, Yejun <yejun.guo at intel.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ef79408e975057e30d8f4afceaa89702fd5f27da
---

 libavfilter/dnn/dnn_backend_native_layer_mathbinary.c | 13 +++++++++++++
 libavfilter/dnn/dnn_backend_native_layer_mathbinary.h |  1 +
 tools/python/convert_from_tensorflow.py               |  4 +++-
 tools/python/convert_header.py                        |  2 +-
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
index 3fe337f730..222941e952 100644
--- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
@@ -120,6 +120,19 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope
             }
         }
         return 0;
+    case DMBO_MUL:
+        if (params->input0_broadcast || params->input1_broadcast) {
+            for (int i = 0; i < dims_count; ++i) {
+                dst[i] = params->v * src[i];
+            }
+        } else {
+            const DnnOperand *input1 = &operands[input_operand_indexes[1]];
+            const float *src1 = input1->data;
+            for (int i = 0; i < dims_count; ++i) {
+                dst[i] = src[i] * src1[i];
+            }
+        }
+        return 0;
     default:
         return -1;
     }
diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h
index 3c5bc6b2e1..d58b48c747 100644
--- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h
@@ -33,6 +33,7 @@
 typedef enum {
     DMBO_SUB = 0,
     DMBO_ADD = 1,
+    DMBO_MUL = 2,
     DMBO_COUNT
 } DNNMathBinaryOperation;
 
diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py
index 9a495c0a9e..dc3b4e381d 100644
--- a/tools/python/convert_from_tensorflow.py
+++ b/tools/python/convert_from_tensorflow.py
@@ -71,7 +71,7 @@ class TFConverter:
         self.conv2d_scope_names = set()
         self.conv2d_scopename_inputname_dict = {}
         self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5}
-        self.mathbin2code = {'Sub':0, 'Add':1}
+        self.mathbin2code = {'Sub':0, 'Add':1, 'Mul':2}
         self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
         self.name_operand_dict = {}
 
@@ -309,6 +309,8 @@ class TFConverter:
                 self.dump_mathbinary_to_file(node, f)
             elif node.op == 'Add':
                 self.dump_mathbinary_to_file(node, f)
+            elif node.op == 'Mul':
+                self.dump_mathbinary_to_file(node, f)
 
 
     def dump_operands_to_file(self, f):
diff --git a/tools/python/convert_header.py b/tools/python/convert_header.py
index 70270225f1..87899fe72c 100644
--- a/tools/python/convert_header.py
+++ b/tools/python/convert_header.py
@@ -23,4 +23,4 @@ str = 'FFMPEGDNNNATIVE'
 major = 1
 
 # increase minor when we don't have to re-convert the model file
-minor = 2
+minor = 3