[FFmpeg-devel] [PATCH 2/2] lavfi/dnn: refine code to separate processing and detection in backends

[Guo, Yejun]

> -----Original Message-----
> From: Guo, Yejun <yejun.guo at intel.com>
> Sent: Monday, May 17, 2021 1:54 PM
> To: ffmpeg-devel at ffmpeg.org
> Cc: Guo, Yejun <yejun.guo at intel.com>
> Subject: [PATCH 2/2] lavfi/dnn: refine code to separate processing and
> detection in backends
> 
> ---
>  libavfilter/dnn/dnn_backend_native.c   |  2 +-
>  libavfilter/dnn/dnn_backend_openvino.c |  6 ++++--
>  libavfilter/dnn/dnn_backend_tf.c       | 20 +++++++++++++++-----
>  libavfilter/dnn/dnn_io_proc.c          | 18 ++----------------
>  libavfilter/dnn/dnn_io_proc.h          |  3 ++-
>  5 files changed, 24 insertions(+), 25 deletions(-)
> 
> diff --git a/libavfilter/dnn/dnn_backend_native.c
> b/libavfilter/dnn/dnn_backend_native.c
> index b5f1c16538..a6be27f1fd 100644
> --- a/libavfilter/dnn/dnn_backend_native.c
> +++ b/libavfilter/dnn/dnn_backend_native.c
> @@ -314,7 +314,7 @@ static DNNReturnType execute_model_native(const
> DNNModel *model, const char *inp
>          if (native_model->model->frame_pre_proc != NULL) {
>              native_model->model->frame_pre_proc(in_frame, &input,
> native_model->model->filter_ctx);
>          } else {
> -            ff_proc_from_frame_to_dnn(in_frame, &input, native_model-
> >model->func_type, ctx);
> +            ff_proc_from_frame_to_dnn(in_frame, &input, ctx);
>          }
>      }
> 
> diff --git a/libavfilter/dnn/dnn_backend_openvino.c
> b/libavfilter/dnn/dnn_backend_openvino.c
> index 1ff8a720b9..e0781e854a 100644
> --- a/libavfilter/dnn/dnn_backend_openvino.c
> +++ b/libavfilter/dnn/dnn_backend_openvino.c
> @@ -186,15 +186,17 @@ static DNNReturnType
> fill_model_input_ov(OVModel *ov_model, RequestItem *request
>          task = inference->task;
>          switch (task->ov_model->model->func_type) {
>          case DFT_PROCESS_FRAME:
> -        case DFT_ANALYTICS_DETECT:
>              if (task->do_ioproc) {
>                  if (ov_model->model->frame_pre_proc != NULL) {
>                      ov_model->model->frame_pre_proc(task->in_frame, &input,
> ov_model->model->filter_ctx);
>                  } else {
> -                    ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model-
> >model->func_type, ctx);
> +                    ff_proc_from_frame_to_dnn(task->in_frame, &input,
> + ctx);
>                  }
>              }
>              break;
> +        case DFT_ANALYTICS_DETECT:
> +            ff_frame_to_dnn_detect(task->in_frame, &input, ctx);
> +            break;
>          case DFT_ANALYTICS_CLASSIFY:
>              ff_frame_to_dnn_classify(task->in_frame, &input, inference-
> >bbox_index, ctx);
>              break;
> diff --git a/libavfilter/dnn/dnn_backend_tf.c
> b/libavfilter/dnn/dnn_backend_tf.c
> index 5908aeb359..4c16c2bdb0 100644
> --- a/libavfilter/dnn/dnn_backend_tf.c
> +++ b/libavfilter/dnn/dnn_backend_tf.c
> @@ -763,12 +763,22 @@ static DNNReturnType execute_model_tf(const
> DNNModel *model, const char *input_n
>      }
>      input.data = (float *)TF_TensorData(input_tensor);
> 
> -    if (do_ioproc) {
> -        if (tf_model->model->frame_pre_proc != NULL) {
> -            tf_model->model->frame_pre_proc(in_frame, &input, tf_model-
> >model->filter_ctx);
> -        } else {
> -            ff_proc_from_frame_to_dnn(in_frame, &input, tf_model->model-
> >func_type, ctx);
> +    switch (tf_model->model->func_type) {
> +    case DFT_PROCESS_FRAME:
> +        if (do_ioproc) {
> +            if (tf_model->model->frame_pre_proc != NULL) {
> +                tf_model->model->frame_pre_proc(in_frame, &input, tf_model-
> >model->filter_ctx);
> +            } else {
> +                ff_proc_from_frame_to_dnn(in_frame, &input, ctx);
> +            }
>          }
> +        break;
> +    case DFT_ANALYTICS_DETECT:
> +        ff_frame_to_dnn_detect(in_frame, &input, ctx);
> +        break;
> +    default:
> +        avpriv_report_missing_feature(ctx, "model function type %d",
> tf_model->model->func_type);
> +        break;
>      }
> 
>      tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs)); diff --git
> a/libavfilter/dnn/dnn_io_proc.c b/libavfilter/dnn/dnn_io_proc.c index
> 1e2bef3f9a..e01661103b 100644
> --- a/libavfilter/dnn/dnn_io_proc.c
> +++ b/libavfilter/dnn/dnn_io_proc.c
> @@ -94,7 +94,7 @@ DNNReturnType
> ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *l
>      return DNN_SUCCESS;
>  }
> 
> -static DNNReturnType
> proc_from_frame_to_dnn_frameprocessing(AVFrame *frame, DNNData
> *input, void *log_ctx)
> +DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData
> *input,
> +void *log_ctx)
>  {
>      struct SwsContext *sws_ctx;
>      int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
> @@ -243,7 +243,7 @@ DNNReturnType ff_frame_to_dnn_classify(AVFrame
> *frame, DNNData *input, uint32_t
>      return DNN_SUCCESS;
>  }
> 
> -static DNNReturnType proc_from_frame_to_dnn_analytics(AVFrame
> *frame, DNNData *input, void *log_ctx)
> +DNNReturnType ff_frame_to_dnn_detect(AVFrame *frame, DNNData
> *input,
> +void *log_ctx)
>  {
>      struct SwsContext *sws_ctx;
>      int linesizes[4];
> @@ -271,17 +271,3 @@ static DNNReturnType
> proc_from_frame_to_dnn_analytics(AVFrame *frame, DNNData *i
>      sws_freeContext(sws_ctx);
>      return DNN_SUCCESS;
>  }
> -
> -DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData
> *input, DNNFunctionType func_type, void *log_ctx) -{
> -    switch (func_type)
> -    {
> -    case DFT_PROCESS_FRAME:
> -        return proc_from_frame_to_dnn_frameprocessing(frame, input,
> log_ctx);
> -    case DFT_ANALYTICS_DETECT:
> -        return proc_from_frame_to_dnn_analytics(frame, input, log_ctx);
> -    default:
> -        avpriv_report_missing_feature(log_ctx, "model function type %d",
> func_type);
> -        return DNN_ERROR;
> -    }
> -}
> diff --git a/libavfilter/dnn/dnn_io_proc.h b/libavfilter/dnn/dnn_io_proc.h
> index 16dcdd6d1a..daef01aceb 100644
> --- a/libavfilter/dnn/dnn_io_proc.h
> +++ b/libavfilter/dnn/dnn_io_proc.h
> @@ -30,8 +30,9 @@
>  #include "../dnn_interface.h"
>  #include "libavutil/frame.h"
> 
> -DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData
> *input, DNNFunctionType func_type, void *log_ctx);
> +DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData
> *input,
> +void *log_ctx);
>  DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData
> *output, void *log_ctx);
> +DNNReturnType ff_frame_to_dnn_detect(AVFrame *frame, DNNData
> *input,
> +void *log_ctx);
>  DNNReturnType ff_frame_to_dnn_classify(AVFrame *frame, DNNData
> *input, uint32_t bbox_index, void *log_ctx);
> 
Will push this patch (patch 2), and patch 1 V2 next Monday if there's no objection, thanks.