[FFmpeg-devel] [PATCH v3 7/7] libavfilter: Send only input frame for DNN Detect and Classify

[Shubhanshu Saxena]

This commit updates the following two filters to send only the
input frame and send NULL as output frame instead of input frame
to the DNN backends.

1. vf_dnn_detect
2. vf_dnn_classify

Signed-off-by: Shubhanshu Saxena <shubhanshu.e01 at gmail.com>
---
 libavfilter/dnn/dnn_backend_common.c   |  2 +-
 libavfilter/dnn/dnn_backend_openvino.c |  5 +++--
 libavfilter/vf_dnn_classify.c          | 14 ++++++--------
 libavfilter/vf_dnn_detect.c            | 14 ++++++--------
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c
index d2bc016fef..6a9c4cc87f 100644
--- a/libavfilter/dnn/dnn_backend_common.c
+++ b/libavfilter/dnn/dnn_backend_common.c
@@ -38,7 +38,7 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func
         return AVERROR(EINVAL);
     }
 
-    if (!exec_params->out_frame) {
+    if (!exec_params->out_frame && func_type == DFT_PROCESS_FRAME) {
         av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
         return AVERROR(EINVAL);
     }
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index 76dc06c6d7..f5b1454d21 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -272,14 +272,14 @@ static void infer_completion_callback(void *args)
                 av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
                 return;
             }
-            ov_model->model->detect_post_proc(task->out_frame, &output, 1, ov_model->model->filter_ctx);
+            ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx);
             break;
         case DFT_ANALYTICS_CLASSIFY:
             if (!ov_model->model->classify_post_proc) {
                 av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
                 return;
             }
-            ov_model->model->classify_post_proc(task->out_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
+            ov_model->model->classify_post_proc(task->in_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
             break;
         default:
             av_assert0(!"should not reach here");
@@ -819,6 +819,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *
         if (model->func_type == DFT_ANALYTICS_CLASSIFY) {
             // Classification filter has not been completely
             // tested with the sync mode. So, do not support now.
+            avpriv_report_missing_feature(ctx, "classify for sync execution");
             return DNN_ERROR;
         }
 
diff --git a/libavfilter/vf_dnn_classify.c b/libavfilter/vf_dnn_classify.c
index d5ee65d403..d1ba8dffbc 100644
--- a/libavfilter/vf_dnn_classify.c
+++ b/libavfilter/vf_dnn_classify.c
@@ -225,13 +225,12 @@ static int dnn_classify_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t
         AVFrame *in_frame = NULL;
         AVFrame *out_frame = NULL;
         async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
-        if (out_frame) {
-            av_assert0(in_frame == out_frame);
-            ret = ff_filter_frame(outlink, out_frame);
+        if (async_state == DAST_SUCCESS) {
+            ret = ff_filter_frame(outlink, in_frame);
             if (ret < 0)
                 return ret;
             if (out_pts)
-                *out_pts = out_frame->pts + pts;
+                *out_pts = in_frame->pts + pts;
         }
         av_usleep(5000);
     } while (async_state >= DAST_NOT_READY);
@@ -258,7 +257,7 @@ static int dnn_classify_activate(AVFilterContext *filter_ctx)
         if (ret < 0)
             return ret;
         if (ret > 0) {
-            if (ff_dnn_execute_model_classification(&ctx->dnnctx, in, in, ctx->target) != DNN_SUCCESS) {
+            if (ff_dnn_execute_model_classification(&ctx->dnnctx, in, NULL, ctx->target) != DNN_SUCCESS) {
                 return AVERROR(EIO);
             }
         }
@@ -269,9 +268,8 @@ static int dnn_classify_activate(AVFilterContext *filter_ctx)
         AVFrame *in_frame = NULL;
         AVFrame *out_frame = NULL;
         async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
-        if (out_frame) {
-            av_assert0(in_frame == out_frame);
-            ret = ff_filter_frame(outlink, out_frame);
+        if (async_state == DAST_SUCCESS) {
+            ret = ff_filter_frame(outlink, in_frame);
             if (ret < 0)
                 return ret;
             got_frame = 1;
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 809d70b930..637874b2a1 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -368,13 +368,12 @@ static int dnn_detect_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *o
         AVFrame *in_frame = NULL;
         AVFrame *out_frame = NULL;
         async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
-        if (out_frame) {
-            av_assert0(in_frame == out_frame);
-            ret = ff_filter_frame(outlink, out_frame);
+        if (async_state == DAST_SUCCESS) {
+            ret = ff_filter_frame(outlink, in_frame);
             if (ret < 0)
                 return ret;
             if (out_pts)
-                *out_pts = out_frame->pts + pts;
+                *out_pts = in_frame->pts + pts;
         }
         av_usleep(5000);
     } while (async_state >= DAST_NOT_READY);
@@ -401,7 +400,7 @@ static int dnn_detect_activate(AVFilterContext *filter_ctx)
         if (ret < 0)
             return ret;
         if (ret > 0) {
-            if (ff_dnn_execute_model(&ctx->dnnctx, in, in) != DNN_SUCCESS) {
+            if (ff_dnn_execute_model(&ctx->dnnctx, in, NULL) != DNN_SUCCESS) {
                 return AVERROR(EIO);
             }
         }
@@ -412,9 +411,8 @@ static int dnn_detect_activate(AVFilterContext *filter_ctx)
         AVFrame *in_frame = NULL;
         AVFrame *out_frame = NULL;
         async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
-        if (out_frame) {
-            av_assert0(in_frame == out_frame);
-            ret = ff_filter_frame(outlink, out_frame);
+        if (async_state == DAST_SUCCESS) {
+            ret = ff_filter_frame(outlink, in_frame);
             if (ret < 0)
                 return ret;
             got_frame = 1;
-- 
2.25.1