[FFmpeg-devel] [PATCH V6 4/6] lavu: add side data AV_FRAME_DATA_BOUNDING_BOXES

[Guo, Yejun]

Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
---
 doc/APIchanges          |  2 ++
 libavutil/Makefile      |  1 +
 libavutil/boundingbox.h | 79 +++++++++++++++++++++++++++++++++++++++++
 libavutil/frame.c       |  1 +
 libavutil/frame.h       |  7 ++++
 5 files changed, 90 insertions(+)
 create mode 100644 libavutil/boundingbox.h

diff --git a/doc/APIchanges b/doc/APIchanges
index b41dadee8d..160b020d9d 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -14,6 +14,8 @@ libavutil:     2017-10-21
 
 
 API changes, most recent first:
+2021-03-xx - xxxxxxxxxx - lavu 56.xx.100 - frame.h boundingbox.h
+  Add AV_FRAME_DATA_BOUNDING_BOXES
 
 2021-03-21 - xxxxxxxxxx - lavu 56.72.100 - frame.h
   Deprecated av_get_colorspace_name().
diff --git a/libavutil/Makefile b/libavutil/Makefile
index 27bafe9e12..7e1dc713ac 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -11,6 +11,7 @@ HEADERS = adler32.h                                                     \
           avutil.h                                                      \
           base64.h                                                      \
           blowfish.h                                                    \
+          boundingbox.h                                                 \
           bprint.h                                                      \
           bswap.h                                                       \
           buffer.h                                                      \
diff --git a/libavutil/boundingbox.h b/libavutil/boundingbox.h
new file mode 100644
index 0000000000..5e2fdd55e9
--- /dev/null
+++ b/libavutil/boundingbox.h
@@ -0,0 +1,79 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_BOUNDINGBOX_H
+#define AVUTIL_BOUNDINGBOX_H
+
+#include "libavutil/rational.h"
+
+typedef struct AVBoundingBox {
+    /**
+     * Distance in pixels from the top edge of the frame to top
+     * and bottom, and from the left edge of the frame to left and
+     * right, defining the bounding box.
+     */
+    int top;
+    int left;
+    int bottom;
+    int right;
+
+#define AV_BBOX_LABEL_NAME_MAX_SIZE 32
+
+    /**
+     * Detect result with confidence
+     */
+    char detect_label[AV_BBOX_LABEL_NAME_MAX_SIZE];
+    AVRational detect_confidence;
+
+    /**
+     * At most 4 classifications based on the detected bounding box.
+     * For example, we can get max 4 different attributes with 4 different
+     * DNN models on one bounding box.
+     * classify_count is zero if no classification.
+     */
+#define AV_NUM_BBOX_CLASSIFY 4
+    uint32_t classify_count;
+    char classify_labels[AV_NUM_BBOX_CLASSIFY][AV_BBOX_LABEL_NAME_MAX_SIZE];
+    AVRational classify_confidences[AV_NUM_BBOX_CLASSIFY];
+} AVBoundingBox;
+
+typedef struct AVBoundingBoxHeader {
+    /**
+     * Information about how the bounding box is generated.
+     * for example, the DNN model name.
+     */
+    char source[128];
+
+    /**
+     * The size of frame when it is detected.
+     */
+    int frame_width;
+    int frame_height;
+
+    /**
+     * Number of bounding boxes.
+     */
+    uint32_t nb_bbox;
+
+    /**
+     * Pointer to the array of AVBoundingBox.
+     */
+    AVBoundingBox bboxes[];
+} AVBoundingBoxHeader;
+
+#endif
diff --git a/libavutil/frame.c b/libavutil/frame.c
index 31a2117b82..99080af777 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -853,6 +853,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type)
     case AV_FRAME_DATA_VIDEO_ENC_PARAMS:            return "Video encoding parameters";
     case AV_FRAME_DATA_SEI_UNREGISTERED:            return "H.26[45] User Data Unregistered SEI message";
     case AV_FRAME_DATA_FILM_GRAIN_PARAMS:           return "Film grain parameters";
+    case AV_FRAME_DATA_BOUNDING_BOXES:              return "Bounding boxes";
     }
     return NULL;
 }
diff --git a/libavutil/frame.h b/libavutil/frame.h
index a5ed91b20a..9b7145199c 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -198,6 +198,13 @@ enum AVFrameSideDataType {
      * Must be present for every frame which should have film grain applied.
      */
     AV_FRAME_DATA_FILM_GRAIN_PARAMS,
+
+    /**
+     * Bounding boxes for object detection and classification, the data is a BoundingBoxHeader
+     * followed with an array of BoudingBox, and BoundingBoxHeader.nb_bbox is the number of
+     * array element.
+     */
+    AV_FRAME_DATA_BOUNDING_BOXES,
 };
 
 enum AVActiveFormatDescription {
-- 
2.17.1