[FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter support draw text with detection bounding boxes in side_data
Guo, Yejun
yejun.guo at intel.com
Tue May 25 04:08:17 EEST 2021
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of Guo,
> Yejun
> Sent: 2021年5月20日 11:04
> To: FFmpeg development discussions and patches
> <ffmpeg-devel at ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter
> support draw text with detection bounding boxes in side_data
>
>
>
> > -----Original Message-----
> > From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of Ting
> > Fu
> > Sent: 2021年5月14日 16:47
> > To: ffmpeg-devel at ffmpeg.org
> > Subject: [FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter support
> > draw text with detection bounding boxes in side_data
> >
> > This feature can be used with dnn detection by setting vf_drawtext's option
> > text_source=side_data_detection_bboxes, for example:
> > ./ffmpeg -i face.jpeg -vf
> >
> dnn_detect=dnn_backend=openvino:model=face-detection-adas-0001.xml:\
> >
> input=data:output=detection_out:labels=face-detection-adas-0001.label,dra
> > wbox=box_source=
> >
> side_data_detection_bboxes,drawtext=text_source=side_data_detection_bbo
> > xes:fontcolor=green:\
> > fontsize=40, -y face_detect.jpeg
> > Please note, the default fontsize of vf_drawtext is 12, which may be too
> > small to be seen clearly.
> >
> > Signed-off-by: Ting Fu <ting.fu at intel.com>
> > ---
> > doc/filters.texi | 8 ++++
> > libavfilter/vf_drawtext.c | 77
> > ++++++++++++++++++++++++++++++++++++---
> > 2 files changed, 79 insertions(+), 6 deletions(-)
> >
> > diff --git a/doc/filters.texi b/doc/filters.texi
> > index f2ac8c4cc8..d10e6de03d 100644
> > --- a/doc/filters.texi
> > +++ b/doc/filters.texi
> > @@ -10788,6 +10788,14 @@ parameter @var{text}.
> >
> > If both @var{text} and @var{textfile} are specified, an error is thrown.
> >
> > + at item text_source
> > +Text source should be set as side_data_detection_bboxes if you want to
> use
> > text data in
> > +detection bboxes of side data.
> > +
> > +If text source is set, @var{text} and @var{textfile} will be ignored and still
> > use
> > +text data in detection bboxes of side data. So please do not use this
> > parameter
> > +if you are not sure about the text source.
> > +
> > @item reload
> > If set to 1, the @var{textfile} will be reloaded before each frame.
> > Be sure to update it atomically, or it may be read partially, or even fail.
> > diff --git a/libavfilter/vf_drawtext.c b/libavfilter/vf_drawtext.c
> > index 7ea057b812..382d589e26 100644
> > --- a/libavfilter/vf_drawtext.c
> > +++ b/libavfilter/vf_drawtext.c
> > @@ -55,6 +55,7 @@
> > #include "libavutil/time_internal.h"
> > #include "libavutil/tree.h"
> > #include "libavutil/lfg.h"
> > +#include "libavutil/detection_bbox.h"
> > #include "avfilter.h"
> > #include "drawutils.h"
> > #include "formats.h"
> > @@ -199,6 +200,8 @@ typedef struct DrawTextContext {
> > int tc24hmax; ///< 1 if timecode is wrapped to
> 24
> > hours, 0 otherwise
> > int reload; ///< reload text file for each frame
> > int start_number; ///< starting frame number for
> > n/frame_num var
> > + char *text_source_string; ///< the string to specify text data
> > source
> > + enum AVFrameSideDataType text_source;
> > #if CONFIG_LIBFRIBIDI
> > int text_shaping; ///< 1 to shape the text before
> > drawing it
> > #endif
> > @@ -246,6 +249,7 @@ static const AVOption drawtext_options[]= {
> > { "alpha", "apply alpha while rendering", OFFSET(a_expr),
> > AV_OPT_TYPE_STRING, { .str = "1" }, .flags = FLAGS },
> > {"fix_bounds", "check and fix text coords to avoid clipping",
> > OFFSET(fix_bounds), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS},
> > {"start_number", "start frame number for n/frame_num variable",
> > OFFSET(start_number), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS},
> > + {"text_source", "the source of text", OFFSET(text_source_string),
> > AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS },
> >
> > #if CONFIG_LIBFRIBIDI
> > {"text_shaping", "attempt to shape text before drawing",
> > OFFSET(text_shaping), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS},
> > @@ -690,6 +694,16 @@ out:
> > }
> > #endif
> >
> > +static enum AVFrameSideDataType text_source_string_parse(const char
> > *text_source_string)
> > +{
> > + av_assert0(text_source_string);
> > + if (!strcmp(text_source_string, "side_data_detection_bboxes")) {
> > + return AV_FRAME_DATA_DETECTION_BBOXES;
> > + } else {
> > + return AVERROR(EINVAL);
> > + }
> > +}
> > +
> > static av_cold int init(AVFilterContext *ctx)
> > {
> > int err;
> > @@ -731,9 +745,28 @@ static av_cold int init(AVFilterContext *ctx)
> > s->text = av_strdup("");
> > }
> >
> > + if (s->text_source_string) {
> > + s->text_source = text_source_string_parse(s->text_source_string);
> > + if ((int)s->text_source < 0) {
> > + av_log(ctx, AV_LOG_ERROR, "Error text source: %s\n",
> > s->text_source_string);
> > + return AVERROR(EINVAL);
> > + }
> > + }
> > +
> > + if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES) {
> > + if (s->text) {
> > + av_log(ctx, AV_LOG_WARNING, "Multiple texts provided, will
> > use text_source only\n");
> > + av_free(s->text);
> > + }
> > + s->text =
> > av_mallocz(AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE *
> > + (AV_NUM_DETECTION_BBOX_CLASSIFY +
> > 1));
> > + if (!s->text)
> > + return AVERROR(ENOMEM);
> > + }
> > +
> > if (!s->text) {
> > av_log(ctx, AV_LOG_ERROR,
> > - "Either text, a valid file or a timecode must be
> > provided\n");
> > + "Either text, a valid file, a timecode or text source must
> be
> > provided\n");
> > return AVERROR(EINVAL);
> > }
> >
> > @@ -1440,10 +1473,15 @@ continue_on_invalid2:
> >
> > s->var_values[VAR_LINE_H] = s->var_values[VAR_LH] =
> s->max_glyph_h;
> >
> > - s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values,
> > &s->prng);
> > - s->y = s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr, s->var_values,
> > &s->prng);
> > - /* It is necessary if x is expressed from y */
> > - s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values,
> > &s->prng);
> > + if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES) {
> > + s->var_values[VAR_X] = s->x;
> > + s->var_values[VAR_Y] = s->y;
> > + } else {
> > + s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr,
> > s->var_values, &s->prng);
> > + s->y = s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr,
> > s->var_values, &s->prng);
> > + /* It is necessary if x is expressed from y */
> > + s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr,
> > s->var_values, &s->prng);
> > + }
> >
> > update_alpha(s);
> > update_color_with_alpha(s, &fontcolor , s->fontcolor );
> > @@ -1511,6 +1549,21 @@ static int filter_frame(AVFilterLink *inlink,
> > AVFrame *frame)
> > AVFilterLink *outlink = ctx->outputs[0];
> > DrawTextContext *s = ctx->priv;
> > int ret;
> > + const AVDetectionBBoxHeader *header = NULL;
> > + const AVDetectionBBox *bbox;
> > + AVFrameSideData *sd;
> > + int loop = 1;
> > +
> > + if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES && sd) {
> > + sd = av_frame_get_side_data(frame,
> > AV_FRAME_DATA_DETECTION_BBOXES);
> > + if (sd) {
> > + header = (AVDetectionBBoxHeader *)sd->data;
> > + loop = header->nb_bboxes;
> > + } else {
> > + av_log(s, AV_LOG_WARNING, "No detection bboxes.\n");
> > + return ff_filter_frame(outlink, frame);
> > + }
> > + }
> >
> > if (s->reload) {
> > if ((ret = load_textfile(ctx)) < 0) {
> > @@ -1536,7 +1589,19 @@ static int filter_frame(AVFilterLink *inlink,
> > AVFrame *frame)
> > s->var_values[VAR_PKT_SIZE] = frame->pkt_size;
> > s->metadata = frame->metadata;
> >
> > - draw_text(ctx, frame, frame->width, frame->height);
> > + for (int i = 0; i < loop; i++) {
> > + if (header) {
> > + bbox = av_get_detection_bbox(header, i);
> > + strcpy(s->text, bbox->detect_label);
> > + for (int j = 0; j < bbox->classify_count; j++) {
> > + strcat(s->text, ", ");
> > + strcat(s->text, bbox->classify_labels[j]);
> > + }
> > + s->x = bbox->x;
> > + s->y = bbox->y - s->fontsize;
> > + }
> > + draw_text(ctx, frame, frame->width, frame->height);
> > + }
> >
> > av_log(ctx, AV_LOG_DEBUG, "n:%d t:%f text_w:%d text_h:%d x:%d
> > y:%d\n",
> > (int)s->var_values[VAR_N], s->var_values[VAR_T],
> > --
>
> any comment? thanks.
>
> A new option is added into vf_drawbox and vf_drawtext to visualize the
> data from detection bounding boxes in the side data of AVFrame.
>
will push tomorrow if there's no objection.
More information about the ffmpeg-devel
mailing list