[FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter support draw text with detection bounding boxes in side_data

Guo, Yejun yejun.guo at intel.com
Tue May 25 04:08:17 EEST 2021



> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of Guo,
> Yejun
> Sent: 2021年5月20日 11:04
> To: FFmpeg development discussions and patches
> <ffmpeg-devel at ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter
> support draw text with detection bounding boxes in side_data
> 
> 
> 
> > -----Original Message-----
> > From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of Ting
> > Fu
> > Sent: 2021年5月14日 16:47
> > To: ffmpeg-devel at ffmpeg.org
> > Subject: [FFmpeg-devel] [PATCH 3/3] libavfilter: vf_drawtext filter support
> > draw text with detection bounding boxes in side_data
> >
> > This feature can be used with dnn detection by setting vf_drawtext's option
> > text_source=side_data_detection_bboxes, for example:
> > ./ffmpeg -i face.jpeg -vf
> >
> dnn_detect=dnn_backend=openvino:model=face-detection-adas-0001.xml:\
> >
> input=data:output=detection_out:labels=face-detection-adas-0001.label,dra
> > wbox=box_source=
> >
> side_data_detection_bboxes,drawtext=text_source=side_data_detection_bbo
> > xes:fontcolor=green:\
> > fontsize=40, -y face_detect.jpeg
> > Please note, the default fontsize of vf_drawtext is 12, which may be too
> > small to be seen clearly.
> >
> > Signed-off-by: Ting Fu <ting.fu at intel.com>
> > ---
> >  doc/filters.texi          |  8 ++++
> >  libavfilter/vf_drawtext.c | 77
> > ++++++++++++++++++++++++++++++++++++---
> >  2 files changed, 79 insertions(+), 6 deletions(-)
> >
> > diff --git a/doc/filters.texi b/doc/filters.texi
> > index f2ac8c4cc8..d10e6de03d 100644
> > --- a/doc/filters.texi
> > +++ b/doc/filters.texi
> > @@ -10788,6 +10788,14 @@ parameter @var{text}.
> >
> >  If both @var{text} and @var{textfile} are specified, an error is thrown.
> >
> > + at item text_source
> > +Text source should be set as side_data_detection_bboxes if you want to
> use
> > text data in
> > +detection bboxes of side data.
> > +
> > +If text source is set, @var{text} and @var{textfile} will be ignored and still
> > use
> > +text data in detection bboxes of side data. So please do not use this
> > parameter
> > +if you are not sure about the text source.
> > +
> >  @item reload
> >  If set to 1, the @var{textfile} will be reloaded before each frame.
> >  Be sure to update it atomically, or it may be read partially, or even fail.
> > diff --git a/libavfilter/vf_drawtext.c b/libavfilter/vf_drawtext.c
> > index 7ea057b812..382d589e26 100644
> > --- a/libavfilter/vf_drawtext.c
> > +++ b/libavfilter/vf_drawtext.c
> > @@ -55,6 +55,7 @@
> >  #include "libavutil/time_internal.h"
> >  #include "libavutil/tree.h"
> >  #include "libavutil/lfg.h"
> > +#include "libavutil/detection_bbox.h"
> >  #include "avfilter.h"
> >  #include "drawutils.h"
> >  #include "formats.h"
> > @@ -199,6 +200,8 @@ typedef struct DrawTextContext {
> >      int tc24hmax;                   ///< 1 if timecode is wrapped to
> 24
> > hours, 0 otherwise
> >      int reload;                     ///< reload text file for each frame
> >      int start_number;               ///< starting frame number for
> > n/frame_num var
> > +    char *text_source_string;       ///< the string to specify text data
> > source
> > +    enum AVFrameSideDataType text_source;
> >  #if CONFIG_LIBFRIBIDI
> >      int text_shaping;               ///< 1 to shape the text before
> > drawing it
> >  #endif
> > @@ -246,6 +249,7 @@ static const AVOption drawtext_options[]= {
> >      { "alpha",       "apply alpha while rendering", OFFSET(a_expr),
> > AV_OPT_TYPE_STRING, { .str = "1"     },          .flags = FLAGS },
> >      {"fix_bounds", "check and fix text coords to avoid clipping",
> > OFFSET(fix_bounds), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS},
> >      {"start_number", "start frame number for n/frame_num variable",
> > OFFSET(start_number), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS},
> > +    {"text_source", "the source of text", OFFSET(text_source_string),
> > AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS },
> >
> >  #if CONFIG_LIBFRIBIDI
> >      {"text_shaping", "attempt to shape text before drawing",
> > OFFSET(text_shaping), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS},
> > @@ -690,6 +694,16 @@ out:
> >  }
> >  #endif
> >
> > +static enum AVFrameSideDataType text_source_string_parse(const char
> > *text_source_string)
> > +{
> > +    av_assert0(text_source_string);
> > +    if (!strcmp(text_source_string, "side_data_detection_bboxes")) {
> > +        return AV_FRAME_DATA_DETECTION_BBOXES;
> > +    } else {
> > +        return AVERROR(EINVAL);
> > +    }
> > +}
> > +
> >  static av_cold int init(AVFilterContext *ctx)
> >  {
> >      int err;
> > @@ -731,9 +745,28 @@ static av_cold int init(AVFilterContext *ctx)
> >              s->text = av_strdup("");
> >      }
> >
> > +    if (s->text_source_string) {
> > +        s->text_source = text_source_string_parse(s->text_source_string);
> > +        if ((int)s->text_source < 0) {
> > +            av_log(ctx, AV_LOG_ERROR, "Error text source: %s\n",
> > s->text_source_string);
> > +            return AVERROR(EINVAL);
> > +        }
> > +    }
> > +
> > +    if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES) {
> > +        if (s->text) {
> > +            av_log(ctx, AV_LOG_WARNING, "Multiple texts provided, will
> > use text_source only\n");
> > +            av_free(s->text);
> > +        }
> > +        s->text =
> > av_mallocz(AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE *
> > +                             (AV_NUM_DETECTION_BBOX_CLASSIFY +
> > 1));
> > +        if (!s->text)
> > +            return AVERROR(ENOMEM);
> > +    }
> > +
> >      if (!s->text) {
> >          av_log(ctx, AV_LOG_ERROR,
> > -               "Either text, a valid file or a timecode must be
> > provided\n");
> > +               "Either text, a valid file, a timecode or text source must
> be
> > provided\n");
> >          return AVERROR(EINVAL);
> >      }
> >
> > @@ -1440,10 +1473,15 @@ continue_on_invalid2:
> >
> >      s->var_values[VAR_LINE_H] = s->var_values[VAR_LH] =
> s->max_glyph_h;
> >
> > -    s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values,
> > &s->prng);
> > -    s->y = s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr, s->var_values,
> > &s->prng);
> > -    /* It is necessary if x is expressed from y  */
> > -    s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values,
> > &s->prng);
> > +    if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES) {
> > +        s->var_values[VAR_X] = s->x;
> > +        s->var_values[VAR_Y] = s->y;
> > +    } else {
> > +        s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr,
> > s->var_values, &s->prng);
> > +        s->y = s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr,
> > s->var_values, &s->prng);
> > +        /* It is necessary if x is expressed from y  */
> > +        s->x = s->var_values[VAR_X] = av_expr_eval(s->x_pexpr,
> > s->var_values, &s->prng);
> > +    }
> >
> >      update_alpha(s);
> >      update_color_with_alpha(s, &fontcolor  , s->fontcolor  );
> > @@ -1511,6 +1549,21 @@ static int filter_frame(AVFilterLink *inlink,
> > AVFrame *frame)
> >      AVFilterLink *outlink = ctx->outputs[0];
> >      DrawTextContext *s = ctx->priv;
> >      int ret;
> > +    const AVDetectionBBoxHeader *header = NULL;
> > +    const AVDetectionBBox *bbox;
> > +    AVFrameSideData *sd;
> > +    int loop = 1;
> > +
> > +    if (s->text_source == AV_FRAME_DATA_DETECTION_BBOXES && sd) {
> > +        sd = av_frame_get_side_data(frame,
> > AV_FRAME_DATA_DETECTION_BBOXES);
> > +        if (sd) {
> > +            header = (AVDetectionBBoxHeader *)sd->data;
> > +            loop = header->nb_bboxes;
> > +        } else {
> > +            av_log(s, AV_LOG_WARNING, "No detection bboxes.\n");
> > +            return ff_filter_frame(outlink, frame);
> > +        }
> > +    }
> >
> >      if (s->reload) {
> >          if ((ret = load_textfile(ctx)) < 0) {
> > @@ -1536,7 +1589,19 @@ static int filter_frame(AVFilterLink *inlink,
> > AVFrame *frame)
> >      s->var_values[VAR_PKT_SIZE] = frame->pkt_size;
> >      s->metadata = frame->metadata;
> >
> > -    draw_text(ctx, frame, frame->width, frame->height);
> > +    for (int i = 0; i < loop; i++) {
> > +        if (header) {
> > +            bbox = av_get_detection_bbox(header, i);
> > +            strcpy(s->text, bbox->detect_label);
> > +            for (int j = 0; j < bbox->classify_count; j++) {
> > +                strcat(s->text, ", ");
> > +                strcat(s->text, bbox->classify_labels[j]);
> > +            }
> > +            s->x = bbox->x;
> > +            s->y = bbox->y - s->fontsize;
> > +        }
> > +        draw_text(ctx, frame, frame->width, frame->height);
> > +    }
> >
> >      av_log(ctx, AV_LOG_DEBUG, "n:%d t:%f text_w:%d text_h:%d x:%d
> > y:%d\n",
> >             (int)s->var_values[VAR_N], s->var_values[VAR_T],
> > --
> 
> any comment? thanks.
> 
> A new option is added into vf_drawbox and vf_drawtext to visualize the
> data from detection bounding boxes in the side data of AVFrame.
> 

will push tomorrow if there's no objection.


More information about the ffmpeg-devel mailing list