[FFmpeg-devel] [PATCH] lavfi: add thumb video filter.
Clément Bœsch
ubitux at gmail.com
Tue Nov 29 17:07:09 CET 2011
From: Clément Bœsch <clement.boesch at smartjog.com>
---
Hi,
This is somewhat a proof of concept for smart thumbnail selection with the video
filters. The algorithm might not be the best but raise interesting results (see
doxy in the main file for reference).
I'm not sure the way frames are buffered is the best (actually it isn't for
sure), but I have nothing better to propose right now; and I'm still confused
with the filters API...
If you want to do some testing, here is a small script I used:
#!/bin/bash
bin=$HOME/ff/ffmpeg/ffmpeg
thumbdir=/tmp/ffthumb
outpage=$thumbdir/index.html
[ $# -ne 1 ] && echo "Usage: $0 <dir>" && exit 1
mkdir -p $thumbdir
content="<html><body>"
for f in $1/*.{mp4,mov,mpg,avi}; do
echo "Processing: $f"
png=$thumbdir/`basename $f`
png_nothumb=${png}_nothumb.png
png_thumb=${png}_thumb.png
$bin -d -i $f -vf scale=300:200 -an -vframes 1 $png_nothumb
$bin -d -i $f -vf thumb=100,scale=300:200 -an -vframes 1 $png_thumb
content="$content
<div style=\"padding:10px\">
<h3>$f</h3>
<img src="$png_nothumb"/>
<img src="$png_thumb"/>
</div>"
done
content="$content</body></html>"
echo "$content" > $outpage
---
Changelog | 1 +
doc/filters.texi | 12 +++
libavfilter/Makefile | 1 +
libavfilter/allfilters.c | 1 +
libavfilter/avfilter.h | 2 +-
libavfilter/vf_thumb.c | 180 ++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 196 insertions(+), 1 deletions(-)
create mode 100644 libavfilter/vf_thumb.c
diff --git a/Changelog b/Changelog
index e64dca6..8c224d9 100644
--- a/Changelog
+++ b/Changelog
@@ -126,6 +126,7 @@ easier to use. The changes are:
- IFF Amiga Continuous Bitmap (ACBM) decoder
- ass filter
- CRI ADX audio format demuxer
+- Thumbnails support (see thumb video filter)
version 0.8:
diff --git a/doc/filters.texi b/doc/filters.texi
index e583e9e..b199851 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -2382,6 +2382,18 @@ For example:
will create two separate outputs from the same input, one cropped and
one padded.
+ at section thumb
+Select potential thumbnail frames.
+
+It accepts as argument the threshold of frames to analyze (default is 100). A
+bigger value will result in a slower analysis and higher memory usage, but is
+likely to be more efficient.
+
+Example of thumbnail creation:
+ at example
+ffmpeg -i in.avi -vf thumb,scale=300:200 -vframes 1 out.png
+ at end example
+
@section transpose
Transpose rows with columns in the input video and optionally flip it.
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 27979a3..1bc9b71 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -78,6 +78,7 @@ OBJS-$(CONFIG_SETTB_FILTER) += vf_settb.o
OBJS-$(CONFIG_SHOWINFO_FILTER) += vf_showinfo.o
OBJS-$(CONFIG_SLICIFY_FILTER) += vf_slicify.o
OBJS-$(CONFIG_SPLIT_FILTER) += vf_split.o
+OBJS-$(CONFIG_THUMB_FILTER) += vf_thumb.o
OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o
OBJS-$(CONFIG_UNSHARP_FILTER) += vf_unsharp.o
OBJS-$(CONFIG_VFLIP_FILTER) += vf_vflip.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 176cfa6..fd2a617 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -89,6 +89,7 @@ void avfilter_register_all(void)
REGISTER_FILTER (SHOWINFO, showinfo, vf);
REGISTER_FILTER (SLICIFY, slicify, vf);
REGISTER_FILTER (SPLIT, split, vf);
+ REGISTER_FILTER (THUMB, thumb, vf);
REGISTER_FILTER (TRANSPOSE, transpose, vf);
REGISTER_FILTER (UNSHARP, unsharp, vf);
REGISTER_FILTER (VFLIP, vflip, vf);
diff --git a/libavfilter/avfilter.h b/libavfilter/avfilter.h
index 50a5e5b..24998a6 100644
--- a/libavfilter/avfilter.h
+++ b/libavfilter/avfilter.h
@@ -29,7 +29,7 @@
#include "libavutil/rational.h"
#define LIBAVFILTER_VERSION_MAJOR 2
-#define LIBAVFILTER_VERSION_MINOR 50
+#define LIBAVFILTER_VERSION_MINOR 51
#define LIBAVFILTER_VERSION_MICRO 0
#define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
diff --git a/libavfilter/vf_thumb.c b/libavfilter/vf_thumb.c
new file mode 100644
index 0000000..4099457
--- /dev/null
+++ b/libavfilter/vf_thumb.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright (C) 2011 Smartjog S.A.S, Clément Bœsch <clement.boesch at smartjog.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Search for appropriate thumb frame to reduce the risk of an inappropriate
+ * selection (such as a black frame) we could get with and absolute seek.
+ *
+ * Algorithm by Vadim Zaliva <lord at crocodile.org>.
+ * @url http://notbrainsurgery.livejournal.com/29773.html
+ */
+
+#include <math.h>
+#include "libavcodec/avcodec.h"
+#include "libavutil/imgutils.h"
+#include "libswscale/swscale.h"
+#include "avfilter.h"
+
+#define HIST_SZ (3*256)
+
+typedef struct {
+ int n; ///< current frame
+ int n_frames; ///< threshold of frames for analysis
+ AVFilterBufferRef **frames; ///< n_frames frames cache
+ int *histogram; ///< array of RGB color distribution histograms
+} ThumbContext;
+
+static av_cold int init(AVFilterContext *ctx, const char *args, void *opaque)
+{
+ ThumbContext *thumb = ctx->priv;
+
+ if (args)
+ thumb->n_frames = strtol(args, NULL, 10);
+ if (thumb->n_frames <= 0)
+ thumb->n_frames = 100;
+ thumb->histogram = av_calloc(thumb->n_frames,
+ HIST_SZ * sizeof(*thumb->histogram));
+ thumb->frames = av_calloc(thumb->n_frames, sizeof(*thumb->frames));
+ if (!thumb->histogram || !thumb->frames) {
+ av_log(ctx, AV_LOG_ERROR,
+ "allocation failure, try to lower the frames threshold\n");
+ return AVERROR(ENOMEM);
+ }
+ av_log(ctx, AV_LOG_INFO, "select thumbnail with threshold of %d frames\n",
+ thumb->n_frames);
+ return 0;
+}
+
+static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)
+{
+ int i, j;
+ const uint8_t *p;
+ AVFilterContext *ctx = inlink->dst;
+ ThumbContext *thumb = ctx->priv;
+ int *hist = thumb->histogram + thumb->n * HIST_SZ;
+ AVFilterBufferRef *picref = inlink->cur_buf;
+
+ p = picref->data[0] + y * picref->linesize[0];
+ for (j = 0; j < h; j++) {
+ for (i = 0; i < inlink->w; i++) {
+ hist[0*256 + p[i*3 ]]++;
+ hist[1*256 + p[i*3 + 1]]++;
+ hist[2*256 + p[i*3 + 2]]++;
+ }
+ p += picref->linesize[0];
+ }
+ avfilter_default_draw_slice(inlink, y, h, slice_dir);
+}
+
+/**
+ * @brief compute Root-mean-square deviation to estimate "closeness"
+ * @param hist color distribution histogram
+ * @param median average color distribution histogram
+ * @return root mean squared error
+ */
+static float frame_rmse(const int *hist, const float *median)
+{
+ int i;
+ float err, mean_sq_err = 0;
+ for (i = 0; i < HIST_SZ; i++) {
+ err = median[i] - (float)hist[i];
+ mean_sq_err += err*err / HIST_SZ;
+ }
+ return sqrtf(mean_sq_err);
+}
+
+static void end_frame(AVFilterLink *inlink)
+{
+ int i, j;
+ float avg[HIST_SZ] = {0}, rmse, min_rmse = -1;
+ int best_frame = 0;
+ ThumbContext *thumb = inlink->dst->priv;
+
+ // keep a reference of each frame
+ thumb->frames[thumb->n] = avfilter_ref_buffer(inlink->cur_buf, AV_PERM_READ);
+
+ // no selection until the buffer of N frames is filled up
+ if (thumb->n < thumb->n_frames - 1) {
+ thumb->n++;
+ return;
+ }
+
+ // average histogram of the N frames
+ for (j = 0; j < FF_ARRAY_ELEMS(avg); j++)
+ for (i = 0; i < thumb->n_frames; i++)
+ avg[j] += (float)thumb->histogram[i*HIST_SZ + j] / thumb->n_frames;
+
+ // find the frame closer to the average using RMSE
+ for (i = 0; i < thumb->n_frames; i++) {
+ rmse = frame_rmse(&thumb->histogram[i*HIST_SZ], avg);
+ if (i == 0 || rmse < min_rmse)
+ best_frame = i, min_rmse = rmse;
+ }
+
+ // free and reset everything (except the best frame buffer)
+ for (i = 0; i < thumb->n_frames; i++) {
+ if (i == best_frame)
+ continue;
+ avfilter_unref_buffer(thumb->frames[i]);
+ thumb->frames[i] = NULL;
+ }
+ memset(thumb->histogram, 0, HIST_SZ * thumb->n_frames);
+ thumb->n = 0;
+
+ // raise the chosen one
+ inlink->cur_buf = thumb->frames[best_frame];
+ avfilter_default_end_frame(inlink);
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+ ThumbContext *thumb = ctx->priv;
+ av_freep(&thumb->histogram);
+}
+
+static int query_formats(AVFilterContext *ctx)
+{
+ static const enum PixelFormat pix_fmts[] = {
+ PIX_FMT_RGB24, PIX_FMT_BGR24,
+ PIX_FMT_NONE
+ };
+ avfilter_set_common_pixel_formats(ctx, avfilter_make_format_list(pix_fmts));
+ return 0;
+}
+
+AVFilter avfilter_vf_thumb = {
+ .name = "thumb",
+ .description = NULL_IF_CONFIG_SMALL("Thumbnail selection filter"),
+ .priv_size = sizeof(ThumbContext),
+ .init = init,
+ .uninit = uninit,
+ .query_formats = query_formats,
+ .inputs = (const AVFilterPad[]) {{ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .get_video_buffer = avfilter_null_get_video_buffer,
+ .start_frame = avfilter_null_start_frame,
+ .draw_slice = draw_slice,
+ .end_frame = end_frame,
+ },{ .name = NULL }},
+ .outputs = (const AVFilterPad[]) {{ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ },{ .name = NULL }},
+};
--
1.7.7.3
More information about the ffmpeg-devel
mailing list