[FFmpeg-devel] [PATCH] lavfi: add thumb video filter.

[Michael Niedermayer]

On Tue, Nov 29, 2011 at 05:07:09PM +0100, Clément Bœsch wrote:
> From: Clément Bœsch <clement.boesch at smartjog.com>
> 
> ---
> Hi,
> 
> This is somewhat a proof of concept for smart thumbnail selection with the video
> filters. The algorithm might not be the best but raise interesting results (see
> doxy in the main file for reference).
>
> I'm not sure the way frames are buffered is the best (actually it isn't for
> sure), but I have nothing better to propose right now; and I'm still confused
> with the filters API...
[...]
> +static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)
> +{
> +    int i, j;
> +    const uint8_t *p;
> +    AVFilterContext *ctx = inlink->dst;
> +    ThumbContext *thumb = ctx->priv;
> +    int *hist = thumb->histogram + thumb->n * HIST_SZ;
> +    AVFilterBufferRef *picref = inlink->cur_buf;
> +
> +    p = picref->data[0] + y * picref->linesize[0];
> +    for (j = 0; j < h; j++) {
> +        for (i = 0; i < inlink->w; i++) {
> +            hist[0*256 + p[i*3    ]]++;
> +            hist[1*256 + p[i*3 + 1]]++;
> +            hist[2*256 + p[i*3 + 2]]++;
> +        }
> +        p += picref->linesize[0];
> +    }

> +    avfilter_default_draw_slice(inlink, y, h, slice_dir);

this should not be here
it should be called from end frame when the correct frame has been
selected


> +}
> +
> +/**
> + * @brief        compute Root-mean-square deviation to estimate "closeness"
> + * @param hist   color distribution histogram
> + * @param median average color distribution histogram
> + * @return       root mean squared error
> + */
> +static float frame_rmse(const int *hist, const float *median)
> +{
> +    int i;
> +    float err, mean_sq_err = 0;
> +    for (i = 0; i < HIST_SZ; i++) {
> +        err = median[i] - (float)hist[i];
> +        mean_sq_err += err*err / HIST_SZ;
> +    }
> +    return sqrtf(mean_sq_err);
> +}
> +
> +static void end_frame(AVFilterLink *inlink)
> +{
> +    int i, j;
> +    float avg[HIST_SZ] = {0}, rmse, min_rmse = -1;
> +    int best_frame = 0;
> +    ThumbContext *thumb = inlink->dst->priv;
> +
> +    // keep a reference of each frame
> +    thumb->frames[thumb->n] = avfilter_ref_buffer(inlink->cur_buf, AV_PERM_READ);
> +
> +    // no selection until the buffer of N frames is filled up
> +    if (thumb->n < thumb->n_frames - 1) {
> +        thumb->n++;
> +        return;
> +    }
> +
> +    // average histogram of the N frames
> +    for (j = 0; j < FF_ARRAY_ELEMS(avg); j++)
> +        for (i = 0; i < thumb->n_frames; i++)
> +            avg[j] += (float)thumb->histogram[i*HIST_SZ + j] / thumb->n_frames;
> +
> +    // find the frame closer to the average using RMSE
> +    for (i = 0; i < thumb->n_frames; i++) {
> +        rmse = frame_rmse(&thumb->histogram[i*HIST_SZ], avg);
> +        if (i == 0 || rmse < min_rmse)
> +            best_frame = i, min_rmse = rmse;
> +    }
> +
> +    // free and reset everything (except the best frame buffer)
> +    for (i = 0; i < thumb->n_frames; i++) {
> +        if (i == best_frame)
> +            continue;
> +        avfilter_unref_buffer(thumb->frames[i]);
> +        thumb->frames[i] = NULL;
> +    }
> +    memset(thumb->histogram, 0, HIST_SZ * thumb->n_frames);
> +    thumb->n = 0;
> +

> +    // raise the chosen one
> +    inlink->cur_buf = thumb->frames[best_frame];
> +    avfilter_default_end_frame(inlink);

this looks wrong, does it even return the correct frame ?

the code here should call avfilter_start_frame / draw_slice/end_frame



> +}
> +
> +static av_cold void uninit(AVFilterContext *ctx)
> +{
> +    ThumbContext *thumb = ctx->priv;
> +    av_freep(&thumb->histogram);
> +}
> +
> +static int query_formats(AVFilterContext *ctx)
> +{
> +    static const enum PixelFormat pix_fmts[] = {
> +        PIX_FMT_RGB24, PIX_FMT_BGR24,
> +        PIX_FMT_NONE
> +    };
> +    avfilter_set_common_pixel_formats(ctx, avfilter_make_format_list(pix_fmts));
> +    return 0;
> +}
> +
> +AVFilter avfilter_vf_thumb = {
> +    .name          = "thumb",
> +    .description   = NULL_IF_CONFIG_SMALL("Thumbnail selection filter"),
> +    .priv_size     = sizeof(ThumbContext),
> +    .init          = init,
> +    .uninit        = uninit,
> +    .query_formats = query_formats,
> +    .inputs    = (const AVFilterPad[])  {{  .name             = "default",
> +                                            .type             = AVMEDIA_TYPE_VIDEO,
> +                                            .get_video_buffer = avfilter_null_get_video_buffer,

> +                                            .start_frame      = avfilter_null_start_frame,

avfilter_start_frame can not be propagated, you need a really empty
function. 

                                                  
> +                                            .draw_slice       = draw_slice,
> +                                            .end_frame        = end_frame,
> +                                        },{ .name = NULL }},

AV_PERM_PRESERVE is missing


[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

While the State exists there can be no freedom; when there is freedom there
will be no State. -- Vladimir Lenin
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 198 bytes
Desc: Digital signature
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20111130/7d49b01b/attachment.asc>