[FFmpeg-devel] [PATCH] libavfilter: added atempo filter (revised patch)

Pavel Koshevoy pkoshevoy at gmail.com
Thu Jun 7 04:22:07 CEST 2012


ping


On 06/06/2012 10:17 AM, pkoshevoy at gmail.com wrote:
> From: Pavel Koshevoy<pkoshevoy at gmail.com>
>
> Added atempo audio filter for adjusting audio tempo without affecting
> pitch. This filter implements WSOLA algorithm with fast cross
> correlation calculation in frequency domain.
>
> Signed-off-by: Pavel Koshevoy<pavel at homestead.aragog.com>
> ---
>   Changelog                |    2 +-
>   configure                |    1 +
>   doc/filters.texi         |   17 +
>   libavfilter/Makefile     |    2 +
>   libavfilter/af_atempo.c  | 1245 ++++++++++++++++++++++++++++++++++++++++++++++
>   libavfilter/allfilters.c |    1 +
>   libavfilter/version.h    |    2 +-
>   7 files changed, 1268 insertions(+), 2 deletions(-)
>   create mode 100644 libavfilter/af_atempo.c
>
> diff --git a/Changelog b/Changelog
> index 41b0bdc..cc25c9b 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -5,7 +5,7 @@ version next:
>   - INI and flat output in ffprobe
>   - Scene detection in libavfilter
>   - Indeo Audio decoder
> -
> +- atempo filter
>
>   version 0.11:
>
> diff --git a/configure b/configure
> index 33bd439..7b82b64 100755
> --- a/configure
> +++ b/configure
> @@ -1687,6 +1687,7 @@ amovie_filter_deps="avcodec avformat"
>   aresample_filter_deps="swresample"
>   ass_filter_deps="libass"
>   asyncts_filter_deps="avresample"
> +atempo_filter_deps="avcodec"
>   blackframe_filter_deps="gpl"
>   boxblur_filter_deps="gpl"
>   colormatrix_filter_deps="gpl"
> diff --git a/doc/filters.texi b/doc/filters.texi
> index d9d503f..0b7dc8e 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -271,6 +271,23 @@ For example, to resample the input audio to 44100Hz:
>   aresample=44100
>   @end example
>
> + at section atempo
> +
> +Adjust audio tempo.
> +
> +The filter accepts exactly one parameter, the audio tempo. If not
> +specified then the filter will assume nominal tempo.
> +
> +For example, to slow down audio to 80% tempo:
> + at example
> +atempo=0.8
> + at end example
> +
> +For example, to speed up audio to 125% tempo:
> + at example
> +atempo=1.25
> + at end example
> +
>   @section ashowinfo
>
>   Show a line containing various information for each input audio frame.
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 29345fc..a1ced51 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -8,6 +8,7 @@ FFLIBS-$(CONFIG_RESAMPLE_FILTER) += avresample
>   FFLIBS-$(CONFIG_ACONVERT_FILTER)             += swresample
>   FFLIBS-$(CONFIG_AMOVIE_FILTER)               += avformat avcodec
>   FFLIBS-$(CONFIG_ARESAMPLE_FILTER)            += swresample
> +FFLIBS-$(CONFIG_ATEMPO_FILTER)               += avcodec
>   FFLIBS-$(CONFIG_MOVIE_FILTER)                += avformat avcodec
>   FFLIBS-$(CONFIG_PAN_FILTER)                  += swresample
>   FFLIBS-$(CONFIG_REMOVELOGO_FILTER)           += avformat avcodec
> @@ -54,6 +55,7 @@ OBJS-$(CONFIG_ASHOWINFO_FILTER)              += af_ashowinfo.o
>   OBJS-$(CONFIG_ASPLIT_FILTER)                 += split.o
>   OBJS-$(CONFIG_ASTREAMSYNC_FILTER)            += af_astreamsync.o
>   OBJS-$(CONFIG_ASYNCTS_FILTER)                += af_asyncts.o
> +OBJS-$(CONFIG_ATEMPO_FILTER)                 += af_atempo.o
>   OBJS-$(CONFIG_EARWAX_FILTER)                 += af_earwax.o
>   OBJS-$(CONFIG_PAN_FILTER)                    += af_pan.o
>   OBJS-$(CONFIG_RESAMPLE_FILTER)               += af_resample.o
> diff --git a/libavfilter/af_atempo.c b/libavfilter/af_atempo.c
> new file mode 100644
> index 0000000..1866d0a
> --- /dev/null
> +++ b/libavfilter/af_atempo.c
> @@ -0,0 +1,1245 @@
> +/*
> + * Copyright (c) 2012 Pavel Koshevoy
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * tempo scaling audio filter -- an implementation of WSOLA algorithm
> + */
> +
> +#include<float.h>
> +#include "libavcodec/avfft.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/eval.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/samplefmt.h"
> +#include "avfilter.h"
> +#include "audio.h"
> +#include "internal.h"
> +
> +/**
> + * A fragment of audio waveform
> + */
> +typedef struct {
> +    // index of the first sample of this fragment in the overall waveform;
> +    // 0: input sample position
> +    // 1: output sample position
> +    int64_t position[2];
> +
> +    // original packed multi-channel samples:
> +    unsigned char *data;
> +
> +    // number of samples in this fragment:
> +    int nsamples;
> +
> +    // FFT transform of the downmixed mono fragment, used for
> +    // fast waveform alignment via correlation in frequency domain:
> +    FFTComplex *xdat;
> +
> +} TAudioFragment;
> +
> +/**
> + * Filter state machine states
> + */
> +typedef enum {
> +    kLoadFragment     = 0,
> +    kAdjustPosition   = 1,
> +    kReloadFragment   = 2,
> +    kOutputOverlapAdd = 3,
> +    kFlushOutput      = 4
> +
> +} TState;
> +
> +/**
> + * Filter state machine
> + */
> +typedef struct {
> +    // ring-buffer of input samples, necessary because some times
> +    // input fragment position may be adjusted backwards:
> +    unsigned char *buffer;
> +
> +    // ring-buffer maximum capacity,
> +    // expressed as number of multi-channel sample units;
> +    //
> +    // for example, given stereo data 1 multi-channel sample unit
> +    // refers to 2 samples for left/right channels:
> +    int ring;
> +
> +    // ring-buffer house keeping:
> +    int size;
> +    int head;
> +    int tail;
> +
> +    // 0: input sample position corresponding to the ring buffer tail
> +    // 1: output sample position
> +    int64_t position[2];
> +
> +    // sample format:
> +    enum AVSampleFormat format;
> +
> +    // number of channels:
> +    int channels;
> +
> +    // row of bytes to skip from one sample to next, across multple channels;
> +    // stride = (number-of-channels * bits-per-sample-per-channel) / 8
> +    int stride;
> +
> +    // fragment window size, power-of-two integer:
> +    int window;
> +
> +    // Hann window coefficients, for feathering
> +    // (blending) the overlapping fragment region:
> +    float *hann;
> +
> +    // tempo scaling factor:
> +    double tempo;
> +
> +    // cumulative alignment drift:
> +    int drift;
> +
> +    // current/previous fragment ring-buffer:
> +    TAudioFragment frag[2];
> +
> +    // current fragment index:
> +    uint64_t nfrag;
> +
> +    // current state:
> +    TState state;
> +
> +    // for fast correlation calculation in frequency domain:
> +    FFTContext *fft_forward;
> +    FFTContext *fft_inverse;
> +    FFTComplex *correlation;
> +
> +    // for managing AVFilterPad::request_frame and AVFilterPad::filter_samples
> +    int request_fulfilled;
> +    AVFilterBufferRef *dst_buffer;
> +    unsigned char *dst;
> +    unsigned char *dst_end;
> +    uint64_t nsamples_in;
> +    uint64_t nsamples_out;
> +
> +} ATempoContext;
> +
> +/**
> + * Initialize filter state.
> + */
> +static void yae_constructor(ATempoContext *atempo)
> +{
> +    atempo->ring = 0;
> +    atempo->size = 0;
> +    atempo->head = 0;
> +    atempo->tail = 0;
> +
> +    atempo->format = AV_SAMPLE_FMT_NONE;
> +    atempo->channels = 0;
> +
> +    atempo->window = 0;
> +    atempo->tempo = 1.0;
> +    atempo->drift = 0;
> +
> +    memset(&atempo->frag[0], 0, sizeof(atempo->frag));
> +
> +    atempo->nfrag = 0;
> +    atempo->state = kLoadFragment;
> +
> +    atempo->position[0] = 0;
> +    atempo->position[1] = 0;
> +
> +    atempo->fft_forward = NULL;
> +    atempo->fft_inverse = NULL;
> +    atempo->correlation = NULL;
> +
> +    atempo->request_fulfilled = 0;
> +    atempo->dst_buffer = NULL;
> +    atempo->dst = NULL;
> +    atempo->dst_end = NULL;
> +    atempo->nsamples_in = 0;
> +    atempo->nsamples_out = 0;
> +}
> +
> +/**
> + * Deallocate filter buffers.
> + */
> +static void yae_destructor(ATempoContext *atempo)
> +{
> +    av_freep(&atempo->frag[0].data);
> +    av_freep(&atempo->frag[1].data);
> +    av_freep(&atempo->frag[0].xdat);
> +    av_freep(&atempo->frag[1].xdat);
> +
> +    av_freep(&atempo->buffer);
> +    av_freep(&atempo->hann);
> +    av_freep(&atempo->correlation);
> +
> +    if (atempo->fft_forward) {
> +        av_fft_end(atempo->fft_forward);
> +        atempo->fft_forward = NULL;
> +    }
> +
> +    if (atempo->fft_inverse) {
> +        av_fft_end(atempo->fft_inverse);
> +        atempo->fft_inverse = NULL;
> +    }
> +
> +}
> +
> +/**
> + * Reset given fragment to initial state
> + */
> +static void yae_clear_frag(TAudioFragment *frag)
> +{
> +    frag->position[0] = 0;
> +    frag->position[1] = 0;
> +    frag->nsamples    = 0;
> +}
> +
> +/**
> + * Reset filter to initial state
> + */
> +static void yae_clear(ATempoContext *atempo)
> +{
> +    atempo->size = 0;
> +    atempo->head = 0;
> +    atempo->tail = 0;
> +
> +    atempo->drift = 0;
> +    atempo->nfrag = 0;
> +    atempo->state = kLoadFragment;
> +
> +    atempo->position[0] = 0;
> +    atempo->position[1] = 0;
> +
> +    yae_clear_frag(&atempo->frag[0]);
> +    yae_clear_frag(&atempo->frag[1]);
> +
> +    // shift left position of 1st fragment by half a window
> +    // so that no re-normalization would be required for
> +    // the left half of the 1st fragment:
> +    atempo->frag[0].position[0] = -(int64_t)(atempo->window / 2);
> +    atempo->frag[0].position[1] = -(int64_t)(atempo->window / 2);
> +
> +    if (atempo->dst_buffer) {
> +        avfilter_unref_buffer(atempo->dst_buffer);
> +        atempo->dst_buffer = NULL;
> +        atempo->dst        = NULL;
> +        atempo->dst_end    = NULL;
> +    }
> +
> +    atempo->request_fulfilled = 0;
> +    atempo->nsamples_in       = 0;
> +    atempo->nsamples_out      = 0;
> +}
> +
> +/**
> + * Prepare filter for processing audio data of given format,
> + * sample rate and number of channels.
> + */
> +static void yae_reset(ATempoContext *atempo,
> +                      enum AVSampleFormat format,
> +                      int sample_rate,
> +                      int channels)
> +{
> +    const int sample_size = av_get_bytes_per_sample(format);
> +    unsigned int nlevels  = 0;
> +    unsigned int pot;
> +
> +    atempo->format   = format;
> +    atempo->channels = channels;
> +    atempo->stride   = sample_size * channels;
> +
> +    // pick a segment window size:
> +    atempo->window = sample_rate / 24;
> +
> +    // adjust window size to be a power-of-two integer:
> +    nlevels = av_log2_c(atempo->window);
> +    pot = 1<<  nlevels;
> +    av_assert0(pot<= atempo->window);
> +
> +    if (pot<  atempo->window) {
> +        atempo->window = pot * 2;
> +        nlevels++;
> +    }
> +
> +    atempo->frag[0].data = av_realloc(atempo->frag[0].data,
> +                                      atempo->window * atempo->stride);
> +
> +    atempo->frag[1].data = av_realloc(atempo->frag[1].data,
> +                                      atempo->window * atempo->stride);
> +
> +    atempo->frag[0].xdat = av_realloc(atempo->frag[0].xdat,
> +                                      atempo->window * 2 *
> +                                      sizeof(FFTComplex));
> +
> +    atempo->frag[1].xdat = av_realloc(atempo->frag[1].xdat,
> +                                      atempo->window * 2 *
> +                                      sizeof(FFTComplex));
> +
> +    // initialize FFT contexts:
> +    if (atempo->fft_forward) {
> +        av_fft_end(atempo->fft_forward);
> +    }
> +
> +    if (atempo->fft_inverse) {
> +        av_fft_end(atempo->fft_inverse);
> +    }
> +
> +    atempo->fft_forward = av_fft_init(nlevels + 1, 0);
> +    atempo->fft_inverse = av_fft_init(nlevels + 1, 1);
> +    atempo->correlation = (FFTComplex *)av_realloc(atempo->correlation,
> +                                                   atempo->window * 2 *
> +                                                   sizeof(FFTComplex));
> +
> +    atempo->ring = atempo->window * 3;
> +    atempo->buffer = av_realloc(atempo->buffer, atempo->ring * atempo->stride);
> +
> +    // sample the Hann window function:
> +    atempo->hann = av_realloc(atempo->hann, atempo->window * sizeof(float));
> +    for (int i = 0; i<  atempo->window; i++) {
> +        double t = (double)i / (double)(atempo->window - 1);
> +        double h = 0.5 * (1.0 - cos(2.0 * M_PI * t));
> +        atempo->hann[i] = (float)h;
> +    }
> +
> +    yae_clear(atempo);
> +}
> +
> +static int yae_set_tempo(ATempoContext *atempo,
> +                         double tempo,
> +                         AVFilterContext *ctx)
> +{
> +    if (tempo<  0.5 || tempo>  2.0) {
> +        av_log(ctx, AV_LOG_ERROR, "tempo value %f exceeds [0.5, 2.0] range\n",
> +               tempo);
> +        return AVERROR(EINVAL);
> +    }
> +
> +    atempo->tempo = tempo;
> +    return 0;
> +}
> +
> +inline static TAudioFragment * yae_curr_frag(ATempoContext *atempo)
> +{
> +    return&atempo->frag[atempo->nfrag % 2];
> +}
> +
> +inline static TAudioFragment * yae_prev_frag(ATempoContext *atempo)
> +{
> +    return&atempo->frag[(atempo->nfrag + 1) % 2];
> +}
> +
> +/**
> + * Find the minimum of two scalars
> + */
> +#define yae_min(TScalar, a, b)                          \
> +    ((TScalar)a<  (TScalar)b ?                          \
> +     (TScalar)a :                                       \
> +     (TScalar)b)
> +
> +/**
> + * Find the maximum of two scalars
> + */
> +#define yae_max(TScalar, a, b)                          \
> +    ((TScalar)a<  (TScalar)b ?                          \
> +     (TScalar)b :                                       \
> +     (TScalar)a)
> +
> +
> +/**
> + * A helper macro for initializing complex data buffer with scalar data
> + * of a given type.
> + */
> +#define yae_init_xdat(TScalar, scalar_max)                              \
> +    do {                                                                \
> +        const unsigned char *src_end =                                  \
> +            src + frag->nsamples * atempo->channels * sizeof(TScalar);  \
> +                                                                        \
> +        FFTComplex *xdat = frag->xdat;                                  \
> +        TScalar tmp;                                                    \
> +                                                                        \
> +        if (atempo->channels == 1) {                                    \
> +            float s;                                                    \
> +                                                                        \
> +            for (; src<  src_end; blend++) {                            \
> +                memcpy(&tmp, src, sizeof(TScalar));                     \
> +                src += sizeof(TScalar);                                 \
> +                                                                        \
> +                s = (float)tmp;                                         \
> +                                                                        \
> +                xdat->re = s;                                           \
> +                xdat->im = 0;                                           \
> +                xdat++;                                                 \
> +            }                                                           \
> +        } else {                                                        \
> +            float s;                                                    \
> +            float t0;                                                   \
> +            float max;                                                  \
> +            float ti;                                                   \
> +            float s0;                                                   \
> +                                                                        \
> +            for (; src<  src_end; blend++) {                            \
> +                memcpy(&tmp, src, sizeof(TScalar));                     \
> +                src += sizeof(TScalar);                                 \
> +                                                                        \
> +                t0 = (float)tmp;                                        \
> +                s = yae_min(float, scalar_max, fabsf(t0));              \
> +                max = (float)t0;                                        \
> +                                                                        \
> +                for (int i = 1; i<  atempo->channels; i++) {            \
> +                    memcpy(&tmp, src, sizeof(TScalar));                 \
> +                    src += sizeof(TScalar);                             \
> +                                                                        \
> +                    ti = (float)tmp;                                    \
> +                    s0 = yae_min(float, scalar_max, fabsf(ti));         \
> +                                                                        \
> +                    if (s<  s0) {                                       \
> +                        s = s0;                                         \
> +                        max = ti;                                       \
> +                    }                                                   \
> +                }                                                       \
> +                                                                        \
> +                xdat->re = max;                                         \
> +                xdat->im = 0;                                           \
> +                xdat++;                                                 \
> +            }                                                           \
> +        }                                                               \
> +    } while (0)
> +
> +/**
> + * Initialize complex data buffer of a given audio fragment
> + * with down-mixed mono data of appropriate scalar type.
> + */
> +static void yae_downmix(ATempoContext *atempo, TAudioFragment *frag)
> +{
> +    // shortcuts:
> +    const unsigned char *src = frag->data;
> +    const float *blend = atempo->hann;
> +
> +    // init complex data buffer used for FFT and Correlation:
> +    memset(frag->xdat, 0, sizeof(FFTComplex) * atempo->window * 2);
> +
> +    if (atempo->format == AV_SAMPLE_FMT_U8) {
> +        yae_init_xdat(unsigned char, 127);
> +    } else if (atempo->format == AV_SAMPLE_FMT_S16) {
> +        yae_init_xdat(short int, 32767);
> +    } else if (atempo->format == AV_SAMPLE_FMT_S32) {
> +        yae_init_xdat(int, 2147483647);
> +    } else if (atempo->format == AV_SAMPLE_FMT_FLT) {
> +        yae_init_xdat(float, 1);
> +    } else if (atempo->format == AV_SAMPLE_FMT_DBL) {
> +        yae_init_xdat(double, 1);
> +    }
> +}
> +
> +/**
> + * Apply Fourier Transform to a given audio fragment.
> + */
> +static void yae_transform(TAudioFragment *frag, FFTContext *fft)
> +{
> +    av_fft_permute(fft, frag->xdat);
> +    av_fft_calc(fft, frag->xdat);
> +}
> +
> +/**
> + * Populate the internal data buffer on as-needed basis.
> + *
> + * @return 0 if requested data was already available or was loaded.
> + * @return AVERROR(EAGAIN) if more input data is required.
> + */
> +static int yae_load_data(ATempoContext *atempo,
> +                         const unsigned char **src_ref,
> +                         const unsigned char *src_end,
> +                         int64_t stop_here)
> +{
> +    // shortcut:
> +    const unsigned char *src = *src_ref;
> +    const int read_size = stop_here - atempo->position[0];
> +
> +    if (stop_here<= atempo->position[0]) {
> +        return 0;
> +    }
> +
> +    // samples are not expected to be skipped:
> +    av_assert0(read_size<= atempo->ring);
> +
> +    while (atempo->position[0]<  stop_here&&  src<  src_end) {
> +        int src_samples = (src_end - src) / atempo->stride;
> +
> +        // load data piece-wise, in order to avoid complicating the logic:
> +        int nsamples = yae_min(int, read_size, src_samples);
> +        int na;
> +        int nb;
> +
> +        nsamples = yae_min(int, nsamples, atempo->ring);
> +        na = yae_min(int, nsamples, atempo->ring - atempo->tail);
> +        nb = yae_min(int, nsamples - na, atempo->ring);
> +
> +        if (na) {
> +            unsigned char *a = atempo->buffer + atempo->tail * atempo->stride;
> +            memcpy(a, src, na * atempo->stride);
> +
> +            src += na * atempo->stride;
> +            atempo->position[0] += na;
> +
> +            atempo->size = yae_min(int, atempo->size + na, atempo->ring);
> +            atempo->tail = (atempo->tail + na) % atempo->ring;
> +            atempo->head =
> +                atempo->size<  atempo->ring ?
> +                atempo->tail - atempo->size :
> +                atempo->tail;
> +        }
> +
> +        if (nb) {
> +            unsigned char *b = atempo->buffer;
> +            memcpy(b, src, nb * atempo->stride);
> +
> +            src += nb * atempo->stride;
> +            atempo->position[0] += nb;
> +
> +            atempo->size = yae_min(int, atempo->size + nb, atempo->ring);
> +            atempo->tail = (atempo->tail + nb) % atempo->ring;
> +            atempo->head =
> +                atempo->size<  atempo->ring ?
> +                atempo->tail - atempo->size :
> +                atempo->tail;
> +        }
> +    }
> +
> +    // pass back the updated source buffer pointer:
> +    *src_ref = src;
> +
> +    // sanity check:
> +    av_assert0(atempo->position[0]<= stop_here);
> +
> +    return atempo->position[0] == stop_here ? 0 : AVERROR(EAGAIN);
> +}
> +
> +/**
> + * Populate current audio fragment data buffer.
> + *
> + * @return 0 when the fragment is ready.
> + * @return AVERROR(EAGAIN) if more input data is required.
> + */
> +static int yae_load_frag(ATempoContext *atempo,
> +                         const unsigned char **src_ref,
> +                         const unsigned char *src_end)
> +{
> +    // shortcuts:
> +    TAudioFragment *frag = yae_curr_frag(atempo);
> +    unsigned char *dst;
> +    int64_t missing;
> +    unsigned int nsamples;
> +    int64_t start;
> +    int64_t zeros;
> +    int na;
> +    int nb;
> +    const unsigned char *a;
> +    const unsigned char *b;
> +    int i0;
> +    int i1;
> +    int n0;
> +    int n1;
> +
> +    int64_t stop_here = frag->position[0] + atempo->window;
> +    if (src_ref&&  yae_load_data(atempo, src_ref, src_end, stop_here) != 0) {
> +        return AVERROR(EAGAIN);
> +    }
> +
> +    // calculate the number of samples we don't have:
> +    missing =
> +        stop_here>  atempo->position[0] ?
> +        stop_here - atempo->position[0] : 0;
> +
> +    nsamples =
> +        missing<  (int64_t)atempo->window ?
> +        (unsigned int)(atempo->window - missing) : 0;
> +
> +    // setup the output buffer:
> +    frag->nsamples = nsamples;
> +    dst = frag->data;
> +
> +    start = atempo->position[0] - atempo->size;
> +    zeros = 0;
> +
> +    if (frag->position[0]<  start) {
> +        // what we don't have we substitute with zeros:
> +        zeros = yae_min(int, start - frag->position[0], nsamples);
> +        av_assert0(zeros != nsamples);
> +
> +        memset(dst, 0, zeros * atempo->stride);
> +        dst += zeros * atempo->stride;
> +    }
> +
> +    if (zeros == nsamples) {
> +        return 0;
> +    }
> +
> +    // get the remaining data from the ring buffer:
> +    na = (atempo->head<  atempo->tail ?
> +          atempo->tail - atempo->head :
> +          atempo->ring - atempo->head);
> +
> +    nb = atempo->head<  atempo->tail ? 0 : atempo->tail;
> +
> +    // sanity check:
> +    av_assert0(nsamples<= zeros + na + nb);
> +
> +    a = atempo->buffer + atempo->head * atempo->stride;
> +    b = atempo->buffer;
> +
> +    i0 = frag->position[0] + zeros - start;
> +    i1 = i0<  na ? 0 : i0 - na;
> +
> +    n0 = i0<  na ? yae_min(int, na - i0, nsamples - zeros) : 0;
> +    n1 = nsamples - zeros - n0;
> +
> +    if (n0) {
> +        memcpy(dst, a + i0 * atempo->stride, n0 * atempo->stride);
> +        dst += n0 * atempo->stride;
> +    }
> +
> +    if (n1) {
> +        memcpy(dst, b + i1 * atempo->stride, n1 * atempo->stride);
> +        dst += n1 * atempo->stride;
> +    }
> +
> +    return 0;
> +}
> +
> +/**
> + * Prepare for loading next audio fragment.
> + */
> +static void yae_advance_to_next_frag(ATempoContext *atempo)
> +{
> +    const double fragment_step = atempo->tempo * (double)(atempo->window / 2);
> +
> +    const TAudioFragment *prev;
> +    TAudioFragment       *frag;
> +
> +    atempo->nfrag++;
> +    prev = yae_prev_frag(atempo);
> +    frag = yae_curr_frag(atempo);
> +
> +    frag->position[0] = prev->position[0] + (int64_t)fragment_step;
> +    frag->position[1] = prev->position[1] + atempo->window / 2;
> +    frag->nsamples    = 0;
> +}
> +
> +/**
> + * Calculate alignment offset for given fragment
> + * relative to the previous fragment.
> + *
> + * @return alignment offset of current fragment relative to previous.
> + */
> +static int yae_align(TAudioFragment *frag,
> +                     const TAudioFragment *prev,
> +                     const int window,
> +                     const int delta_max,
> +                     const int drift,
> +                     FFTComplex *correlation,
> +                     FFTContext *fft_inverse)
> +{
> +    const FFTComplex *xa = prev->xdat;
> +    const FFTComplex *xb = frag->xdat;
> +    FFTComplex       *xc = correlation;
> +
> +    int       best_offset = -drift;
> +    FFTSample best_metric = -FLT_MAX;
> +
> +    int i0;
> +    int i1;
> +
> +    for (int i = 0; i<  window * 2; i++, xa++, xb++, xc++) {
> +        xc->re = (xa->re * xb->re + xa->im * xb->im);
> +        xc->im = (xa->im * xb->re - xa->re * xb->im);
> +    }
> +
> +    // apply inverse FFT:
> +    av_fft_permute(fft_inverse, correlation);
> +    av_fft_calc(fft_inverse, correlation);
> +
> +    // identify peaks:
> +
> +    i0 = yae_max(int, window / 2 - delta_max - drift, 0);
> +    i0 = yae_min(int, i0, window);
> +
> +    i1 = yae_min(int, window / 2 + delta_max - drift, window - window / 16);
> +    i1 = yae_max(int, i1, 0);
> +
> +    xc = correlation + i0;
> +    for (int i = i0; i<  i1; i++, xc++) {
> +        FFTSample metric = xc->re;
> +
> +        // normalize:
> +        FFTSample drifti = (FFTSample)(drift + i);
> +        metric *= drifti * drifti;
> +
> +        if (metric>  best_metric) {
> +            best_metric = metric;
> +            best_offset = i - window / 2;
> +        }
> +    }
> +
> +    return best_offset;
> +}
> +
> +/**
> + * Adjust current fragment position for better alignment
> + * with previous fragment.
> + *
> + * @return alignment correction.
> + */
> +static int yae_adjust_position(ATempoContext *atempo)
> +{
> +    const TAudioFragment *prev = yae_prev_frag(atempo);
> +    TAudioFragment       *frag = yae_curr_frag(atempo);
> +
> +    const int delta_max  = atempo->window / 2;
> +    const int correction = yae_align(frag,
> +                                     prev,
> +                                     atempo->window,
> +                                     delta_max,
> +                                     atempo->drift,
> +                                     atempo->correlation,
> +                                     atempo->fft_inverse);
> +
> +    if (correction) {
> +        // adjust fragment position:
> +        frag->position[0] -= correction;
> +
> +        // clear so that the fragment can be reloaded:
> +        frag->nsamples = 0;
> +
> +        // update cumulative correction drift counter:
> +        atempo->drift += correction;
> +    }
> +
> +    return correction;
> +}
> +
> +/**
> + * A helper macro for blending the overlap region of previous
> + * and current audio fragment.
> + */
> +#define yae_blend(TScalar)                                              \
> +    do {                                                                \
> +        const TScalar *aaa = (const TScalar *)a;                        \
> +        const TScalar *bbb = (const TScalar *)b;                        \
> +                                                                        \
> +        TScalar *out     = (TScalar *)dst;                              \
> +        TScalar *out_end = (TScalar *)dst_end;                          \
> +                                                                        \
> +        for (int64_t i = 0; i<  overlap&&  out<  out_end;               \
> +             i++, atempo->position[1]++, wa++, wb++) {                  \
> +            float w0 = *wa;                                             \
> +            float w1 = *wb;                                             \
> +                                                                        \
> +            for (int j = 0; j<  atempo->channels;                       \
> +                 j++, aaa++, bbb++, out++) {                            \
> +                float t0 = (float)*aaa;                                 \
> +                float t1 = (float)*bbb;                                 \
> +                                                                        \
> +                *out =                                                  \
> +                    frag->position[0] + i<  0 ?                         \
> +                    *aaa :                                              \
> +                    (TScalar)(t0 * w0 + t1 * w1);                       \
> +            }                                                           \
> +        }                                                               \
> +        dst = (unsigned char *)out;                                     \
> +    } while (0)
> +
> +/**
> + * Blend the overlap region of previous and current audio fragment
> + * and output the results to the given destination buffer.
> + *
> + * @return 0 if the overlap region was completely stored in the dst buffer.
> + * @return AVERROR(EAGAIN) if more destination buffer space is required.
> + */
> +static int yae_overlap_add(ATempoContext  *atempo,
> +                           unsigned char **dst_ref,
> +                           unsigned char  *dst_end)
> +{
> +    // shortcuts:
> +    const TAudioFragment *prev = yae_prev_frag(atempo);
> +    const TAudioFragment *frag = yae_curr_frag(atempo);
> +
> +    const int64_t start_here = yae_max(int64_t,
> +                                       atempo->position[1],
> +                                       frag->position[1]);
> +
> +    const int64_t stop_here = yae_min(int64_t,
> +                                      prev->position[1] + prev->nsamples,
> +                                      frag->position[1] + frag->nsamples);
> +
> +    const int64_t overlap = stop_here - start_here;
> +
> +    const int64_t ia = start_here - prev->position[1];
> +    const int64_t ib = start_here - frag->position[1];
> +
> +    const float *wa = atempo->hann + ia;
> +    const float *wb = atempo->hann + ib;
> +
> +    const unsigned char *a = prev->data + ia * atempo->stride;
> +    const unsigned char *b = frag->data + ib * atempo->stride;
> +
> +    unsigned char *dst = *dst_ref;
> +
> +    av_assert0(start_here<= stop_here&&
> +               frag->position[1]<= start_here&&
> +               overlap<= frag->nsamples);
> +
> +    if (atempo->format == AV_SAMPLE_FMT_U8) {
> +        yae_blend(unsigned char);
> +    } else if (atempo->format == AV_SAMPLE_FMT_S16) {
> +        yae_blend(short int);
> +    } else if (atempo->format == AV_SAMPLE_FMT_S32) {
> +        yae_blend(int);
> +    } else if (atempo->format == AV_SAMPLE_FMT_FLT) {
> +        yae_blend(float);
> +    } else if (atempo->format == AV_SAMPLE_FMT_DBL) {
> +        yae_blend(double);
> +    }
> +
> +    // pass-back the updated destination buffer pointer:
> +    *dst_ref = dst;
> +
> +    return atempo->position[1] == stop_here ? 0 : AVERROR(EAGAIN);
> +}
> +
> +/**
> + * Feed as much data to the filter as it is able to consume
> + * and receive as much processed data in the destination buffer
> + * as it is able to produce or store.
> + */
> +static void
> +yae_apply(ATempoContext *atempo,
> +          const unsigned char **src_ref,
> +          const unsigned char *src_end,
> +          unsigned char **dst_ref,
> +          unsigned char *dst_end)
> +{
> +    while (1) {
> +        if (atempo->state == kLoadFragment) {
> +            // load additional data for the current fragment:
> +            if (yae_load_frag(atempo, src_ref, src_end) != 0) {
> +                break;
> +            }
> +
> +            // build a multi-resolution pyramid for fragment alignment:
> +            yae_downmix(atempo, yae_curr_frag(atempo));
> +
> +            // apply FFT:
> +            yae_transform(yae_curr_frag(atempo), atempo->fft_forward);
> +
> +            // must load the second fragment before alignment can start:
> +            if (!atempo->nfrag) {
> +                yae_advance_to_next_frag(atempo);
> +                continue;
> +            }
> +
> +            atempo->state = kAdjustPosition;
> +        }
> +
> +        if (atempo->state == kAdjustPosition) {
> +            // adjust position for better alignment:
> +            if (yae_adjust_position(atempo)) {
> +                // reload the fragment at the corrected position, so that the
> +                // Hann window blending would not require normalization:
> +                atempo->state = kReloadFragment;
> +            } else {
> +                atempo->state = kOutputOverlapAdd;
> +            }
> +        }
> +
> +        if (atempo->state == kReloadFragment) {
> +            // load additional data if necessary due to position adjustment:
> +            if (yae_load_frag(atempo, src_ref, src_end) != 0) {
> +                break;
> +            }
> +
> +            // build a multi-resolution pyramid for fragment alignment:
> +            yae_downmix(atempo, yae_curr_frag(atempo));
> +
> +            // apply FFT:
> +            yae_transform(yae_curr_frag(atempo), atempo->fft_forward);
> +
> +            atempo->state = kOutputOverlapAdd;
> +        }
> +
> +        if (atempo->state == kOutputOverlapAdd) {
> +            // overlap-add and output the result:
> +            if (yae_overlap_add(atempo, dst_ref, dst_end) != 0) {
> +                break;
> +            }
> +
> +            // advance to the next fragment, repeat:
> +            yae_advance_to_next_frag(atempo);
> +            atempo->state = kLoadFragment;
> +        }
> +    }
> +}
> +
> +/**
> + * Flush any buffered data from the filter.
> + *
> + * @return 0 if all data was completely stored in the dst buffer.
> + * @return AVERROR(EAGAIN) if more destination buffer space is required.
> + */
> +static int yae_flush(ATempoContext *atempo,
> +                     unsigned char **dst_ref,
> +                     unsigned char *dst_end)
> +{
> +    TAudioFragment *frag = yae_curr_frag(atempo);
> +    int64_t overlap_end;
> +    int64_t start_here;
> +    int64_t stop_here;
> +    int64_t offset;
> +
> +    const unsigned char *src;
> +    unsigned char *dst;
> +
> +    int src_size;
> +    int dst_size;
> +    int nbytes;
> +
> +    atempo->state = kFlushOutput;
> +
> +    if (atempo->position[0] == frag->position[0] + frag->nsamples&&
> +        atempo->position[1] == frag->position[1] + frag->nsamples) {
> +        // the current fragment is already flushed:
> +        return 0;
> +    }
> +
> +    if (frag->position[0] + frag->nsamples<  atempo->position[0]) {
> +        // finish loading the current (possibly partial) fragment:
> +        yae_load_frag(atempo, NULL, NULL);
> +
> +        if (atempo->nfrag) {
> +            // build a multi-resolution pyramid for fragment alignment:
> +            yae_downmix(atempo, frag);
> +
> +            // apply FFT:
> +            yae_transform(frag, atempo->fft_forward);
> +
> +            // align current fragment to previous fragment:
> +            if (yae_adjust_position(atempo)) {
> +                // reload the current fragment due to adjusted position:
> +                yae_load_frag(atempo, NULL, NULL);
> +            }
> +        }
> +    }
> +
> +    // flush the overlap region:
> +    overlap_end = frag->position[1] + yae_min(int64_t,
> +                                              atempo->window / 2,
> +                                              frag->nsamples);
> +
> +    while (atempo->position[1]<  overlap_end) {
> +        if (yae_overlap_add(atempo, dst_ref, dst_end) != 0) {
> +            return AVERROR(EAGAIN);
> +        }
> +    }
> +
> +    // flush the remaininder of the current fragment:
> +    start_here = yae_max(int64_t, atempo->position[1], overlap_end);
> +    stop_here = frag->position[1] + frag->nsamples;
> +    offset = start_here - frag->position[1];
> +    av_assert0(start_here<= stop_here&&  frag->position[1]<= start_here);
> +
> +    src = frag->data + offset * atempo->stride;
> +    dst = (unsigned char *)*dst_ref;
> +
> +    src_size = (int)(stop_here - start_here) * atempo->stride;
> +    dst_size = dst_end - dst;
> +    nbytes = yae_min(int, src_size, dst_size);
> +
> +    memcpy(dst, src, nbytes);
> +    dst += nbytes;
> +
> +    atempo->position[1] += (nbytes / atempo->stride);
> +
> +    // pass-back the updated destination buffer pointer:
> +    *dst_ref = (unsigned char *)dst;
> +
> +    return atempo->position[1] == stop_here ? 0 : AVERROR(EAGAIN);
> +}
> +
> +static av_cold int init(AVFilterContext *ctx, const char *args, void *opaque)
> +{
> +    ATempoContext *atempo = ctx->priv;
> +    yae_constructor(atempo);
> +
> +    if (args) {
> +        char   *tail = NULL;
> +        double tempo = av_strtod(args,&tail);
> +        return yae_set_tempo(atempo, tempo, ctx);
> +    }
> +
> +    return 0;
> +}
> +
> +/**
> + * Deallocate any memory held by the filter,
> + * release any buffer references, etc.
> + *
> + * This does not need to deallocate the AVFilterContext::priv memory itself.
> + */
> +static av_cold void uninit(AVFilterContext *ctx)
> +{
> +    ATempoContext *atempo = ctx->priv;
> +    yae_destructor(atempo);
> +}
> +
> +static int query_formats(AVFilterContext *ctx)
> +{
> +    AVFilterChannelLayouts *layouts = NULL;
> +    AVFilterFormats        *formats = NULL;
> +
> +    enum AVSampleFormat sample_fmts[] = {
> +        AV_SAMPLE_FMT_U8,
> +        AV_SAMPLE_FMT_S16,
> +        AV_SAMPLE_FMT_S32,
> +        AV_SAMPLE_FMT_FLT,
> +        AV_SAMPLE_FMT_DBL,
> +        AV_SAMPLE_FMT_NONE
> +    };
> +
> +    layouts = ff_all_channel_layouts();
> +    if (!layouts) {
> +        return AVERROR(ENOMEM);
> +    }
> +    ff_set_common_channel_layouts(ctx, layouts);
> +
> +    formats = avfilter_make_format_list(sample_fmts);
> +    if (!formats) {
> +        return AVERROR(ENOMEM);
> +    }
> +    avfilter_set_common_sample_formats(ctx, formats);
> +
> +    formats = ff_all_samplerates();
> +    if (!formats) {
> +        return AVERROR(ENOMEM);
> +    }
> +
> +    ff_set_common_samplerates(ctx, formats);
> +    return 0;
> +}
> +
> +/**
> + * Check configuration properties of the input link
> + * and update filters internal state as necessary.
> + *
> + * Return zero on success, AVERRROR(..) on failure.
> + */
> +static int
> +config_props(AVFilterLink *inlink)
> +{
> +    AVFilterContext  *ctx = inlink->dst;
> +    ATempoContext *atempo = ctx->priv;
> +
> +    enum AVSampleFormat format = (enum AVSampleFormat)inlink->format;
> +    int sample_rate = (int)inlink->sample_rate;
> +    int channels = av_get_channel_layout_nb_channels(inlink->channel_layout);
> +
> +    yae_reset(atempo, format, sample_rate, channels);
> +    return 0;
> +}
> +
> +/**
> + * Samples filtering callback that.
> + * This is where a filter receives audio data and processes it.
> + */
> +static void filter_samples(AVFilterLink *inlink,
> +                           AVFilterBufferRef *src_buffer)
> +{
> +    AVFilterContext  *ctx = inlink->dst;
> +    ATempoContext *atempo = ctx->priv;
> +    AVFilterLink *outlink = ctx->outputs[0];
> +
> +    int n_in = src_buffer->audio->nb_samples;
> +    int n_out = (int)(0.5 + ((double)n_in) / atempo->tempo);
> +
> +    const unsigned char *src = src_buffer->data[0];
> +    const unsigned char *src_end = src + n_in * atempo->stride;
> +
> +    while (src<  src_end) {
> +        if (!atempo->dst_buffer) {
> +            atempo->dst_buffer = ff_get_audio_buffer(outlink,
> +                                                     AV_PERM_WRITE,
> +                                                     n_out);
> +            avfilter_copy_buffer_ref_props(atempo->dst_buffer, src_buffer);
> +
> +            atempo->dst = atempo->dst_buffer->data[0];
> +            atempo->dst_end = atempo->dst + n_out * atempo->stride;
> +        }
> +
> +        yae_apply(atempo,&src, src_end,&atempo->dst, atempo->dst_end);
> +
> +        if (atempo->dst == atempo->dst_end) {
> +            atempo->dst_buffer->audio->sample_rate = outlink->sample_rate;
> +            atempo->dst_buffer->audio->nb_samples  = n_out;
> +
> +            // adjust the PTS:
> +            atempo->dst_buffer->pts =
> +                av_rescale(outlink->time_base.den,
> +                           atempo->nsamples_out,
> +                           outlink->time_base.num * outlink->sample_rate);
> +
> +            ff_filter_samples(outlink, atempo->dst_buffer);
> +            atempo->dst_buffer = NULL;
> +            atempo->dst        = NULL;
> +            atempo->dst_end    = NULL;
> +
> +            atempo->nsamples_out += n_out;
> +            atempo->request_fulfilled = 1;
> +        }
> +    }
> +
> +    atempo->nsamples_in += n_in;
> +    avfilter_unref_buffer(src_buffer);
> +}
> +
> +/**
> + * Frame request callback.  A call to this should result in at least
> + * one frame being output over the given link.  This should return
> + * zero on success.
> + */
> +static int request_frame(AVFilterLink *outlink)
> +{
> +    AVFilterContext  *ctx = outlink->src;
> +    ATempoContext *atempo = ctx->priv;
> +    int ret;
> +
> +    atempo->request_fulfilled = 0;
> +    do {
> +        ret = avfilter_request_frame(ctx->inputs[0]);
> +    }
> +    while (!atempo->request_fulfilled&&  ret>= 0);
> +
> +    if (ret == AVERROR_EOF) {
> +        // flush the filter:
> +        int n_max = atempo->ring;
> +        int n_out;
> +        int err = AVERROR(EAGAIN);
> +
> +        while (err == AVERROR(EAGAIN)) {
> +            if (!atempo->dst_buffer) {
> +                atempo->dst_buffer = ff_get_audio_buffer(outlink,
> +                                                         AV_PERM_WRITE,
> +                                                         n_max);
> +
> +                atempo->dst = atempo->dst_buffer->data[0];
> +                atempo->dst_end = atempo->dst + n_max * atempo->stride;
> +            }
> +
> +            err = yae_flush(atempo,&atempo->dst, atempo->dst_end);
> +
> +            n_out = ((atempo->dst - atempo->dst_buffer->data[0]) /
> +                     atempo->stride);
> +
> +            if (n_out) {
> +                atempo->dst_buffer->audio->sample_rate = outlink->sample_rate;
> +                atempo->dst_buffer->audio->nb_samples  = n_out;
> +
> +                // adjust the PTS:
> +                atempo->dst_buffer->pts =
> +                    av_rescale(outlink->time_base.den,
> +                               atempo->nsamples_out,
> +                               outlink->time_base.num * outlink->sample_rate);
> +
> +                ff_filter_samples(outlink, atempo->dst_buffer);
> +                atempo->dst_buffer = NULL;
> +                atempo->dst = NULL;
> +                atempo->dst_end = NULL;
> +
> +                atempo->nsamples_out += n_out;
> +            }
> +        }
> +
> +        if (atempo->dst_buffer) {
> +            avfilter_unref_buffer(atempo->dst_buffer);
> +            atempo->dst_buffer = NULL;
> +            atempo->dst = NULL;
> +            atempo->dst_end = NULL;
> +        }
> +
> +        return AVERROR_EOF;
> +    }
> +
> +    return ret;
> +}
> +
> +/**
> + * Make the filter instance process a command.
> + *
> + * @param cmd   -- an alphanumeric command to process.
> + * @param arg   -- the argument for the command.
> + * @param res   -- a buffer with size res_size for filter response.
> + * @param flags -- command flags
> + *
> + * When AVFILTER_CMD_FLAG_FAST flag is set time consuming commands
> + * should be treated as unsupported.
> + *
> + * @return 0 on success, otherwise an error code.
> + * @return AVERROR(ENOSYS) on unsupported commands.
> + */
> +static int process_command(AVFilterContext *ctx,
> +                           const char *cmd,
> +                           const char *arg,
> +                           char *res,
> +                           int res_len,
> +                           int flags)
> +{
> +    ATempoContext *atempo = ctx->priv;
> +
> +    if (strcmp(cmd, "tempo") == 0) {
> +        char   *tail = NULL;
> +        double tempo = av_strtod(arg,&tail);
> +        return yae_set_tempo(atempo, tempo, ctx);
> +    }
> +
> +    return AVERROR(ENOSYS);
> +}
> +
> +AVFilter avfilter_af_atempo = {
> +    .name            = "atempo",
> +    .description     = NULL_IF_CONFIG_SMALL("Adjust audio tempo."),
> +    .init            =&init,
> +    .uninit          =&uninit,
> +    .query_formats   =&query_formats,
> +    .process_command =&process_command,
> +    .priv_size       = sizeof(ATempoContext),
> +
> +    .inputs    = (const AVFilterPad[]) {
> +        { .name            = "default",
> +          .type            = AVMEDIA_TYPE_AUDIO,
> +          .filter_samples  =&filter_samples,
> +          .config_props    =&config_props,
> +          .min_perms       = AV_PERM_READ, },
> +        { .name = NULL}
> +    },
> +
> +    .outputs   = (const AVFilterPad[]) {
> +        { .name            = "default",
> +          .request_frame   =&request_frame,
> +          .type            = AVMEDIA_TYPE_AUDIO, },
> +        { .name = NULL}
> +    },
> +};
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index b9d44f2..e8c8406 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -44,6 +44,7 @@ void avfilter_register_all(void)
>       REGISTER_FILTER (ASPLIT,      asplit,      af);
>       REGISTER_FILTER (ASTREAMSYNC, astreamsync, af);
>       REGISTER_FILTER (ASYNCTS,     asyncts,     af);
> +    REGISTER_FILTER (ATEMPO,      atempo,      af);
>       REGISTER_FILTER (EARWAX,      earwax,      af);
>       REGISTER_FILTER (PAN,         pan,         af);
>       REGISTER_FILTER (SILENCEDETECT, silencedetect, af);
> diff --git a/libavfilter/version.h b/libavfilter/version.h
> index 76f649e..c90b4ad 100644
> --- a/libavfilter/version.h
> +++ b/libavfilter/version.h
> @@ -30,7 +30,7 @@
>
>   #define LIBAVFILTER_VERSION_MAJOR  2
>   #define LIBAVFILTER_VERSION_MINOR 78
> -#define LIBAVFILTER_VERSION_MICRO 100
> +#define LIBAVFILTER_VERSION_MICRO 101
>
>   #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
>                                                  LIBAVFILTER_VERSION_MINOR, \



More information about the ffmpeg-devel mailing list