[FFmpeg-devel] [PATCH 1/2] libavdevice/decklink: Add support for EIA-708 output over SDI

Marton Balint cus at passwd.hu
Sun Aug 26 18:20:21 EEST 2018


On Wed, 22 Aug 2018, Devin Heitmueller wrote:

> Hook in libklvanc and use it for output of EIA-708 captions over
> SDI.  The bulk of this patch is just general support for ancillary
> data for the Decklink SDI module - the real work for construction
> of the EIA-708 CDP and VANC line construction is done by libklvanc.
>
> Libklvanc can be found at: https://github.com/stoth68000/libklvanc
>
> Updated to reflect feedback from Marton Balint <cus at passwd.hu>,
> Carl Eugen Hoyos <ceffmpeg at gmail.com>, Aaron Levinson
> <alevinsn_dev at levland.net>, and Moritz Barsnick <barsnick at gmx.net>

Thanks, I have a few minor (mostly code style) comments below:

>
> Signed-off-by: Devin Heitmueller <dheitmueller at ltnglobal.com>
> ---
> configure                       |   4 +
> libavcodec/v210enc.c            |   9 ++
> libavdevice/decklink_common.cpp |  16 +++-
> libavdevice/decklink_common.h   |  10 +++
> libavdevice/decklink_enc.cpp    | 179 ++++++++++++++++++++++++++++++++++++++--
> 5 files changed, 207 insertions(+), 11 deletions(-)
>
> diff --git a/configure b/configure
> index 9b5421d5a8..ecc86c606f 100755
> --- a/configure
> +++ b/configure
> @@ -239,6 +239,7 @@ External library support:
>   --enable-libiec61883     enable iec61883 via libiec61883 [no]
>   --enable-libilbc         enable iLBC de/encoding via libilbc [no]
>   --enable-libjack         enable JACK audio sound server [no]
> +  --enable-libklvanc       enable Kernel Labs VANC processing [no]
>   --enable-libkvazaar      enable HEVC encoding via libkvazaar [no]
>   --enable-liblensfun      enable lensfun lens correction [no]
>   --enable-libmodplug      enable ModPlug via libmodplug [no]
> @@ -1700,6 +1701,7 @@ EXTERNAL_LIBRARY_LIST="
>     libiec61883
>     libilbc
>     libjack
> +    libklvanc
>     libkvazaar
>     libmodplug
>     libmp3lame
> @@ -3210,6 +3212,7 @@ decklink_deps_any="libdl LoadLibrary"
> decklink_indev_deps="decklink threads"
> decklink_indev_extralibs="-lstdc++"
> decklink_outdev_deps="decklink threads"
> +decklink_outdev_suggest="libklvanc"
> decklink_outdev_extralibs="-lstdc++"
> libndi_newtek_indev_deps="libndi_newtek"
> libndi_newtek_indev_extralibs="-lndi"
> @@ -6034,6 +6037,7 @@ enabled libgsm            && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do
>                                    check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break;
>                                done || die "ERROR: libgsm not found"; }
> enabled libilbc           && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs
> +enabled libklvanc         && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc
> enabled libkvazaar        && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get
> enabled liblensfun        && require_pkg_config liblensfun lensfun lensfun.h lf_db_new
> # While it may appear that require is being used as a pkg-config
> diff --git a/libavcodec/v210enc.c b/libavcodec/v210enc.c
> index a6afbbfc41..b9dcf9a672 100644
> --- a/libavcodec/v210enc.c
> +++ b/libavcodec/v210enc.c
> @@ -123,6 +123,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
>     int aligned_width = ((avctx->width + 47) / 48) * 48;
>     int stride = aligned_width * 8 / 3;
>     int line_padding = stride - ((avctx->width * 8 + 11) / 12) * 4;
> +    AVFrameSideData *side_data;
>     int h, w, ret;
>     uint8_t *dst;
> 
> @@ -233,6 +234,14 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
>         }
>     }
> 
> +    side_data = av_frame_get_side_data(pic, AV_FRAME_DATA_A53_CC);
> +    if (side_data && side_data->size) {
> +        uint8_t *buf = av_packet_new_side_data(pkt, AV_PKT_DATA_A53_CC, side_data->size);
> +        if (!buf)
> +            return AVERROR(ENOMEM);
> +        memcpy(buf, side_data->data, side_data->size);
> +    }
> +
>     pkt->flags |= AV_PKT_FLAG_KEY;
>     *got_packet = 1;
>     return 0;

Strictly speaking the v210enc part should be separate patch.

> diff --git a/libavdevice/decklink_common.cpp b/libavdevice/decklink_common.cpp
> index aab9d85b94..503417bb35 100644
> --- a/libavdevice/decklink_common.cpp
> +++ b/libavdevice/decklink_common.cpp
> @@ -239,10 +239,18 @@ int ff_decklink_set_format(AVFormatContext *avctx,
>                                            &support, NULL) != S_OK)
>             return -1;
>     } else {
> -        if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, bmdFormat8BitYUV,
> -                                           bmdVideoOutputFlagDefault,
> -                                           &support, NULL) != S_OK)
> -        return -1;
> +        if (!ctx->supports_vanc || ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, ctx->raw_format,
> +                                                                  bmdVideoOutputVANC,
> +                                                                  &support, NULL) != S_OK) {
> +            /* Try without VANC enabled */
> +            if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, ctx->raw_format,
> +                                               bmdVideoOutputFlagDefault,
> +                                               &support, NULL) != S_OK) {
> +                return -1;
> +            }
> +            ctx->supports_vanc = 0;
> +        }
> +
>     }
>     if (support == bmdDisplayModeSupported)
>         return 0;
> diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h
> index 96b001c2d8..128144f50d 100644
> --- a/libavdevice/decklink_common.h
> +++ b/libavdevice/decklink_common.h
> @@ -27,6 +27,9 @@
> 
> #include "libavutil/thread.h"
> #include "decklink_common_c.h"
> +#if CONFIG_LIBKLVANC
> +#include "libklvanc/vanc.h"
> +#endif
> 
> #ifdef _WIN32
> #define DECKLINK_BOOL BOOL
> @@ -97,6 +100,7 @@ struct decklink_ctx {
>     int bmd_width;
>     int bmd_height;
>     int bmd_field_dominance;
> +    int supports_vanc;
>
>     /* Capture buffer queue */
>     AVPacketQueue queue;
> @@ -114,6 +118,7 @@ struct decklink_ctx {
>     AVStream *audio_st;
>     AVStream *video_st;
>     AVStream *teletext_st;
> +    uint16_t cdp_sequence_num;
>
>     /* Options */
>     int list_devices;
> @@ -124,6 +129,7 @@ struct decklink_ctx {
>     DecklinkPtsSource audio_pts_source;
>     DecklinkPtsSource video_pts_source;
>     int draw_bars;
> +    BMDPixelFormat raw_format;
>
>     int frames_preroll;
>     int frames_buffer;
> @@ -133,6 +139,10 @@ struct decklink_ctx {
>     int frames_buffer_available_spots;
>     int autodetect;
> 
> +#if CONFIG_LIBKLVANC
> +    struct klvanc_context_s *vanc_ctx;
> +#endif
> +
>     int channels;
>     int audio_depth;
> };
> diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp
> index 28ab928cd5..de7758cb91 100644
> --- a/libavdevice/decklink_enc.cpp
> +++ b/libavdevice/decklink_enc.cpp
> @@ -38,17 +38,25 @@ extern "C" {
> 
> #include "decklink_common.h"
> #include "decklink_enc.h"
> -
> +#if CONFIG_LIBKLVANC
> +#include "libklvanc/vanc.h"
> +#include "libklvanc/vanc-lines.h"
> +#include "libklvanc/pixels.h"
> +#endif
> 
> /* DeckLink callback class declaration */
> class decklink_frame : public IDeckLinkVideoFrame
> {
> public:
>     decklink_frame(struct decklink_ctx *ctx, AVFrame *avframe, AVCodecID codec_id, int height, int width) :
> -        _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _height(height), _width(width),  _refs(1) { }
> +        _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width),  _refs(1) { }
>     decklink_frame(struct decklink_ctx *ctx, AVPacket *avpacket, AVCodecID codec_id, int height, int width) :
> -        _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _height(height), _width(width), _refs(1) { }
> -
> +        _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { }
> +    virtual ~decklink_frame()
> +    {
> +        if (_ancillary)
> +            _ancillary->Release();
> +    };

Does not matter much, but I prefer if you put this in the Release() 
method instead where we are freeing other attributes...

>     virtual long           STDMETHODCALLTYPE GetWidth      (void)          { return _width; }
>     virtual long           STDMETHODCALLTYPE GetHeight     (void)          { return _height; }
>     virtual long           STDMETHODCALLTYPE GetRowBytes   (void)
> @@ -87,8 +95,24 @@ public:
>     }
>
>     virtual HRESULT STDMETHODCALLTYPE GetTimecode     (BMDTimecodeFormat format, IDeckLinkTimecode **timecode) { return S_FALSE; }
> -    virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary)               { return S_FALSE; }
> -
> +    virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary)
> +    {
> +        *ancillary = _ancillary;
> +        if (_ancillary) {
> +            _ancillary->AddRef();
> +            return S_OK;
> +        } else {
> +            return S_FALSE;
> +        }
> +    }
> +    virtual HRESULT STDMETHODCALLTYPE SetAncillaryData(IDeckLinkVideoFrameAncillary *ancillary)
> +    {
> +        if (_ancillary)
> +            _ancillary->Release();
> +        _ancillary = ancillary;
> +        _ancillary->AddRef();
> +        return S_OK;
> +    }
>     virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) { return E_NOINTERFACE; }
>     virtual ULONG   STDMETHODCALLTYPE AddRef(void)                            { return ++_refs; }
>     virtual ULONG   STDMETHODCALLTYPE Release(void)
> @@ -106,6 +130,7 @@ public:
>     AVFrame *_avframe;
>     AVPacket *_avpacket;
>     AVCodecID _codec_id;
> +    IDeckLinkVideoFrameAncillary *_ancillary;
>     int _height;
>     int _width;
> 
> @@ -156,10 +181,13 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st)
>                    " Only AV_PIX_FMT_UYVY422 is supported.\n");
>             return -1;
>         }
> +        ctx->raw_format = bmdFormat8BitYUV;
>     } else if (c->codec_id != AV_CODEC_ID_V210) {
>         av_log(avctx, AV_LOG_ERROR, "Unsupported codec type!"
>                " Only V210 and wrapped frame with AV_PIX_FMT_UYVY422 are supported.\n");
>         return -1;
> +    } else {
> +        ctx->raw_format = bmdFormat10BitYUV;
>     }
>
>     if (ff_decklink_set_configs(avctx, DIRECTION_OUT) < 0) {
> @@ -173,7 +201,7 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st)
>         return -1;
>     }
>     if (ctx->dlo->EnableVideoOutput(ctx->bmd_mode,
> -                                    bmdVideoOutputFlagDefault) != S_OK) {
> +                                    ctx->supports_vanc ? bmdVideoOutputVANC : bmdVideoOutputFlagDefault) != S_OK) {
>         av_log(avctx, AV_LOG_ERROR, "Could not enable video output!\n");
>         return -1;
>     }
> @@ -264,11 +292,132 @@ av_cold int ff_decklink_write_trailer(AVFormatContext *avctx)
>     pthread_mutex_destroy(&ctx->mutex);
>     pthread_cond_destroy(&ctx->cond);
> 
> +#if CONFIG_LIBKLVANC
> +    klvanc_context_destroy(ctx->vanc_ctx);
> +#endif
> +
>     av_freep(&cctx->ctx);
>
>     return 0;
> }
> 
> +#if CONFIG_LIBKLVANC
> +static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *ctx,
> +                                   AVPacket *pkt, decklink_frame *frame)
> +{
> +    struct klvanc_line_set_s vanc_lines = { 0 };
> +    int ret, size, i;
> +
> +    if (!ctx->supports_vanc)
> +        return 0;
> +
> +    const uint8_t *data = av_packet_get_side_data(pkt, AV_PKT_DATA_A53_CC, &size);
> +    if (data) {
> +        struct klvanc_packet_eia_708b_s *pkt;
> +        uint16_t *cdp;
> +        uint16_t len;
> +        uint8_t cc_count = size / 3;
> +
> +        ret = klvanc_create_eia708_cdp(&pkt);
> +        if (ret)
> +            return AVERROR(ENOMEM);
> +
> +        ret = klvanc_set_framerate_EIA_708B(pkt, ctx->bmd_tb_num, ctx->bmd_tb_den);
> +        if (ret) {
> +            av_log(avctx, AV_LOG_ERROR, "Invalid framerate specified: %lld/%lld\n",
> +                   ctx->bmd_tb_num, ctx->bmd_tb_den);
> +            klvanc_destroy_eia708_cdp(pkt);
> +            return AVERROR(EINVAL);
> +        }
> +
> +        if (cc_count > KLVANC_MAX_CC_COUNT) {
> +            av_log(avctx, AV_LOG_ERROR, "Illegal cc_count received: %d\n", cc_count);
> +            cc_count = KLVANC_MAX_CC_COUNT;
> +        }
> +
> +        /* CC data */
> +        pkt->header.ccdata_present = 1;
> +        pkt->header.caption_service_active = 1;
> +        pkt->ccdata.cc_count = cc_count;
> +        for (i = 0; i < cc_count; i++) {
> +            if (data [3*i] & 0x04)
> +                pkt->ccdata.cc[i].cc_valid = 1;
> +            pkt->ccdata.cc[i].cc_type = data[3*i] & 0x03;
> +            pkt->ccdata.cc[i].cc_data[0] = data[3*i+1];
> +            pkt->ccdata.cc[i].cc_data[1] = data[3*i+2];
> +        }
> +
> +        klvanc_finalize_EIA_708B(pkt, ctx->cdp_sequence_num++);
> +        ret = klvanc_convert_EIA_708B_to_words(pkt, &cdp, &len);
> +        klvanc_destroy_eia708_cdp(pkt);
> +        if (ret != 0) {
> +            av_log(avctx, AV_LOG_ERROR, "Failed converting 708 packet to words\n");
> +            return AVERROR(ENOMEM);
> +        }
> +
> +        ret = klvanc_line_insert(ctx->vanc_ctx, &vanc_lines, cdp, len, 11, 0);
> +        free(cdp);
> +        if (ret != 0) {
> +            av_log(avctx, AV_LOG_ERROR, "VANC line insertion failed\n");
> +            return AVERROR(ENOMEM);
> +        }
> +    }
> +
> +    IDeckLinkVideoFrameAncillary *vanc;
> +    int result = ctx->dlo->CreateAncillaryData(bmdFormat10BitYUV, &vanc);
> +    if (result != S_OK) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to create vanc\n");

> +        for (i = 0; i < vanc_lines.num_lines; i++)
> +            klvanc_line_free(vanc_lines.lines[i]);

This line freeing cleanup code should go into the end of the function and 
you should use goto there to do it on error. The next patch can also use 
it, because there are tons of this.

This way you can also get rid of the klvanc_line_free() functions in the 
for() loop below.

> +        return -1;
> +    }
> +
> +    /* Now that we've got all the VANC lines in a nice orderly manner, generate the
> +       final VANC sections for the Decklink output */
> +    for (i = 0; i < vanc_lines.num_lines; i++) {
> +        struct klvanc_line_s *line = vanc_lines.lines[i];
> +        int real_line;
> +        void *buf;
> +
> +        if (!line)
> +            break;
> +
> +        real_line = line->line_number;
> +#if 0
> +        /* FIXME: include hack for certain Decklink cards which mis-represent
> +           line numbers for pSF frames */
> +        if (decklink_sys->b_psf_interlaced)
> +            real_line = Calculate1080psfVancLine(line->line_number);
> +#endif

Remove the foreign code, keep the fixme.

> +        result = vanc->GetBufferForVerticalBlankingLine(real_line, &buf);
> +        if (result != S_OK) {
> +            av_log(avctx, AV_LOG_ERROR, "Failed to get VANC line %d: %d", real_line, result);
> +            klvanc_line_free(line);
> +            continue;
> +        }
> +
> +        /* Generate the full line taking into account all VANC packets on that line */
> +        result = klvanc_generate_vanc_line_v210(ctx->vanc_ctx, line, (uint8_t *) buf,
> +                                                ctx->bmd_width);
> +        if (result) {
> +            av_log(avctx, AV_LOG_ERROR, "Failed to generate VANC line\n");
> +            klvanc_line_free(line);
> +            continue;
> +        }
> +
> +        klvanc_line_free(line);
> +    }
> +
> +    result = frame->SetAncillaryData(vanc);
> +    vanc->Release();
> +    if (result != S_OK) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set vanc: %d", result);
> +        return AVERROR(EIO);
> +    }
> +    return 0;
> +}
> +#endif
> +
> static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt)
> {
>     struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
> @@ -279,6 +428,9 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt)
>     decklink_frame *frame;
>     buffercount_type buffered;
>     HRESULT hr;
> +#if CONFIG_LIBKLVANC
> +    int ret;
> +#endif
>
>     if (st->codecpar->codec_id == AV_CODEC_ID_WRAPPED_AVFRAME) {
>         if (tmp->format != AV_PIX_FMT_UYVY422 ||
> @@ -303,6 +455,12 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt)
>         }
>
>         frame = new decklink_frame(ctx, avpacket, st->codecpar->codec_id, ctx->bmd_height, ctx->bmd_width);
> +
> +#if CONFIG_LIBKLVANC
> +        ret = decklink_construct_vanc(avctx, ctx, pkt, frame);
> +        if (ret)

Or maybe you can simply write

if (decklink_construct_vanc(avctx, ctx, pkt, frame))
     av_log error

to avoid the extra ret variable declaration which you are not using 
anyway.

> +            av_log(avctx, AV_LOG_ERROR, "Failed to construct VANC\n");
> +#endif
>     }
>
>     if (!frame) {
> @@ -393,6 +551,13 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx)
>     ctx->list_formats = cctx->list_formats;
>     ctx->preroll      = cctx->preroll;
>     cctx->ctx = ctx;
> +#if CONFIG_LIBKLVANC
> +    if (klvanc_context_create(&ctx->vanc_ctx) < 0) {
> +        av_log(avctx, AV_LOG_ERROR, "Cannot create VANC library context\n");
> +        return AVERROR(ENOMEM);
> +    }
> +    ctx->supports_vanc = 1;
> +#endif
>
>     /* List available devices and exit. */
>     if (ctx->list_devices) {

Thanks,
Marton


More information about the ffmpeg-devel mailing list