[FFmpeg-devel] [PATCH] qsvenc: write a53 caption data to SEI

Will Kelleher wkelleher at gogoair.com
Wed Nov 11 22:53:46 CET 2015


On 11/07, Ivan Uskov wrote:
> Although   the  code  looks  ok  by itself, I believe it is bad idea to place
> H.264-specific   code    to   the   function   which  is  common    for   all
> encoders.  I believe H.264-specific user data insertion should  locates  into
> the qsvenc_h264.c 
> I.e. there is necessary some kind of 'SetEncodeCtrl' callback which points to
> function into the encoder-specific module.
> I believe if you will define a callback pointer QSVEncContext::SetEncodeCtrlCB,
> setup  it  into  qsv_enc_init()  of the qsvenc_h264.c and call if it non-zero
> from   ff_qsv_encode()  we  will  have  good base to extend add user data for
> MPEG2 later. Else we will get very bulky and ugly ff_qsv_encode().
> I can release something but I hope you will able to catch my idea.
> Please let me know if something is unclear.

Hi Ivan,

Here is an updated patch.  Let me know if this is what you were thinking.

will

-------------- next part --------------
>From bbe29adf8eedf7944241c7e5e2704a2c8163faf5 Mon Sep 17 00:00:00 2001
From: Will Kelleher <wkelleher at gogoair.com>
Date: Tue, 27 Oct 2015 12:08:45 -0500
Subject: [PATCH] qsvenc: write a53 caption data to SEI

---
 libavcodec/qsv_internal.h |  3 +++
 libavcodec/qsvenc.c       | 39 ++++++++++++++++++++++++---
 libavcodec/qsvenc.h       |  5 ++++
 libavcodec/qsvenc_h264.c  | 67 ++++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 109 insertions(+), 5 deletions(-)

diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index b9ad199..c235e07 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -45,6 +45,8 @@
 
 #define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
 
+#define QSV_MAX_ENC_PAYLOAD 2       // # of mfxEncodeCtrl payloads supported
+
 #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
     (MFX_VERSION_MAJOR > (MAJOR) ||         \
      MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
@@ -52,6 +54,7 @@
 typedef struct QSVFrame {
     AVFrame *frame;
     mfxFrameSurface1 *surface;
+    mfxEncodeCtrl enc_ctrl;
 
     mfxFrameSurface1 surface_internal;
 
diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index df1f777..73789b5 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -30,6 +30,7 @@
 #include "libavutil/log.h"
 #include "libavutil/time.h"
 #include "libavutil/imgutils.h"
+#include "libavcodec/bytestream.h"
 
 #include "avcodec.h"
 #include "internal.h"
@@ -370,12 +371,26 @@ int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
     return 0;
 }
 
+static void free_encoder_ctrl_payloads(mfxEncodeCtrl* enc_ctrl)
+{
+    if (enc_ctrl) {
+        int i;
+        for (i = 0; i < enc_ctrl->NumPayload && i < QSV_MAX_ENC_PAYLOAD; i++) {
+            mfxPayload* pay = enc_ctrl->Payload[i];
+            av_free(enc_ctrl->Payload[i]->Data);
+            av_free(pay);
+        }
+        enc_ctrl->NumPayload = 0;
+    }
+}
+
 static void clear_unused_frames(QSVEncContext *q)
 {
     QSVFrame *cur = q->work_frames;
     while (cur) {
         if (cur->surface && !cur->surface->Data.Locked) {
             cur->surface = NULL;
+            free_encoder_ctrl_payloads(&cur->enc_ctrl);
             av_frame_unref(cur->frame);
         }
         cur = cur->next;
@@ -408,6 +423,11 @@ static int get_free_frame(QSVEncContext *q, QSVFrame **f)
         av_freep(&frame);
         return AVERROR(ENOMEM);
     }
+    frame->enc_ctrl.Payload = av_mallocz(sizeof(mfxPayload*) * QSV_MAX_ENC_PAYLOAD);
+    if (!frame->enc_ctrl.Payload) {
+        av_freep(&frame);
+        return AVERROR(ENOMEM);
+    }
     *last = frame;
 
     *f = frame;
@@ -416,7 +436,7 @@ static int get_free_frame(QSVEncContext *q, QSVFrame **f)
 }
 
 static int submit_frame(QSVEncContext *q, const AVFrame *frame,
-                        mfxFrameSurface1 **surface)
+                        QSVFrame **new_frame)
 {
     QSVFrame *qf;
     int ret;
@@ -477,7 +497,7 @@ static int submit_frame(QSVEncContext *q, const AVFrame *frame,
 
     qf->surface->Data.TimeStamp = av_rescale_q(frame->pts, q->avctx->time_base, (AVRational){1, 90000});
 
-    *surface = qf->surface;
+    *new_frame = qf;
 
     return 0;
 }
@@ -502,15 +522,21 @@ int ff_qsv_encode(AVCodecContext *avctx, QSVEncContext *q,
 
     mfxFrameSurface1 *surf = NULL;
     mfxSyncPoint sync      = NULL;
+    QSVFrame *qsv_frame = NULL;
+    mfxEncodeCtrl* enc_ctrl = NULL;
     int ret;
 
     if (frame) {
-        ret = submit_frame(q, frame, &surf);
+        ret = submit_frame(q, frame, &qsv_frame);
         if (ret < 0) {
             av_log(avctx, AV_LOG_ERROR, "Error submitting the frame for encoding.\n");
             return ret;
         }
     }
+    if (qsv_frame) {
+        surf = qsv_frame->surface;
+        enc_ctrl = &qsv_frame->enc_ctrl;
+    }
 
     ret = av_new_packet(&new_pkt, q->packet_size);
     if (ret < 0) {
@@ -526,8 +552,12 @@ int ff_qsv_encode(AVCodecContext *avctx, QSVEncContext *q,
     bs->Data      = new_pkt.data;
     bs->MaxLength = new_pkt.size;
 
+    if (q->set_encode_ctrl_cb) {
+        q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
+    }
+
     do {
-        ret = MFXVideoENCODE_EncodeFrameAsync(q->session, NULL, surf, bs, &sync);
+        ret = MFXVideoENCODE_EncodeFrameAsync(q->session, enc_ctrl, surf, bs, &sync);
         if (ret == MFX_WRN_DEVICE_BUSY) {
             av_usleep(500);
             continue;
@@ -627,6 +657,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q)
     while (cur) {
         q->work_frames = cur->next;
         av_frame_free(&cur->frame);
+        av_free(cur->enc_ctrl.Payload);
         av_freep(&cur);
         cur = q->work_frames;
     }
diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
index 3dd7afe..60701e4 100644
--- a/libavcodec/qsvenc.h
+++ b/libavcodec/qsvenc.h
@@ -34,6 +34,9 @@
 #include "avcodec.h"
 #include "qsv_internal.h"
 
+typedef int SetEncodeCtrlCB (AVCodecContext *avctx,
+                             const AVFrame *frame, mfxEncodeCtrl* enc_ctrl);
+
 typedef struct QSVEncContext {
     AVCodecContext *avctx;
 
@@ -77,7 +80,9 @@ typedef struct QSVEncContext {
     int look_ahead_depth;
     int look_ahead_downsampling;
 
+    int a53_cc;
     char *load_plugins;
+    SetEncodeCtrlCB *set_encode_ctrl_cb;
 } QSVEncContext;
 
 int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q);
diff --git a/libavcodec/qsvenc_h264.c b/libavcodec/qsvenc_h264.c
index 0e5a26c..d9b24c1 100644
--- a/libavcodec/qsvenc_h264.c
+++ b/libavcodec/qsvenc_h264.c
@@ -40,10 +40,75 @@ typedef struct QSVH264EncContext {
     QSVEncContext qsv;
 } QSVH264EncContext;
 
+static int qsv_h264_set_encode_ctrl(AVCodecContext *avctx,
+                                    const AVFrame *frame, mfxEncodeCtrl* enc_ctrl)
+{
+    AVFrameSideData *side_data = NULL;
+    QSVH264EncContext *qh264 = avctx->priv_data;
+    QSVEncContext *q = &qh264->qsv;
+
+    if (q->a53_cc && frame) {
+        side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
+        if (side_data) {
+
+            int sei_payload_size = 0;
+            mfxU8* sei_data = NULL;
+            mfxPayload* payload = NULL;
+
+            sei_payload_size = side_data->size + 13;
+
+            sei_data = av_mallocz(sei_payload_size);
+            if (!sei_data) {
+                av_log(avctx, AV_LOG_ERROR, "No memory for CC, skipping...\n");
+                return AVERROR(ENOMEM);
+            }
+
+            // SEI header
+            sei_data[0] = 4;
+            sei_data[1] = sei_payload_size - 2; // size of SEI data
+
+            // country code
+            sei_data[2] = 181;
+            sei_data[3] = 0;
+            sei_data[4] = 49;
+
+            // ATSC_identifier - using 'GA94' only
+            AV_WL32(sei_data + 5,
+                MKTAG('G', 'A', '9', '4'));
+            sei_data[9] = 3;
+            sei_data[10] =
+                ((side_data->size/3) & 0x1f) | 0xC0;
+
+            sei_data[11] = 0xFF; // reserved
+
+            memcpy(sei_data + 12, side_data->data, side_data->size);
+
+            sei_data[side_data->size+12] = 255;
+
+            payload = av_mallocz(sizeof(mfxPayload));
+            if (!payload) {
+                av_log(avctx, AV_LOG_ERROR, "No memory, skipping captions\n");
+                av_freep(&sei_data);
+                return AVERROR(ENOMEM);
+            }
+            payload->BufSize = side_data->size + 13;
+            payload->NumBit = payload->BufSize * 8;
+            payload->Type = 4;
+            payload->Data = sei_data;
+
+            enc_ctrl->NumExtParam = 0;
+            enc_ctrl->NumPayload = 1;
+            enc_ctrl->Payload[0] = payload;
+        }
+    }
+    return 0;
+}
+
 static av_cold int qsv_enc_init(AVCodecContext *avctx)
 {
     QSVH264EncContext *q = avctx->priv_data;
 
+    q->qsv.set_encode_ctrl_cb = qsv_h264_set_encode_ctrl;
     return ff_qsv_enc_init(avctx, &q->qsv);
 }
 
@@ -97,7 +162,7 @@ static const AVOption options[] = {
     { "slow",        NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_TARGETUSAGE_3  },            INT_MIN, INT_MAX, VE, "preset" },
     { "slower",      NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_TARGETUSAGE_2  },            INT_MIN, INT_MAX, VE, "preset" },
     { "veryslow",    NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_TARGETUSAGE_BEST_QUALITY  }, INT_MIN, INT_MAX, VE, "preset" },
-
+    {"a53cc",          "Use A53 Closed Captions (if available)",          OFFSET(qsv.a53_cc),        AV_OPT_TYPE_BOOL,   {.i64 = 0}, 0, 1, VE},
     { NULL },
 };
 
-- 
2.6.2



More information about the ffmpeg-devel mailing list