[FFmpeg-devel] [PATCH v5 6/6] avcodec/qsvdec: Implement SEI parsing for QSV decoders
softworkz
ffmpegagent at gmail.com
Fri Jul 1 23:48:54 EEST 2022
From: softworkz <softworkz at hotmail.com>
Signed-off-by: softworkz <softworkz at hotmail.com>
---
libavcodec/Makefile | 6 +-
libavcodec/hevcdsp.c | 4 +
libavcodec/qsvdec.c | 234 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 243 insertions(+), 1 deletion(-)
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 3b8f7b5e01..7b98391745 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -144,7 +144,11 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o
OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
OBJS-$(CONFIG_QSV) += qsv.o
-OBJS-$(CONFIG_QSVDEC) += qsvdec.o
+OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_slice.o h264_cabac.o h264_cavlc.o \
+ h264_direct.o h264_mb.o h264_picture.o h264_loopfilter.o \
+ h264dec.o h264_refs.o cabac.o hevcdec.o hevc_refs.o \
+ hevc_filter.o hevc_cabac.o hevc_mvs.o hevcpred.o hevcdsp.o \
+ h274.o dovi_rpu.o mpeg12dec.o
OBJS-$(CONFIG_QSVENC) += qsvenc.o
OBJS-$(CONFIG_RANGECODER) += rangecoder.o
OBJS-$(CONFIG_RDFT) += rdft.o
diff --git a/libavcodec/hevcdsp.c b/libavcodec/hevcdsp.c
index 2ca551df1d..c7a436d30f 100644
--- a/libavcodec/hevcdsp.c
+++ b/libavcodec/hevcdsp.c
@@ -22,6 +22,8 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#include "config_components.h"
+
#include "hevcdsp.h"
static const int8_t transform[32][32] = {
@@ -257,6 +259,7 @@ int i = 0;
break;
}
+#if CONFIG_HEVC_DECODER
#if ARCH_AARCH64
ff_hevc_dsp_init_aarch64(hevcdsp, bit_depth);
#elif ARCH_ARM
@@ -270,4 +273,5 @@ int i = 0;
#elif ARCH_LOONGARCH
ff_hevc_dsp_init_loongarch(hevcdsp, bit_depth);
#endif
+#endif
}
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 5fc5bed4c8..e854f363ec 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -49,6 +49,12 @@
#include "hwconfig.h"
#include "qsv.h"
#include "qsv_internal.h"
+#include "h264dec.h"
+#include "h264_sei.h"
+#include "hevcdec.h"
+#include "hevc_ps.h"
+#include "hevc_sei.h"
+#include "mpeg12.h"
static const AVRational mfx_tb = { 1, 90000 };
@@ -60,6 +66,8 @@ static const AVRational mfx_tb = { 1, 90000 };
AV_NOPTS_VALUE : pts_tb.num ? \
av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
+#define PAYLOAD_BUFFER_SIZE 65535
+
typedef struct QSVAsyncFrame {
mfxSyncPoint *sync;
QSVFrame *frame;
@@ -101,6 +109,9 @@ typedef struct QSVContext {
mfxExtBuffer **ext_buffers;
int nb_ext_buffers;
+
+ mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
+ Mpeg1Context mpeg_ctx;
} QSVContext;
static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -599,6 +610,210 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
return 0;
}
#endif
+static int find_start_offset(mfxU8 data[4])
+{
+ if (data[0] == 0 && data[1] == 0 && data[2] == 1)
+ return 3;
+
+ if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
+ return 4;
+
+ return 0;
+}
+
+static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+ H264SEIContext sei = { 0 };
+ GetBitContext gb = { 0 };
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
+ mfxU64 ts;
+ int ret;
+
+ while (1) {
+ int start;
+ memset(payload.Data, 0, payload.BufSize);
+
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ start = find_start_offset(payload.Data);
+
+ switch (payload.Type) {
+ case SEI_TYPE_BUFFERING_PERIOD:
+ case SEI_TYPE_PIC_TIMING:
+ continue;
+ }
+
+ if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else {
+ ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
+ }
+ }
+
+ if (out)
+ return ff_h264_export_frame_props(avctx, &sei, NULL, out);
+
+ return 0;
+}
+
+static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out)
+{
+ HEVCSEI sei = { 0 };
+ HEVCParamSets ps = { 0 };
+ GetBitContext gb = { 0 };
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
+ mfxFrameSurface1 *surface = &out->surface;
+ mfxU64 ts;
+ int ret, has_logged = 0;
+
+ while (1) {
+ int start;
+ memset(payload.Data, 0, payload.BufSize);
+
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ if (!has_logged) {
+ has_logged = 1;
+ av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n");
+ av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
+ }
+
+ if (ts != surface->Data.TimeStamp) {
+ av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
+ }
+
+ start = find_start_offset(payload.Data);
+
+ av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start);
+
+ switch (payload.Type) {
+ case SEI_TYPE_BUFFERING_PERIOD:
+ case SEI_TYPE_PIC_TIMING:
+ continue;
+ case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
+ // There seems to be a bug in MSDK
+ payload.NumBit -= 8;
+
+ break;
+ case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
+ // There seems to be a bug in MSDK
+ payload.NumBit = 48;
+
+ break;
+ case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
+ // There seems to be a bug in MSDK
+ if (payload.NumBit == 552)
+ payload.NumBit = 528;
+ break;
+ }
+
+ if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else {
+ ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
+ }
+ }
+
+ if (has_logged) {
+ av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
+ }
+
+ if (out && out->frame)
+ return ff_hevc_set_side_data(avctx, &sei, NULL, out->frame);
+
+ return 0;
+}
+
+static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+ Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
+ mfxU64 ts;
+ int ret;
+
+ while (1) {
+ int start;
+
+ memset(payload.Data, 0, payload.BufSize);
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ start = find_start_offset(payload.Data);
+
+ start++;
+
+ ff_mpeg_decode_user_data(avctx, mpeg_ctx, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
+
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start]));
+ }
+
+ if (!out)
+ return 0;
+
+ if (mpeg_ctx->a53_buf_ref) {
+
+ AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
+ if (!sd)
+ av_buffer_unref(&mpeg_ctx->a53_buf_ref);
+ mpeg_ctx->a53_buf_ref = NULL;
+ }
+
+ if (mpeg_ctx->has_stereo3d) {
+ AVStereo3D *stereo = av_stereo3d_create_side_data(out);
+ if (!stereo)
+ return AVERROR(ENOMEM);
+
+ *stereo = mpeg_ctx->stereo3d;
+ mpeg_ctx->has_stereo3d = 0;
+ }
+
+ if (mpeg_ctx->has_afd) {
+ AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD, 1);
+ if (!sd)
+ return AVERROR(ENOMEM);
+
+ *sd->data = mpeg_ctx->afd;
+ mpeg_ctx->has_afd = 0;
+ }
+
+ return 0;
+}
static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
AVFrame *frame, int *got_frame,
@@ -636,6 +851,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
insurf, &outsurf, sync);
if (ret == MFX_WRN_DEVICE_BUSY)
av_usleep(500);
+ else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
+ parse_sei_mpeg12(avctx, q, NULL);
} while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
@@ -677,6 +894,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
return AVERROR_BUG;
}
+ switch (avctx->codec_id) {
+ case AV_CODEC_ID_MPEG2VIDEO:
+ ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
+ break;
+ case AV_CODEC_ID_H264:
+ ret = parse_sei_h264(avctx, q, out_frame->frame);
+ break;
+ case AV_CODEC_ID_HEVC:
+ ret = parse_sei_hevc(avctx, q, out_frame);
+ break;
+ default:
+ ret = 0;
+ }
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
+
out_frame->queued += 1;
aframe = (QSVAsyncFrame){ sync, out_frame };
--
ffmpeg-codebot
More information about the ffmpeg-devel
mailing list