[FFmpeg-devel] [PATCH] Add support for RockChip Media Process Platform
LongChair .
LongChair at hotmail.com
Tue Sep 5 21:55:39 EEST 2017
Le 05/09/2017 à 19:19, Mark Thompson a écrit :
> On 05/09/17 12:44, LongChair . wrote:
>> From: LongChair <LongChair at hotmail.com>
>>
>> This adds hardware decoding for h264 / HEVC / VP8 using MPP Rockchip API.
>> Will return frames holding an AVDRMFrameDescriptor struct in buf[0] that allows drm / dmabuf usage.
>> Was tested on RK3288 (TinkerBoard) and RK3328.
>> ---
>> Changelog | 1 +
>> configure | 15 +-
>> libavcodec/Makefile | 3 +
>> libavcodec/allcodecs.c | 3 +
>> libavcodec/rkmppdec.c | 567 +++++++++++++++++++++++++++++++++++++++++++++++++
>> 5 files changed, 588 insertions(+), 1 deletion(-)
>> create mode 100644 libavcodec/rkmppdec.c
>>
>> diff --git a/Changelog b/Changelog
>> index 1dfb8b5..622e5d0 100644
>> --- a/Changelog
>> +++ b/Changelog
>> @@ -35,6 +35,7 @@ version <next>:
>> - pseudocolor video filter
>> - raw G.726 muxer and demuxer, left- and right-justified
>> - NewTek NDI input/output device
>> +- Addition of Rockchip MPP hardware decoding
> Use the same style as other entries, so just state what has been added; i.e. "- Rockchip MPP hardware decoder".
Ok no problem.
>> version 3.3:
>> - CrystalHD decoder moved to new decode API
>> diff --git a/configure b/configure
>> index ebd561d..1df5763 100755
>> --- a/configure
>> +++ b/configure
>> @@ -307,6 +307,7 @@ External library support:
>> --disable-nvenc disable Nvidia video encoding code [autodetect]
>> --enable-omx enable OpenMAX IL code [no]
>> --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no]
>> + --enable-rkmpp enable Rockchip Media Process Platform code [no]
>> --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
>> --disable-vda disable Apple Video Decode Acceleration code [autodetect]
>> --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
>> @@ -1520,6 +1521,7 @@ EXTERNAL_LIBRARY_VERSION3_LIST="
>> libopencore_amrnb
>> libopencore_amrwb
>> libvo_amrwbenc
>> + rkmpp
>> "
>>
>> EXTERNAL_LIBRARY_GPLV3_LIST="
>> @@ -2757,6 +2759,8 @@ h264_qsv_decoder_deps="libmfx"
>> h264_qsv_decoder_select="h264_mp4toannexb_bsf h264_parser qsvdec h264_qsv_hwaccel"
>> h264_qsv_encoder_deps="libmfx"
>> h264_qsv_encoder_select="qsvenc"
>> +h264_rkmpp_decoder_deps="rkmpp"
>> +h264_rkmpp_decoder_select="h264_mp4toannexb_bsf"
>> h264_vaapi_encoder_deps="VAEncPictureParameterBufferH264"
>> h264_vaapi_encoder_select="vaapi_encode golomb"
>> h264_vda_decoder_deps="vda"
>> @@ -2772,6 +2776,8 @@ hevc_qsv_decoder_deps="libmfx"
>> hevc_qsv_decoder_select="hevc_mp4toannexb_bsf hevc_parser qsvdec hevc_qsv_hwaccel"
>> hevc_qsv_encoder_deps="libmfx"
>> hevc_qsv_encoder_select="hevcparse qsvenc"
>> +hevc_rkmpp_decoder_deps="rkmpp"
>> +hevc_rkmpp_decoder_select="hevc_mp4toannexb_bsf"
>> hevc_vaapi_encoder_deps="VAEncPictureParameterBufferHEVC"
>> hevc_vaapi_encoder_select="vaapi_encode golomb"
>> mjpeg_cuvid_decoder_deps="cuda cuvid"
>> @@ -2811,6 +2817,8 @@ vp8_cuvid_decoder_deps="cuda cuvid"
>> vp8_mediacodec_decoder_deps="mediacodec"
>> vp8_qsv_decoder_deps="libmfx"
>> vp8_qsv_decoder_select="qsvdec vp8_qsv_hwaccel vp8_parser"
>> +vp8_rkmpp_decoder_deps="rkmpp"
>> +vp8_rkmpp_decoder_select="rkmpp"
> I think this line isn't wanted?
Yeah, missed that one :)
>
>> vp8_vaapi_encoder_deps="VAEncPictureParameterBufferVP8"
>> vp8_vaapi_encoder_select="vaapi_encode"
>> vp9_cuvid_decoder_deps="cuda cuvid"
>> @@ -6008,7 +6016,12 @@ enabled openssl && { use_pkg_config openssl openssl/ssl.h OPENSSL_init
>> check_lib openssl openssl/ssl.h SSL_library_init -lssl -lcrypto -lws2_32 -lgdi32 ||
>> die "ERROR: openssl not found"; }
>> enabled qtkit_indev && { check_header_objcc QTKit/QTKit.h || disable qtkit_indev; }
>> -
>> +enabled rkmpp && { { require_pkg_config rockchip_mpp rockchip/rk_mpi.h mpp_create ||
>> + die "ERROR : Rockchip MPP was not found."; } &&
>> + { check_func_headers rockchip/rk_mpi_cmd.h "MPP_DEC_GET_FREE_PACKET_SLOT_COUNT" ||
>> + die "ERROR: Rockchip MPP is outdated, please get a more recent one."; } &&
>> + { enabled libdrm || die "ERROR: rkmpp requires --enable-libdrm"; }
>> + }
> Keep the blank line after the list of normal checks and before the more complex cases below.
i Hope something like this makes it more readable :
enabled rkmpp && {
{ require_pkg_config rockchip_mpp rockchip/rk_mpi.h mpp_create ||
die "ERROR : Rockchip MPP was not found."; }
&& { check_func_headers rockchip/rk_mpi_cmd.h "MPP_DEC_GET_FREE_PACKET_SLOT_COUNT" ||
die "ERROR: Rockchip MPP is outdated, please get a more recent one."; }
&& { enabled libdrm ||
die "ERROR: rkmpp requires --enable-libdrm"; }
}
> Also needs a rebase - qtkit_indev was removed.
Will rebase
>
>> if enabled gcrypt; then
>> GCRYPT_CONFIG="${cross_prefix}libgcrypt-config"
>> if "${GCRYPT_CONFIG}" --version > /dev/null 2>&1; then
>> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
>> index 982d7f5..b75912f 100644
>> --- a/libavcodec/Makefile
>> +++ b/libavcodec/Makefile
>> @@ -336,6 +336,7 @@ OBJS-$(CONFIG_H264_VDA_DECODER) += vda_h264_dec.o
>> OBJS-$(CONFIG_H264_OMX_ENCODER) += omx.o
>> OBJS-$(CONFIG_H264_QSV_DECODER) += qsvdec_h2645.o
>> OBJS-$(CONFIG_H264_QSV_ENCODER) += qsvenc_h264.o
>> +OBJS-$(CONFIG_H264_RKMPP_DECODER) += rkmppdec.o
>> OBJS-$(CONFIG_H264_VAAPI_ENCODER) += vaapi_encode_h264.o vaapi_encode_h26x.o
>> OBJS-$(CONFIG_H264_VIDEOTOOLBOX_ENCODER) += videotoolboxenc.o
>> OBJS-$(CONFIG_HAP_DECODER) += hapdec.o hap.o
>> @@ -350,6 +351,7 @@ OBJS-$(CONFIG_NVENC_HEVC_ENCODER) += nvenc_hevc.o
>> OBJS-$(CONFIG_HEVC_QSV_DECODER) += qsvdec_h2645.o
>> OBJS-$(CONFIG_HEVC_QSV_ENCODER) += qsvenc_hevc.o hevc_ps_enc.o \
>> hevc_data.o
>> +OBJS-$(CONFIG_HEVC_RKMPP_DECODER) += rkmppdec.o
>> OBJS-$(CONFIG_HEVC_VAAPI_ENCODER) += vaapi_encode_h265.o vaapi_encode_h26x.o
>> OBJS-$(CONFIG_HNM4_VIDEO_DECODER) += hnm4video.o
>> OBJS-$(CONFIG_HQ_HQA_DECODER) += hq_hqa.o hq_hqadata.o hq_hqadsp.o \
>> @@ -621,6 +623,7 @@ OBJS-$(CONFIG_VP8_DECODER) += vp8.o vp56rac.o
>> OBJS-$(CONFIG_VP8_CUVID_DECODER) += cuvid.o
>> OBJS-$(CONFIG_VP8_MEDIACODEC_DECODER) += mediacodecdec.o
>> OBJS-$(CONFIG_VP8_QSV_DECODER) += qsvdec_other.o
>> +OBJS-$(CONFIG_VP8_RKMPP_DECODER) += rkmppdec.o
>> OBJS-$(CONFIG_VP8_VAAPI_ENCODER) += vaapi_encode_vp8.o
>> OBJS-$(CONFIG_VP9_DECODER) += vp9.o vp9data.o vp9dsp.o vp9lpf.o vp9recon.o \
>> vp9block.o vp9prob.o vp9mvs.o vp56rac.o \
>> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
>> index 1e5942d..2d7124a 100644
>> --- a/libavcodec/allcodecs.c
>> +++ b/libavcodec/allcodecs.c
>> @@ -212,6 +212,7 @@ static void register_all(void)
>> REGISTER_DECODER(H264_MEDIACODEC, h264_mediacodec);
>> REGISTER_DECODER(H264_MMAL, h264_mmal);
>> REGISTER_DECODER(H264_QSV, h264_qsv);
>> + REGISTER_DECODER(H264_RKMPP, h264_rkmpp);
>> REGISTER_DECODER(H264_VDA, h264_vda);
>> #if FF_API_VDPAU
>> REGISTER_DECODER(H264_VDPAU, h264_vdpau);
>> @@ -219,6 +220,7 @@ static void register_all(void)
>> REGISTER_ENCDEC (HAP, hap);
>> REGISTER_DECODER(HEVC, hevc);
>> REGISTER_DECODER(HEVC_QSV, hevc_qsv);
>> + REGISTER_DECODER(HEVC_RKMPP, hevc_rkmpp);
>> REGISTER_DECODER(HNM4_VIDEO, hnm4_video);
>> REGISTER_DECODER(HQ_HQA, hq_hqa);
>> REGISTER_DECODER(HQX, hqx);
>> @@ -372,6 +374,7 @@ static void register_all(void)
>> REGISTER_DECODER(VP6F, vp6f);
>> REGISTER_DECODER(VP7, vp7);
>> REGISTER_DECODER(VP8, vp8);
>> + REGISTER_DECODER(VP8_RKMPP, vp8_rkmpp);
>> REGISTER_DECODER(VP9, vp9);
>> REGISTER_DECODER(VQA, vqa);
>> REGISTER_DECODER(BITPACKED, bitpacked);
>> diff --git a/libavcodec/rkmppdec.c b/libavcodec/rkmppdec.c
>> new file mode 100644
>> index 0000000..37ea074
>> --- /dev/null
>> +++ b/libavcodec/rkmppdec.c
>> @@ -0,0 +1,567 @@
>> +/*
>> + * RockChip MPP Video Decoder
>> + * Copyright (c) 2017 Lionel CHAZALLON
>> + *
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#include <drm_fourcc.h>
>> +#include <pthread.h>
>> +#include <rockchip/mpp_buffer.h>
>> +#include <rockchip/rk_mpi.h>
>> +#include <time.h>
>> +#include <unistd.h>
>> +
>> +#include "avcodec.h"
>> +#include "decode.h"
>> +#include "internal.h"
>> +#include "libavutil/buffer.h"
>> +#include "libavutil/common.h"
>> +#include "libavutil/frame.h"
>> +#include "libavutil/hwcontext.h"
>> +#include "libavutil/hwcontext_drm.h"
>> +#include "libavutil/imgutils.h"
>> +#include "libavutil/log.h"
>> +
>> +#define RECEIVE_FRAME_TIMEOUT 100
>> +#define FRAMEGROUP_MAX_FRAMES 16
>> +
>> +typedef struct {
>> + MppCtx ctx;
>> + MppApi *mpi;
>> + MppBufferGroup frame_group;
>> +
>> + char first_frame;
>> + char first_packet;
>> + char eos_reached;
>> +
>> + AVBufferRef *frames_ref;
>> + AVBufferRef *device_ref;
>> +} RKMPPDecoder;
>> +
>> +typedef struct {
>> + AVClass *av_class;
>> + AVBufferRef *decoder_ref;
>> +} RKMPPDecodeContext;
>> +
>> +typedef struct {
>> + MppFrame frame;
>> + AVBufferRef *decoder_ref;
>> +} RKMPPFrameContext;
>> +
>> +static MppCodingType rkmpp_get_codingtype(AVCodecContext *avctx)
>> +{
>> + switch (avctx->codec_id) {
>> + case AV_CODEC_ID_H264: return MPP_VIDEO_CodingAVC;
>> + case AV_CODEC_ID_HEVC: return MPP_VIDEO_CodingHEVC;
>> + case AV_CODEC_ID_VP8: return MPP_VIDEO_CodingVP8;
>> + default: return MPP_VIDEO_CodingUnused;
>> + }
>> +}
>> +
>> +static int rkmpp_get_frameformat(MppFrameFormat mppformat)
>> +{
>> + switch (mppformat) {
>> + case MPP_FMT_YUV420SP: return DRM_FORMAT_NV12;
>> +#ifdef DRM_FORMAT_NV12_10
>> + case MPP_FMT_YUV420SP_10BIT: return DRM_FORMAT_NV12_10;
>> +#endif
>> + default: return 0;
>> + }
>> +}
>> +
>> +static int rkmpp_write_data(AVCodecContext *avctx, uint8_t *buffer, int size, int64_t pts)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
>> + int ret = MPP_NOK;
>> + MppPacket packet;
>> +
>> + // create the MPP packet
>> + ret = mpp_packet_init(&packet, buffer, size);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to init MPP packet (code = %d)\n", ret);
>> + return AVERROR_UNKNOWN;
>> + }
>> +
>> + mpp_packet_set_pts(packet, pts);
>> +
>> + if (!buffer)
>> + mpp_packet_set_eos(packet);
>> +
>> + ret = decoder->mpi->decode_put_packet(decoder->ctx, packet);
>> + if (ret != MPP_OK) {
>> + if (ret == MPP_ERR_BUFFER_FULL) {
>> + av_log(avctx, AV_LOG_DEBUG, "Buffer full writing %d bytes to decoder\n", size);
>> + ret = AVERROR(EAGAIN);
>> + } else
>> + ret = AVERROR_UNKNOWN;
>> + }
>> + else
>> + av_log(avctx, AV_LOG_DEBUG, "Wrote %d bytes to decoder\n", size);
>> +
>> + mpp_packet_deinit(&packet);
>> +
>> + return ret;
>> +}
>> +
>> +static int rkmpp_close_decoder(AVCodecContext *avctx)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + av_buffer_unref(&rk_context->decoder_ref);
>> + return 0;
>> +}
>> +
>> +static void rkmpp_release_decoder(void *opaque, uint8_t *data)
>> +{
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)data;
>> +
>> + decoder->mpi->reset(decoder->ctx);
>> + mpp_destroy(decoder->ctx);
>> + decoder->ctx = NULL;
>> +
>> + if (decoder->frame_group) {
>> + mpp_buffer_group_put(decoder->frame_group);
>> + decoder->frame_group = NULL;
>> + }
>> +
>> + av_buffer_unref(&decoder->frames_ref);
>> + av_buffer_unref(&decoder->device_ref);
>> +
>> + av_free(decoder);
>> +}
>> +
>> +static int rkmpp_init_decoder(AVCodecContext *avctx)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = NULL;
>> + MppCodingType codectype = MPP_VIDEO_CodingUnused;
>> + int ret = MPP_NOK;
>> + RK_S64 paramS64;
>> + RK_S32 paramS32;
>> +
>> + avctx->pix_fmt = AV_PIX_FMT_DRM_PRIME;
>> +
>> + // create a decoder and a ref to it
>> + decoder = av_mallocz(sizeof(RKMPPDecoder));
>> + if (!decoder) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + rk_context->decoder_ref = av_buffer_create((uint8_t *)decoder, sizeof(*decoder), rkmpp_release_decoder,
>> + NULL, AV_BUFFER_FLAG_READONLY);
>> + if (!rk_context->decoder_ref) {
>> + av_free(decoder);
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + av_log(avctx, AV_LOG_DEBUG, "Initializing RKMPP decoder.\n");
>> +
>> + codectype = rkmpp_get_codingtype(avctx);
>> + if (codectype == MPP_VIDEO_CodingUnused) {
>> + av_log(avctx, AV_LOG_ERROR, "Unknown codec type (%d).\n", avctx->codec_id);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + // Create the MPP context
>> + ret = mpp_create(&decoder->ctx, &decoder->mpi);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to create MPP context (code = %d).\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + // initialize mpp
>> + ret = mpp_init(decoder->ctx, MPP_CTX_DEC, codectype);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to initialize MPP context (code = %d).\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + // make decode calls blocking with a timeout
>> + paramS32 = MPP_POLL_BLOCK;
>> + ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK, ¶mS32);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to set blocking mode on MPI (code = %d).\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + paramS64 = RECEIVE_FRAME_TIMEOUT;
>> + ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK_TIMEOUT, ¶mS64);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to set block timeout on MPI (code = %d).\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + ret = mpp_buffer_group_get_internal(&decoder->frame_group, MPP_BUFFER_TYPE_DRM);
>> + if (ret) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to retrieve buffer group (code = %d)\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + ret = decoder->mpi->control(decoder->ctx, MPP_DEC_SET_EXT_BUF_GROUP, decoder->frame_group);
>> + if (ret) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to assign buffer group (code = %d)\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + ret = mpp_buffer_group_limit_config(decoder->frame_group, 0, FRAMEGROUP_MAX_FRAMES);
>> + if (ret) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to set buffer group limit (code = %d)\n", ret);
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + decoder->first_packet = 1;
>> +
>> + av_log(avctx, AV_LOG_DEBUG, "RKMPP decoder initialized successfully.\n");
>> +
>> + decoder->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM);
>> + if (!decoder->device_ref) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> + ret = av_hwdevice_ctx_init(decoder->device_ref);
>> + if (ret < 0)
>> + goto fail;
>> +
>> + return 0;
>> +
>> +fail:
>> + av_log(avctx, AV_LOG_ERROR, "Failed to initialize RKMPP decoder.\n");
>> + rkmpp_close_decoder(avctx);
>> + return ret;
>> +}
>> +
>> +static int rkmpp_send_packet(AVCodecContext *avctx, const AVPacket *avpkt)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
>> + int ret = MPP_NOK;
>> +
>> + // handle EOF
>> + if (!avpkt->size) {
>> + av_log(avctx, AV_LOG_DEBUG, "End of stream.\n");
>> + decoder->eos_reached = 1;
>> + ret = rkmpp_write_data(avctx, NULL, 0, 0);
>> + if (ret)
>> + av_log(avctx, AV_LOG_ERROR, "Failed to send EOS to decoder (code = %d)\n", ret);
>> + return ret;
>> + }
>> +
>> + // on first packet, send extradata
>> + if (decoder->first_packet) {
>> + if (avctx->extradata_size) {
>> + ret = rkmpp_write_data(avctx, avctx->extradata,
>> + avctx->extradata_size,
>> + avpkt->pts);
>> + if (ret) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to write extradata to decoder (code = %d)\n", ret);
>> + return ret;
>> + }
>> + }
>> + decoder->first_packet = 0;
>> + }
>> +
>> + // now send packet
>> + ret = rkmpp_write_data(avctx, avpkt->data, avpkt->size, avpkt->pts);
>> + if (ret && ret!=AVERROR(EAGAIN))
>> + av_log(avctx, AV_LOG_ERROR, "Failed to write data to decoder (code = %d)\n", ret);
>> +
>> + return ret;
>> +}
>> +
>> +static void rkmpp_release_frame(void *opaque, uint8_t *data)
>> +{
>> + AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)data;
>> + AVBufferRef *framecontextref = (AVBufferRef *)opaque;
>> + RKMPPFrameContext *framecontext = (RKMPPFrameContext *)framecontextref->data;
>> +
>> + mpp_frame_deinit(&framecontext->frame);
>> + av_buffer_unref(&framecontext->decoder_ref);
>> + av_buffer_unref(&framecontextref);
>> +
>> + av_free(desc);
>> +}
>> +
>> +static int rkmpp_retrieve_frame(AVCodecContext *avctx, AVFrame *frame)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
>> + RKMPPFrameContext *framecontext = NULL;
>> + AVBufferRef *framecontextref = NULL;
>> + int ret = MPP_NOK;
>> + MppFrame mppframe = NULL;
>> + MppBuffer buffer = NULL;
>> + AVDRMFrameDescriptor *desc = NULL;
>> + AVDRMLayerDescriptor *layer = NULL;
>> + int retrycount = 0;
>> +
>> + // on start of decoding, MPP can return -1, which is supposed to be expected
>> + // this is due to some internal MPP init which is not completed, that will
>> + // only happen in the first few frames queries, but should not be interpreted
>> + // as an error, Therefore we need to retry a couple times when we get -1
>> + // in order to let it time to complete it's init, then we sleep a bit between retries.
>> +retry_get_frame:
>> + ret = decoder->mpi->decode_get_frame(decoder->ctx, &mppframe);
>> + if (ret != MPP_OK && ret != MPP_ERR_TIMEOUT && !decoder->first_frame) {
>> + if (retrycount < 5) {
>> + av_log(avctx, AV_LOG_DEBUG, "Failed to get a frame, retrying (code = %d, retrycount = %d)\n", ret, retrycount);
>> + usleep(10000);
>> + retrycount++;
>> + goto retry_get_frame;
>> + } else {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to get a frame from MPP (code = %d)\n", ret);
>> + goto fail;
>> + }
>> + }
> How does the decode_get_frame() call actually work wrt blocking? If you don't get a frame here (and it isn't EOF) then you print a message saying it was a timeout - was it actually a timeout, or does this also cover cases where there wasn't enough data to decode a frame? (I'm concerned that this might be introducing delays for no good reason.)
decode_get_frame() will block for a maximum of RECEIVE_FRAME_TIMEOUT
(set to 100ms in the decoder init) currently. I understand your concern
about that value and it was something that i probably discussed with
Rockchip.
When we feed data to the decoder, we feed as much as we can checking the
input buffer slots, and we will give it as much data as it can take.
Once this is done, all we can do, as we cannot write more data is wait
on decode_get_frame.
It will either return a frame or timeout in which case, the vpu would
have probably eaten some packets and freed some input slots that we will
feed right away.
>
> Also, presumably this uses the timeout number set above (RECEIVE_FRAME_TIMEOUT) - why does it have the value it does?
Well i have no precise spurce for that value, and we could adjust it if
you feel it's too high. From en experimental point of view, i wouldn't
get it lower than 20-30 ms.
Let me know if lower values would look more appropriate.
>> +
>> + if (mppframe) {
>> + // Check wether we have a special frame or not
>> + if (mpp_frame_get_info_change(mppframe)) {
>> + AVHWFramesContext *hwframes;
>> +
>> + av_log(avctx, AV_LOG_INFO, "Decoder noticed an info change (%dx%d), format=%d\n",
>> + (int)mpp_frame_get_width(mppframe), (int)mpp_frame_get_height(mppframe),
>> + (int)mpp_frame_get_fmt(mppframe));
>> +
>> + avctx->width = mpp_frame_get_width(mppframe);
>> + avctx->height = mpp_frame_get_height(mppframe);
>> +
>> + decoder->mpi->control(decoder->ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
>> + decoder->first_frame = 1;
>> +
>> + av_buffer_unref(&decoder->frames_ref);
>> +
>> + decoder->frames_ref = av_hwframe_ctx_alloc(decoder->device_ref);
>> + if (!decoder->frames_ref) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> + hwframes = (AVHWFramesContext*)decoder->frames_ref->data;
>> + hwframes->format = AV_PIX_FMT_DRM_PRIME;
>> + hwframes->sw_format = AV_PIX_FMT_NV12;
> It isn't NV12 in the 10-bit case. We don't have a pixfmt to represent the Rockchip 10-bit format - would it be sensible to add one (probably needs some support to actually convert it to something sensible)? If not, probably put something else here (does NONE work?) to emphasise that it's opaque and shouldn't be interacted with directly.
This is right, the Rockchip DRM_FORMAT_NV12_10 doesnt have any
equivalent that i know in ffmpeg (10 bits packed).
Converting that packed data looks rather heavy imho, i could though in
that case put AV_PIX_FMT_NONE in the sw_format so that it doesn't get
misinterpreted.
>
>> + hwframes->width = avctx->width;
>> + hwframes->height = avctx->height;
>> + ret = av_hwframe_ctx_init(decoder->frames_ref);
>> + if (ret < 0)
>> + goto fail;
>> +
>> + // here decoder is fully initialized, we need to feed it again with data
>> + ret = AVERROR(EAGAIN);
>> + goto fail;
>> + } else if (mpp_frame_get_eos(mppframe)) {
>> + av_log(avctx, AV_LOG_DEBUG, "Received a EOS frame.\n");
>> + decoder->eos_reached = 1;
>> + ret = AVERROR_EOF;
>> + goto fail;
>> + } else if (mpp_frame_get_discard(mppframe)) {
>> + av_log(avctx, AV_LOG_DEBUG, "Received a discard frame.\n");
>> + ret = AVERROR(EAGAIN);
>> + goto fail;
>> + } else if (mpp_frame_get_errinfo(mppframe)) {
>> + av_log(avctx, AV_LOG_ERROR, "Received a errinfo frame.\n");
>> + ret = AVERROR_UNKNOWN;
>> + goto fail;
>> + }
>> +
>> + // here we should have a valid frame
>> + av_log(avctx, AV_LOG_DEBUG, "Received a frame.\n");
>> +
>> + // setup general frame fields
>> + frame->format = AV_PIX_FMT_DRM_PRIME;
>> + frame->width = mpp_frame_get_width(mppframe);
>> + frame->height = mpp_frame_get_height(mppframe);
>> + frame->pts = mpp_frame_get_pts(mppframe);
> Can you get other frame metadata here? (SAR, all of the colour stuff?)
well there are a few other functions that i didn't use that seem to be
colorspace related here
https://github.com/rockchip-linux/mpp/blob/30032ae303956788bd5b709c9bc70867b0ea95d6/inc/mpp_frame.h#L242-L253
I never looked into those yet, but if you feel like some would be
helpful and important I could have another looks there.
> Does anything sensible happen with interlaced streams?
there are flags defined for this in the MPP headers, but i don't know if
they are effectively used in MppFrame. Would need to check further.
https://github.com/rockchip-linux/mpp/blob/30032ae303956788bd5b709c9bc70867b0ea95d6/inc/mpp_frame.h#L28-L43
>
>> +
>> + // now setup the frame buffer info
>> + buffer = mpp_frame_get_buffer(mppframe);
>> + if (buffer) {
>> + desc = av_mallocz(sizeof(AVDRMFrameDescriptor));
>> + if (!desc) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + desc->nb_objects = 1;
>> + desc->objects[0].fd = mpp_buffer_get_fd(buffer);
>> + desc->objects[0].size = mpp_buffer_get_size(buffer);
>> +
>> + desc->nb_layers = 1;
>> + layer = &desc->layers[0];
>> + layer->format = rkmpp_get_frameformat(mpp_frame_get_fmt(mppframe));
>> + layer->nb_planes = 2;
>> +
>> + layer->planes[0].object_index = 0;
>> + layer->planes[0].offset = 0;
>> + layer->planes[0].pitch = mpp_frame_get_hor_stride(mppframe);
>> +
>> + layer->planes[1].object_index = 0;
>> + layer->planes[1].offset = layer->planes[0].pitch * mpp_frame_get_ver_stride(mppframe);
>> + layer->planes[1].pitch = layer->planes[0].pitch;
>> +
>> + // we also allocate a struct in buf[0] that will allow to hold additionnal information
>> + // for releasing properly MPP frames and decoder
>> + framecontextref = av_buffer_allocz(sizeof(*framecontext));
>> + if (!framecontextref) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + // MPP decoder needs to be closed only when all frames have been released.
>> + framecontext = (RKMPPFrameContext *)framecontextref->data;
>> + framecontext->decoder_ref = av_buffer_ref(rk_context->decoder_ref);
>> + framecontext->frame = mppframe;
>> +
>> + frame->data[0] = (uint8_t *)desc;
>> + frame->buf[0] = av_buffer_create((uint8_t *)desc, sizeof(*desc), rkmpp_release_frame,
>> + framecontextref, AV_BUFFER_FLAG_READONLY);
>> +
>> + if (!frame->buf[0]) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + frame->hw_frames_ctx = av_buffer_ref(decoder->frames_ref);
>> + if (!frame->hw_frames_ctx) {
>> + ret = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + decoder->first_frame = 0;
>> + return 0;
>> + } else {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret);
>> + mpp_frame_deinit(&mppframe);
>> + }
>> + } else if (decoder->eos_reached) {
>> + return AVERROR_EOF;
>> + } else if (ret == MPP_ERR_TIMEOUT) {
>> + av_log(avctx, AV_LOG_DEBUG, "Timeout when trying to get a frame from MPP\n");
>> + }
>> +
>> + return AVERROR(EAGAIN);
>> +
>> +fail:
>> + if (mppframe)
>> + mpp_frame_deinit(&mppframe);
>> +
>> + if (framecontext)
>> + av_buffer_unref(&framecontext->decoder_ref);
>> +
>> + if (framecontextref)
>> + av_buffer_unref(&framecontextref);
>> +
>> + if (desc)
>> + av_free(desc);
>> +
>> + return ret;
>> +}
>> +
>> +static int rkmpp_receive_frame(AVCodecContext *avctx, AVFrame *frame)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
>> + int ret = MPP_NOK;
>> + AVPacket pkt = {0};
>> + RK_S32 freeslots;
>> +
>> + if (!decoder->eos_reached) {
>> + // we get the available slots in decoder
>> + ret = decoder->mpi->control(decoder->ctx, MPP_DEC_GET_FREE_PACKET_SLOT_COUNT, &freeslots);
>> + if (ret != MPP_OK) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to get decoder free slots (code = %d).\n", ret);
>> + return ret;
>> + }
>> +
>> + if (freeslots > 0) {
>> + ret = ff_decode_get_packet(avctx, &pkt);
>> + if (ret < 0 && ret != AVERROR_EOF) {
>> + return ret;
>> + }
>> +
>> + ret = rkmpp_send_packet(avctx, &pkt);
>> + av_packet_unref(&pkt);
>> +
>> + if (ret < 0) {
>> + av_log(avctx, AV_LOG_ERROR, "Failed to send packet to decoder (code = %d)\n", ret);
>> + return ret;
>> + }
>> + }
>> +
>> + // make sure we keep decoder full
>> + if (freeslots > 1 && decoder->first_frame)
>> + return AVERROR(EAGAIN);
>> + }
>> +
>> + return rkmpp_retrieve_frame(avctx, frame);
>> +}
>> +
>> +static void rkmpp_flush(AVCodecContext *avctx)
>> +{
>> + RKMPPDecodeContext *rk_context = avctx->priv_data;
>> + RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
>> + int ret = MPP_NOK;
>> +
>> + av_log(avctx, AV_LOG_DEBUG, "Flush.\n");
>> +
>> + ret = decoder->mpi->reset(decoder->ctx);
>> + if (ret == MPP_OK) {
>> + decoder->first_frame = 1;
>> + decoder->first_packet = 1;
>> + } else
>> + av_log(avctx, AV_LOG_ERROR, "Failed to reset MPI (code = %d)\n", ret);
>> +}
>> +
>> +
>> +#define RKMPP_DEC_CLASS(NAME) \
>> + static const AVClass rkmpp_##NAME##_dec_class = { \
>> + .class_name = "rkmpp_" #NAME "_dec", \
>> + .version = LIBAVUTIL_VERSION_INT, \
>> + };
>> +
>> +#define RKMPP_DEC(NAME, ID, BSFS) \
>> + RKMPP_DEC_CLASS(NAME) \
>> + AVCodec ff_##NAME##_rkmpp_decoder = { \
>> + .name = #NAME "_rkmpp", \
>> + .long_name = NULL_IF_CONFIG_SMALL(#NAME " (rkmpp)"), \
>> + .type = AVMEDIA_TYPE_VIDEO, \
>> + .id = ID, \
>> + .priv_data_size = sizeof(RKMPPDecodeContext), \
>> + .init = rkmpp_init_decoder, \
>> + .close = rkmpp_close_decoder, \
>> + .receive_frame = rkmpp_receive_frame, \
>> + .flush = rkmpp_flush, \
>> + .priv_class = &rkmpp_##NAME##_dec_class, \
>> + .capabilities = AV_CODEC_CAP_DELAY, \
>> + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
>> + .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \
>> + AV_PIX_FMT_NONE}, \
>> + .bsfs = BSFS, \
>> + };
>> +
>> +RKMPP_DEC(h264, AV_CODEC_ID_H264, "h264_mp4toannexb")
>> +RKMPP_DEC(hevc, AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
>> +RKMPP_DEC(vp8, AV_CODEC_ID_VP8, NULL)
>>
> Generally looks good, and I was able to test it a bit. I'll try to test some more in the next few days.
>
> Thanks,
>
> - Mark
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
More information about the ffmpeg-devel
mailing list