[FFmpeg-devel] [PATCH v2 11/11] avcodec: add vvdec H.266/VVC decoder

Nuo Mi nuomi2021 at gmail.com
Sat Jan 9 09:34:21 EET 2021


you can download test clips here:
https://www.itu.int/wftp3/av-arch/jvet-site/bitstream_exchange/VVC/under_test/VTM-11.0/

76.71% (191/249) clips are md5 matched with VTM 11:

passed:

10b400_A_Bytedance_2.bit
10b400_B_Bytedance_2.bit
8b400_A_Bytedance_2.bit
8b400_B_Bytedance_2.bit
8b420_A_Bytedance_2.bit
8b420_B_Bytedance_2.bit
ACTPIC_A_Huawei_3.bit
ACTPIC_B_Huawei_3.bit
ACTPIC_C_Huawei_3.bit
AFF_A_HUAWEI_2.bit
AFF_B_HUAWEI_2.bit
ALF_A_Huawei_3.bit
ALF_B_Huawei_3.bit
ALF_C_KDDI_2.bit
ALF_D_Qualcomm_2.bit
AMVR_A_HHI_3.bit
AMVR_B_HHI_3.bit
APSALF_A_Qualcomm_2.bit
APSLMCS_A_Dolby_3.bit
APSLMCS_B_Dolby_3.bit
APSLMCS_C_Dolby_2.bit
APSMULT_A_MediaTek_3.bit
APSMULT_B_MediaTek_3.bit
AUD_A_Broadcom_3.bit
BCW_A_MediaTek_3.bit
BCW_A_MediaTek_4.bit
BDOF_A_MediaTek_3.bit
BDOF_A_MediaTek_4.bit
BDPCM_A_Orange_2.bit
CCALF_A_Sharp_3.bit
CCALF_B_Sharp_3.bit
CCALF_C_Sharp_3.bit
CCALF_D_Sharp_3.bit
CCLM_A_KDDI_1.bit
CIIP_A_MediaTek_3.bit
CIIP_A_MediaTek_4.bit
CodingToolsSets_A_Tencent_2.bit
CodingToolsSets_B_Tencent_2.bit
CodingToolsSets_C_Tencent_2.bit
CodingToolsSets_D_Tencent_2.bit
CROP_A_Panasonic_3.bit
CROP_B_Panasonic_4.bit
CST_A_MediaTek_3.bit
CTU_A_MediaTek_3.bit
CTU_A_MediaTek_4.bit
CTU_B_MediaTek_3.bit
CTU_B_MediaTek_4.bit
CTU_C_MediaTek_3.bit
CTU_C_MediaTek_4.bit
CUBEMAP_A_MediaTek_3.bit
CUBEMAP_B_MediaTek_3.bit
CUBEMAP_C_MediaTek_3.bit
DEBLOCKING_A_Sharp_3.bit
DEBLOCKING_B_Sharp_2.bit
DEBLOCKING_C_Huawei_3.bit
DEBLOCKING_E_Ericsson_2.bit
DEBLOCKING_E_Ericsson_3.bit
DEBLOCKING_F_Ericsson_1.bit
DEBLOCKING_F_Ericsson_2.bit
DMVR_A_Huawei_3.bit
DMVR_B_KDDI_3.bit
DPB_A_Sharplabs_2.bit
DPB_B_Sharplabs_2.bit
DQ_A_HHI_3.bit
ENT444HIGHTIER_A_Sony_3.bit
ENT444HIGHTIER_B_Sony_3.bit
ENT444HIGHTIER_C_Sony_3.bit
ENT444HIGHTIER_D_Sony_3.bit
ENT444MAINTIER_A_Sony_3.bit
ENT444MAINTIER_B_Sony_3.bit
ENT444MAINTIER_C_Sony_3.bit
ENT444MAINTIER_D_Sony_3.bit
ENTHIGHTIER_A_Sony_3.bit
ENTHIGHTIER_B_Sony_3.bit
ENTHIGHTIER_C_Sony_3.bit
ENTHIGHTIER_D_Sony_3.bit
ENTMAINTIER_A_Sony_3.bit
ENTMAINTIER_B_Sony_3.bit
ENTMAINTIER_C_Sony_3.bit
ENTMAINTIER_D_Sony_3.bit
ENTROPY_A_Chipsnmedia_2.bit
ENTROPY_A_Qualcomm_2.bit
ENTROPY_B_Sharp_2.bit
ERP_A_MediaTek_3.bit
FILLER_A_Bytedance_1.bit
GPM_A_Alibaba_3.bit
HLG_A_NHK_2.bit
HLG_B_NHK_2.bit
HRD_A_Fujitsu_3.bit
HRD_B_Fujitsu_2.bit
IBC_A_Tencent_2.bit
IBC_B_Tencent_2.bit
IBC_C_Tencent_2.bit
IBC_D_Tencent_2.bit
IP_B_Nokia_1.bit
ISP_A_HHI_3.bit
ISP_B_HHI_3.bit
JCCR_A_Nokia_2.bit
JCCR_B_Nokia_2.bit
JCCR_C_HHI_3.bit
JCCR_E_Nokia_1.bit
JCCR_F_Nokia_1.bit
LFNST_A_LGE_3.bit
LFNST_B_LGE_3.bit
LFNST_C_HHI_3.bit
LMCS_A_Dolby_3.bit
LOSSLESS_A_HHI_3.bit
LOSSLESS_B_HHI_3.bit
LTRP_A_ERICSSON_2.bit
MERGE_A_Qualcomm_2.bit
MERGE_B_Qualcomm_2.bit
MERGE_C_Qualcomm_2.bit
MERGE_D_Qualcomm_2.bit
MERGE_E_Qualcomm_2.bit
MERGE_F_Qualcomm_2.bit
MERGE_G_Qualcomm_2.bit
MERGE_H_Qualcomm_2.bit
MERGE_I_Qualcomm_2.bit
MERGE_J_Qualcomm_2.bit
MIP_A_HHI_3.bit
MIP_B_HHI_3.bit
MPM_A_LGE_3.bit
MRLP_A_HHI_2.bit
MRLP_B_HHI_2.bit
MTS_A_LGE_3.bit
MTS_B_LGE_3.bit
MTS_LFNST_A_LGE_3.bit
MTS_LFNST_B_LGE_3.bit
MVCOMP_A_Sharp_2.bit
PDPC_A_Qualcomm_3.bit
PDPC_B_Qualcomm_3.bit
PDPC_C_Qualcomm_2.bit
PHSH_B_Sharp_1.bit
POC_A_Nokia_1.bit
POUT_A_Sharplabs_2.bit
PPS_B_Bytedance_1.bit
PPS_C_Bytedance_1.bit
PQ_A_Dolby_1.bit
PROF_A_Interdigital_3.bit
PROF_B_Interdigital_3.bit
PSEXT_A_Nokia_2.bit
PSEXT_B_Nokia_2.bit
QTBTT_A_MediaTek_3.bit
QTBTT_A_MediaTek_4.bit
QUANT_A_Huawei_2.bit
QUANT_B_Huawei_2.bit
QUANT_C_Huawei_2.bit
QUANT_D_Huawei_2.bit
RAP_C_HHI_1.bit
RAP_D_HHI_1.bit
RPL_A_ERICSSON_2.bit
SAO_A_SAMSUNG_3.bit
SAO_B_SAMSUNG_3.bit
SAO_C_SAMSUNG_3.bit
SbTMVP_A_Bytedance_3.bit
SbTMVP_B_Bytedance_3.bit
SBT_A_HUAWEI_2.bit
SCALING_A_InterDigital_1.bit
SCALING_B_InterDigital_1.bit
SCALING_C_InterDigital_1.bit
SDH_A_Dolby_2.bit
SLICES_A_HUAWEI_2.bit
SMVD_A_HUAWEI_2.bit
SPS_A_Bytedance_1.bit
SPS_B_Bytedance_1.bit
SPS_C_Bytedance_1.bit
TEMPSCAL_A_Panasonic_4.bit
TEMPSCAL_C_Panasonic_3.bit
TILE_A_Nokia_2.bit
TILE_B_Nokia_2.bit
TILE_C_Nokia_2.bit
TILE_D_Nokia_2.bit
TILE_E_Nokia_2.bit
TILE_F_Nokia_2.bit
TILE_G_Nokia_2.bit
TMVP_A_Chipsnmedia_3.bit
TMVP_B_Chipsnmedia_3.bit
TMVP_C_Chipsnmedia_3.bit
TMVP_D_Chipsnmedia_3.bit
TRANS_A_Chipsnmedia_2.bit
TRANS_B_Chipsnmedia_2.bit
TRANS_C_Chipsnmedia_2.bit
TRANS_D_Chipsnmedia_2.bit
WPP_A_Sharp_3.bit
WPP_B_Sharp_2.bit
WP_A_InterDigital_3.bit
WP_B_InterDigital_3.bit
WRAP_A_InterDigital_4.bit
WRAP_B_InterDigital_4.bit
WRAP_C_InterDigital_4.bit
WRAP_D_InterDigital_4.bit
---
 configure               |   5 +-
 libavcodec/Makefile     |   1 +
 libavcodec/allcodecs.c  |   1 +
 libavcodec/libvvdec.cpp | 244 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 250 insertions(+), 1 deletion(-)
 create mode 100644 libavcodec/libvvdec.cpp

diff --git a/configure b/configure
index b41f2af151..cefdff75fe 100755
--- a/configure
+++ b/configure
@@ -285,6 +285,7 @@ External library support:
   --enable-libvorbis       enable Vorbis en/decoding via libvorbis,
                            native implementation exists [no]
   --enable-libvpx          enable VP8 and VP9 de/encoding via libvpx [no]
+  --enable-libvvdec        enable VVC video decoding via libvvdec [no]
   --enable-libwebp         enable WebP encoding via libwebp [no]
   --enable-libx264         enable H.264 encoding via x264 [no]
   --enable-libx265         enable HEVC encoding via x265 [no]
@@ -1816,6 +1817,7 @@ EXTERNAL_LIBRARY_LIST="
     libvmaf
     libvorbis
     libvpx
+    libvvdec
     libwebp
     libxml2
     libzimg
@@ -3284,6 +3286,7 @@ libvpx_vp8_decoder_deps="libvpx"
 libvpx_vp8_encoder_deps="libvpx"
 libvpx_vp9_decoder_deps="libvpx"
 libvpx_vp9_encoder_deps="libvpx"
+libvvdec_decoder_deps="libvvdec"
 libwebp_encoder_deps="libwebp"
 libwebp_anim_encoder_deps="libwebp"
 libx262_encoder_deps="libx262"
@@ -6461,7 +6464,7 @@ enabled libvpx            && {
         die "libvpx enabled but no supported decoders found"
     fi
 }
-
+enabled libvvdec          && require_pkg_config libvvdec "libvvdec >= 0.1.2" vvdec/version.h VVDEC_VERSION_MAJOR
 enabled libwebp           && {
     enabled libwebp_encoder      && require_pkg_config libwebp "libwebp >= 0.2.0" webp/encode.h WebPGetEncoderVersion
     enabled libwebp_anim_encoder && check_pkg_config libwebp_anim_encoder "libwebpmux >= 0.4.0" webp/mux.h WebPAnimEncoderOptionsInit; }
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index d4cd64e43c..cf734144f6 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1059,6 +1059,7 @@ OBJS-$(CONFIG_LIBVPX_VP8_DECODER)         += libvpxdec.o
 OBJS-$(CONFIG_LIBVPX_VP8_ENCODER)         += libvpxenc.o
 OBJS-$(CONFIG_LIBVPX_VP9_DECODER)         += libvpxdec.o libvpx.o
 OBJS-$(CONFIG_LIBVPX_VP9_ENCODER)         += libvpxenc.o libvpx.o
+OBJS-$(CONFIG_LIBVVDEC_DECODER)           += libvvdec.o
 OBJS-$(CONFIG_LIBWEBP_ENCODER)            += libwebpenc_common.o libwebpenc.o
 OBJS-$(CONFIG_LIBWEBP_ANIM_ENCODER)       += libwebpenc_common.o libwebpenc_animencoder.o
 OBJS-$(CONFIG_LIBX262_ENCODER)            += libx264.o
diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
index f00d524747..545c38f541 100644
--- a/libavcodec/allcodecs.c
+++ b/libavcodec/allcodecs.c
@@ -752,6 +752,7 @@ extern AVCodec ff_libvpx_vp8_encoder;
 extern AVCodec ff_libvpx_vp8_decoder;
 extern AVCodec ff_libvpx_vp9_encoder;
 extern AVCodec ff_libvpx_vp9_decoder;
+extern AVCodec ff_libvvdec_decoder;
 /* preferred over libwebp */
 extern AVCodec ff_libwebp_anim_encoder;
 extern AVCodec ff_libwebp_encoder;
diff --git a/libavcodec/libvvdec.cpp b/libavcodec/libvvdec.cpp
new file mode 100644
index 0000000000..4229d45701
--- /dev/null
+++ b/libavcodec/libvvdec.cpp
@@ -0,0 +1,244 @@
+/*
+ * vvdec H.266/VVC decoder
+ * Copyright (c) 2020 Nuo Mi <nuomi2021 at gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * VVC decoder support via libvvdec
+ */
+
+#include "vvdec/vvdec.h"
+
+extern "C" {
+
+#include "libavutil/common.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/internal.h"
+
+#include "avcodec.h"
+#include "h2645_parse.h"
+#include "internal.h"
+#include "profiles.h"
+
+typedef struct VVDecContext {
+    AVCodecContext *avctx;
+    vvdec::VVDec* vvdec;
+
+    H2645Packet pkt;
+    int is_nalff;           ///< this flag is != 0 if bitstream is encapsulated
+                            ///< as a format defined in 14496-15
+    int nal_length_size;    ///< Number of bytes used for nal length (1, 2 or 4)
+} VVDecContext;
+
+}
+
+static int map_error(int vret)
+{
+    switch(vret)
+    {
+        case vvdec::VVDEC_OK :                  return 0;
+        case vvdec::VVDEC_ERR_UNSPECIFIED:      return AVERROR_UNKNOWN;
+        case vvdec::VVDEC_ERR_INITIALIZE:       return AVERROR_BUG;
+        case vvdec::VVDEC_ERR_ALLOCATE:         return AVERROR(ENOMEM);
+        case vvdec::VVDEC_NOT_ENOUGH_MEM:       return AVERROR(ENOMEM);
+        case vvdec::VVDEC_ERR_PARAMETER:        return AVERROR(EINVAL);
+        case vvdec::VVDEC_ERR_NOT_SUPPORTED:    return AVERROR(ENOSYS);
+        case vvdec::VVDEC_ERR_RESTART_REQUIRED: return AVERROR_BUG;
+        case vvdec::VVDEC_ERR_CPU:              return AVERROR(ENOSYS);
+        case vvdec::VVDEC_TRY_AGAIN:            return AVERROR(EAGAIN);
+        case vvdec::VVDEC_EOF:                  return AVERROR(EOF);
+        default:                                return AVERROR_UNKNOWN;
+    }
+}
+
+static int check_vret(VVDecContext* s, int vret) {
+    vvdec::VVDec* vvdec = s->vvdec;
+    if (vret && vret != vvdec::VVDEC_EOF) {
+        av_log(s->avctx, AV_LOG_ERROR, "vvdec returns error: %s\n", vvdec->getErrorMsg(vret));
+    }
+    return map_error(vret);
+}
+
+static const enum AVPixelFormat pix_fmts_8bit[] = {
+    AV_PIX_FMT_GRAY8, AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P
+};
+
+static const enum AVPixelFormat pix_fmts_10bit[] = {
+    AV_PIX_FMT_GRAY10, AV_PIX_FMT_YUV420P10,
+    AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10
+};
+
+static AVPixelFormat get_format(const vvdec::Frame* frame)
+{
+    if (frame->m_eColorFormat != vvdec::VVC_CF_INVALID) {
+        switch (frame->m_uiBitDepth) {
+            case 8:
+                return pix_fmts_8bit[frame->m_eColorFormat];
+            case 10:
+                return pix_fmts_10bit[frame->m_eColorFormat];
+        }
+    }
+    return AV_PIX_FMT_NONE;
+}
+
+static int set_pix_fmt(VVDecContext* s, const vvdec::Frame* frame)
+{
+    AVPixelFormat format = get_format(frame);
+    if (format == AV_PIX_FMT_NONE) {
+        av_log(s->avctx, AV_LOG_ERROR,
+               "unsupported, depth = %d, color format = %d.\n", frame->m_uiBitDepth, frame->m_eColorFormat);
+        return AVERROR_INVALIDDATA;
+    }
+    s->avctx->pix_fmt = format;
+    return 0;
+}
+
+static int copy_to_avframe(VVDecContext* s, const vvdec::Frame* src, AVFrame *dest)
+{
+    AVCodecContext* avctx = s->avctx;
+    int i, ret, width, height;
+    uint8_t *data[4];
+    int linesize[4];
+
+    ret = set_pix_fmt(s, src);
+    if (ret < 0)
+        return ret;
+    width = (int)src->m_uiWidth;
+    height = (int)src->m_uiHeight;
+    if (width != avctx->width || height != avctx->height) {
+        av_log(avctx, AV_LOG_INFO, "dimension change! %dx%d -> %dx%d\n",
+                avctx->width, avctx->height, width, height);
+        if ((ret = ff_set_dimensions(avctx, width, height)) < 0)
+            return ret;
+    }
+    if ((ret = ff_get_buffer(avctx, dest, 0)) < 0)
+        return ret;
+
+    for (i = 0; i < 3; i++) {
+        const vvdec::Component& plane = src->m_cComponent[i];
+        data[i] = plane.m_pucBuffer;
+        linesize[i] = plane.m_iStride;
+    }
+    data[3] = 0; linesize[3] = 0;
+
+    //TODO: conformance window?
+    av_image_copy(dest->data, dest->linesize, (const uint8_t **)data, linesize,
+                avctx->pix_fmt, width, height);
+    return 0;
+}
+
+static int vvdec_decode(AVCodecContext *avctx, void *data, int *got_frame,
+                      AVPacket *avpkt)
+{
+    VVDecContext *s = (VVDecContext *)avctx->priv_data;
+    vvdec::VVDec* vvdec = s->vvdec;
+    AVFrame *picture      = (AVFrame*)data;
+    int vret, ret = 0;
+    vvdec::Frame* frame = NULL;
+
+    if (!avpkt->size) {
+        vret = vvdec->flush(&frame);
+        if ((ret = check_vret(s, vret)) < 0)  {
+            goto error_out;
+        }
+    } else {
+        uint8_t *new_extradata;
+        int new_extradata_size;
+        new_extradata = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA,
+                                            &new_extradata_size);
+        if (new_extradata && new_extradata_size > 0) {
+            return AVERROR_PATCHWELCOME;
+        }
+
+        vvdec::AccessUnit au;
+        au.m_pucBuffer = (unsigned char*)avpkt->data;
+        au.m_iUsedSize  = avpkt->size;
+        au.m_uiCts = avpkt->pts;
+        au.m_bCtsValid = true;
+        vret = vvdec->decode(au, &frame);
+        if (vret && vret != vvdec::VVDEC_TRY_AGAIN) {
+            if ((ret = check_vret(s, vret)) < 0)  {
+                goto error_out;
+            }
+        }
+
+    }
+    if (frame) {
+        ret = copy_to_avframe(s, frame, picture);
+        if (ret < 0)
+            goto error_out;
+        *got_frame = 1;
+    }
+    return 0;
+error_out:
+    if (frame) {
+        vvdec->objectUnref(frame);
+    }
+    return ret;
+}
+
+static av_cold int vvdec_free(AVCodecContext *avctx)
+{
+    VVDecContext *ctx           = (VVDecContext *)avctx->priv_data;
+    vvdec::VVDec* vvdec = ctx->vvdec;
+    if (vvdec) {
+        vvdec->uninit();
+        delete vvdec;
+    }
+    return 0;
+}
+
+static av_cold int vvdec_init(AVCodecContext *avctx)
+{
+    VVDecContext *s = (VVDecContext *)avctx->priv_data;
+    vvdec::VVDecParameter param;
+    vvdec::VVDec* vvdec = new vvdec::VVDec;
+    int vret, ret = 0;
+
+    av_log(avctx, AV_LOG_INFO, "VVDec version: %s\n", vvdec->getVersionNumber());
+
+    s->avctx = avctx;
+    s->vvdec = vvdec;
+    vret = vvdec->init(param);
+    if ((ret = check_vret(s, vret)) < 0) {
+        delete vvdec;
+        s->vvdec = NULL;
+    }
+    return ret;
+}
+
+extern "C" {
+
+AVCodec ff_libvvdec_decoder = {
+    .name           = "libvvdec",
+    .long_name      = NULL_IF_CONFIG_SMALL("Fraunhofer Versatile Video Decoder (VVdeC)"),
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_VVC,
+    .capabilities   = AV_CODEC_CAP_AUTO_THREADS | AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
+    .profiles       = NULL_IF_CONFIG_SMALL(ff_vvc_profiles),
+    .wrapper_name   = "libvvdec",
+    .priv_data_size = sizeof(VVDecContext),
+    .init           = vvdec_init,
+    .decode         = vvdec_decode,
+    .close          = vvdec_free,
+};
+
+}
-- 
2.25.1



More information about the ffmpeg-devel mailing list