[FFmpeg-cvslog] avcodec: add LCEVC decoding support via LCEVCdec

James Almer git at videolan.org
Mon Sep 23 16:21:19 EEST 2024


ffmpeg | branch: master | James Almer <jamrial at gmail.com> | Tue Apr  2 16:33:02 2024 -0300| [660e7e6a0ef0981c968e95d192b1ef094dbc53ad] | committer: James Almer

avcodec: add LCEVC decoding support via LCEVCdec

Signed-off-by: James Almer <jamrial at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=660e7e6a0ef0981c968e95d192b1ef094dbc53ad
---

 configure                     |   2 +-
 libavcodec/Makefile           |   1 +
 libavcodec/avcodec.c          |   2 +
 libavcodec/avcodec_internal.h |   4 +
 libavcodec/decode.c           |  68 ++++++++-
 libavcodec/lcevcdec.c         | 319 ++++++++++++++++++++++++++++++++++++++++++
 libavcodec/lcevcdec.h         |  42 ++++++
 libavcodec/pthread_frame.c    |   3 +
 8 files changed, 439 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index f07f6c7c46..09f844a608 100755
--- a/configure
+++ b/configure
@@ -4034,7 +4034,7 @@ cws2fws_extralibs="zlib_extralibs"
 
 # libraries, in any order
 avcodec_deps="avutil"
-avcodec_suggest="libm stdatomic"
+avcodec_suggest="libm stdatomic liblcevc_dec"
 avdevice_deps="avformat avcodec avutil"
 avdevice_suggest="libm stdatomic"
 avfilter_deps="avutil"
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 936fc3415a..a4fcce3b42 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -46,6 +46,7 @@ OBJS = ac3_parser.o                                                     \
        get_buffer.o                                                     \
        imgconvert.o                                                     \
        jni.o                                                            \
+       lcevcdec.o                                                       \
        mathtables.o                                                     \
        mediacodec.o                                                     \
        mpeg12framerate.o                                                \
diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index 8d1a280323..d1daf47611 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -446,6 +446,8 @@ av_cold void ff_codec_close(AVCodecContext *avctx)
 
         ff_refstruct_unref(&avci->pool);
         ff_refstruct_pool_uninit(&avci->progress_frame_pool);
+        if (av_codec_is_decoder(avctx->codec))
+            ff_decode_internal_uninit(avctx);
 
         ff_hwaccel_uninit(avctx);
 
diff --git a/libavcodec/avcodec_internal.h b/libavcodec/avcodec_internal.h
index 31745b89b1..184d7b526c 100644
--- a/libavcodec/avcodec_internal.h
+++ b/libavcodec/avcodec_internal.h
@@ -68,6 +68,10 @@ void ff_decode_flush_buffers(struct AVCodecContext *avctx);
 void ff_encode_flush_buffers(struct AVCodecContext *avctx);
 
 struct AVCodecInternal *ff_decode_internal_alloc(void);
+void ff_decode_internal_sync(struct AVCodecContext *dst,
+                             const struct AVCodecContext *src);
+void ff_decode_internal_uninit(struct AVCodecContext *avctx);
+
 struct AVCodecInternal *ff_encode_internal_alloc(void);
 
 void ff_codec_close(struct AVCodecContext *avctx);
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index e4e92e34e4..fb642799b1 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -48,6 +48,7 @@
 #include "hwaccel_internal.h"
 #include "hwconfig.h"
 #include "internal.h"
+#include "lcevcdec.h"
 #include "packet_internal.h"
 #include "progressframe.h"
 #include "refstruct.h"
@@ -89,6 +90,11 @@ typedef struct DecodeContext {
      * (global or attached to packets) side data over bytestream.
      */
     uint64_t side_data_pref_mask;
+
+    FFLCEVCContext *lcevc;
+    int lcevc_frame;
+    int width;
+    int height;
 } DecodeContext;
 
 static DecodeContext *decode_ctx(AVCodecInternal *avci)
@@ -1597,6 +1603,40 @@ int ff_attach_decode_data(AVFrame *frame)
     return 0;
 }
 
+static void update_frame_props(AVCodecContext *avctx, AVFrame *frame)
+{
+    AVCodecInternal    *avci = avctx->internal;
+    DecodeContext        *dc = decode_ctx(avci);
+
+    dc->lcevc_frame = dc->lcevc && avctx->codec_type == AVMEDIA_TYPE_VIDEO &&
+                      av_frame_get_side_data(frame, AV_FRAME_DATA_LCEVC);
+
+    if (dc->lcevc_frame) {
+        dc->width     = frame->width;
+        dc->height    = frame->height;
+        frame->width  = frame->width  * 2 / FFMAX(frame->sample_aspect_ratio.den, 1);
+        frame->height = frame->height * 2 / FFMAX(frame->sample_aspect_ratio.num, 1);
+    }
+}
+
+static void attach_post_process_data(AVCodecContext *avctx, AVFrame *frame)
+{
+    AVCodecInternal    *avci = avctx->internal;
+    DecodeContext        *dc = decode_ctx(avci);
+
+    if (dc->lcevc_frame) {
+        FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
+
+        fdd->post_process_opaque = ff_refstruct_ref(dc->lcevc);
+        fdd->post_process_opaque_free = ff_lcevc_unref;
+        fdd->post_process = ff_lcevc_process;
+
+        frame->width  = dc->width;
+        frame->height = dc->height;
+    }
+    dc->lcevc_frame = 0;
+}
+
 int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
 {
     const FFHWAccel *hwaccel = ffhwaccel(avctx->hwaccel);
@@ -1640,8 +1680,10 @@ int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
             ret = hwaccel->alloc_frame(avctx, frame);
             goto end;
         }
-    } else
+    } else {
         avctx->sw_pix_fmt = avctx->pix_fmt;
+        update_frame_props(avctx, frame);
+    }
 
     ret = avctx->get_buffer2(avctx, frame, flags);
     if (ret < 0)
@@ -1653,6 +1695,8 @@ int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
     if (ret < 0)
         goto fail;
 
+    attach_post_process_data(avctx, frame);
+
 end:
     if (avctx->codec_type == AVMEDIA_TYPE_VIDEO && !override_dimensions &&
         !(ffcodec(avctx->codec)->caps_internal & FF_CODEC_CAP_EXPORTS_CROPPING)) {
@@ -1953,6 +1997,12 @@ int ff_decode_preinit(AVCodecContext *avctx)
     if (ret < 0)
         return ret;
 
+    if (!(avctx->export_side_data & AV_CODEC_EXPORT_DATA_ENHANCEMENTS)) {
+        ret = ff_lcevc_alloc(&dc->lcevc);
+        if (ret < 0 && (avctx->err_recognition & AV_EF_EXPLODE))
+            return ret;
+    }
+
 #if FF_API_DROPCHANGED
     if (avctx->flags & AV_CODEC_FLAG_DROPCHANGED)
         av_log(avctx, AV_LOG_WARNING, "The dropchanged flag is deprecated.\n");
@@ -2187,3 +2237,19 @@ AVCodecInternal *ff_decode_internal_alloc(void)
 {
     return av_mallocz(sizeof(DecodeContext));
 }
+
+void ff_decode_internal_sync(AVCodecContext *dst, const AVCodecContext *src)
+{
+    const DecodeContext *src_dc = decode_ctx(src->internal);
+    DecodeContext *dst_dc = decode_ctx(dst->internal);
+
+    ff_refstruct_replace(&dst_dc->lcevc, src_dc->lcevc);
+}
+
+void ff_decode_internal_uninit(AVCodecContext *avctx)
+{
+    AVCodecInternal *avci = avctx->internal;
+    DecodeContext *dc = decode_ctx(avci);
+
+    ff_refstruct_unref(&dc->lcevc);
+}
diff --git a/libavcodec/lcevcdec.c b/libavcodec/lcevcdec.c
new file mode 100644
index 0000000000..ceeece3aa9
--- /dev/null
+++ b/libavcodec/lcevcdec.c
@@ -0,0 +1,319 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config_components.h"
+
+#include "libavutil/avassert.h"
+#include "libavutil/frame.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/log.h"
+#include "libavutil/mem.h"
+#include "decode.h"
+#include "lcevcdec.h"
+
+#if CONFIG_LIBLCEVC_DEC
+static LCEVC_ColorFormat map_format(int format)
+{
+    switch (format) {
+    case AV_PIX_FMT_YUV420P:
+        return LCEVC_I420_8;
+    case AV_PIX_FMT_YUV420P10:
+        return LCEVC_I420_10_LE;
+    case AV_PIX_FMT_NV12:
+        return LCEVC_NV12_8;
+    case AV_PIX_FMT_NV21:
+        return LCEVC_NV21_8;
+    case AV_PIX_FMT_GRAY8:
+        return LCEVC_GRAY_8;
+    }
+
+    return LCEVC_ColorFormat_Unknown;
+}
+
+static int alloc_base_frame(void *logctx, LCEVC_DecoderHandle decoder,
+                            const AVFrame *frame, LCEVC_PictureHandle *picture)
+{
+    LCEVC_PictureDesc desc;
+    LCEVC_ColorFormat fmt = map_format(frame->format);
+    LCEVC_PictureLockHandle lock;
+    uint8_t *data[4] = { NULL };
+    int linesizes[4] = { 0 };
+    uint32_t planes;
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_DefaultPictureDesc(&desc, fmt, frame->width, frame->height);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    desc.cropTop    = frame->crop_top;
+    desc.cropBottom = frame->crop_bottom;
+    desc.cropLeft   = frame->crop_left;
+    desc.cropRight  = frame->crop_right;
+    desc.sampleAspectRatioNum  = frame->sample_aspect_ratio.num;
+    desc.sampleAspectRatioDen  = frame->sample_aspect_ratio.den;
+
+    /* Allocate LCEVC Picture */
+    res = LCEVC_AllocPicture(decoder, &desc, picture);
+    if (res != LCEVC_Success) {
+        return AVERROR_EXTERNAL;
+    }
+    res = LCEVC_LockPicture(decoder, *picture, LCEVC_Access_Write, &lock);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    res = LCEVC_GetPicturePlaneCount(decoder, *picture, &planes);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    for (unsigned i = 0; i < planes; i++) {
+        LCEVC_PicturePlaneDesc plane;
+
+        res = LCEVC_GetPictureLockPlaneDesc(decoder, lock, i, &plane);
+        if (res != LCEVC_Success)
+            return AVERROR_EXTERNAL;
+
+        data[i] = plane.firstSample;
+        linesizes[i] = plane.rowByteStride;
+    }
+
+    av_image_copy2(data, linesizes, frame->data, frame->linesize,
+                   frame->format, frame->width, frame->height);
+
+    res = LCEVC_UnlockPicture(decoder, lock);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    return 0;
+}
+
+static int alloc_enhanced_frame(void *logctx, LCEVC_DecoderHandle decoder,
+                                const AVFrame *frame, LCEVC_PictureHandle *picture)
+{
+    LCEVC_PictureDesc desc ;
+    LCEVC_ColorFormat fmt = map_format(frame->format);
+    LCEVC_PicturePlaneDesc planes[4] = { 0 };
+    int width = frame->width * 2 / FFMAX(frame->sample_aspect_ratio.den, 1);
+    int height = frame->height * 2 / FFMAX(frame->sample_aspect_ratio.num, 1);
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_DefaultPictureDesc(&desc, fmt, width, height);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    /* Set plane description */
+    for (int i = 0; i < 4; i++) {
+        planes[i].firstSample = frame->data[i];
+        planes[i].rowByteStride = frame->linesize[i];
+    }
+
+    /* Allocate LCEVC Picture */
+    res = LCEVC_AllocPictureExternal(decoder, &desc, NULL, planes, picture);
+    if (res != LCEVC_Success) {
+        return AVERROR_EXTERNAL;
+    }
+    return 0;
+}
+
+static int lcevc_send_frame(void *logctx, FFLCEVCContext *lcevc, const AVFrame *in)
+{
+    const AVFrameSideData *sd = av_frame_get_side_data(in, AV_FRAME_DATA_LCEVC);
+    LCEVC_PictureHandle picture;
+    LCEVC_ReturnCode res;
+    int ret = 0;
+
+    if (!sd)
+        return 1;
+
+    res = LCEVC_SendDecoderEnhancementData(lcevc->decoder, in->pts, 0, sd->data, sd->size);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    ret = alloc_base_frame(logctx, lcevc->decoder, in, &picture);
+    if (ret < 0)
+        return ret;
+
+    res = LCEVC_SendDecoderBase(lcevc->decoder, in->pts, 0, picture, -1, NULL);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    memset(&picture, 0, sizeof(picture));
+    ret = alloc_enhanced_frame(logctx, lcevc->decoder, in, &picture);
+    if (ret < 0)
+        return ret;
+
+    res = LCEVC_SendDecoderPicture(lcevc->decoder, picture);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    return 0;
+}
+
+static int generate_output(void *logctx, FFLCEVCContext *lcevc, AVFrame *out)
+{
+    LCEVC_PictureDesc desc;
+    LCEVC_DecodeInformation info;
+    LCEVC_PictureHandle picture;
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_ReceiveDecoderPicture(lcevc->decoder, &picture, &info);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    res = LCEVC_GetPictureDesc(lcevc->decoder, picture, &desc);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    out->crop_top = desc.cropTop;
+    out->crop_bottom = desc.cropBottom;
+    out->crop_left = desc.cropLeft;
+    out->crop_right = desc.cropRight;
+    out->sample_aspect_ratio.num = desc.sampleAspectRatioNum;
+    out->sample_aspect_ratio.den = desc.sampleAspectRatioDen;
+    out->width = desc.width + out->crop_left + out->crop_right;
+    out->height = desc.height + out->crop_top + out->crop_bottom;
+
+    res = LCEVC_FreePicture(lcevc->decoder, picture);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    return 0;
+}
+
+static int lcevc_receive_frame(void *logctx, FFLCEVCContext *lcevc, AVFrame *out)
+{
+    LCEVC_PictureHandle picture;
+    LCEVC_ReturnCode res;
+    int ret;
+
+    ret = generate_output(logctx, lcevc, out);
+    if (ret < 0)
+        return ret;
+
+    while (1) {
+        res = LCEVC_ReceiveDecoderBase (lcevc->decoder, &picture);
+        if (res != LCEVC_Success && res != LCEVC_Again)
+            return AVERROR_EXTERNAL;
+
+        if (res == LCEVC_Again)
+            break;
+
+        res = LCEVC_FreePicture(lcevc->decoder, picture);
+        if (res != LCEVC_Success)
+            return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+static void event_callback(LCEVC_DecoderHandle dec, LCEVC_Event event,
+    LCEVC_PictureHandle pic, const LCEVC_DecodeInformation *info,
+    const uint8_t *data, uint32_t size, void *logctx)
+{
+    switch (event) {
+    case LCEVC_Log:
+        av_log(logctx, AV_LOG_INFO, "%s\n", data);
+        break;
+    default:
+        break;
+    }
+}
+
+static void lcevc_free(FFRefStructOpaque unused, void *obj)
+{
+    FFLCEVCContext *lcevc = obj;
+    if (lcevc->initialized)
+        LCEVC_DestroyDecoder(lcevc->decoder);
+    memset(lcevc, 0, sizeof(*lcevc));
+}
+#endif
+
+static int lcevc_init(FFLCEVCContext *lcevc, void *logctx)
+{
+#if CONFIG_LIBLCEVC_DEC
+    LCEVC_AccelContextHandle dummy = { 0 };
+    const int32_t event = LCEVC_Log;
+#endif
+
+    if (lcevc->initialized)
+        return 0;
+
+#if CONFIG_LIBLCEVC_DEC
+    if (LCEVC_CreateDecoder(&lcevc->decoder, dummy) != LCEVC_Success) {
+        av_log(logctx, AV_LOG_ERROR, "Failed to create LCEVC decoder\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    LCEVC_ConfigureDecoderInt(lcevc->decoder, "log_level", 4);
+    LCEVC_ConfigureDecoderIntArray(lcevc->decoder, "events", 1, &event);
+    LCEVC_SetDecoderEventCallback(lcevc->decoder, event_callback, logctx);
+
+    if (LCEVC_InitializeDecoder(lcevc->decoder) != LCEVC_Success) {
+        av_log(logctx, AV_LOG_ERROR, "Failed to initialize LCEVC decoder\n");
+        LCEVC_DestroyDecoder(lcevc->decoder);
+        return AVERROR_EXTERNAL;
+    }
+
+#endif
+    lcevc->initialized = 1;
+
+    return 0;
+}
+
+int ff_lcevc_process(void *logctx, AVFrame *frame)
+{
+    FrameDecodeData  *fdd = (FrameDecodeData*)frame->private_ref->data;
+    FFLCEVCContext *lcevc = fdd->post_process_opaque;
+    int ret;
+
+    if (!lcevc->initialized) {
+        ret = lcevc_init(lcevc, logctx);
+        if (ret < 0)
+            return ret;
+    }
+
+#if CONFIG_LIBLCEVC_DEC
+    ret = lcevc_send_frame(logctx, lcevc, frame);
+    if (ret)
+        return ret < 0 ? ret : 0;
+
+    lcevc_receive_frame(logctx, lcevc, frame);
+    if (ret < 0)
+        return ret;
+
+    av_frame_remove_side_data(frame, AV_FRAME_DATA_LCEVC);
+#endif
+
+    return 0;
+}
+
+int ff_lcevc_alloc(FFLCEVCContext **plcevc)
+{
+    FFLCEVCContext *lcevc = NULL;
+#if CONFIG_LIBLCEVC_DEC
+    lcevc = ff_refstruct_alloc_ext(sizeof(*lcevc), 0, NULL, lcevc_free);
+    if (!lcevc)
+        return AVERROR(ENOMEM);
+#endif
+    *plcevc = lcevc;
+    return 0;
+}
+
+void ff_lcevc_unref(void *opaque)
+{
+    ff_refstruct_unref(&opaque);
+}
diff --git a/libavcodec/lcevcdec.h b/libavcodec/lcevcdec.h
new file mode 100644
index 0000000000..7334d3a645
--- /dev/null
+++ b/libavcodec/lcevcdec.h
@@ -0,0 +1,42 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_LCEVCDEC_H
+#define AVCODEC_LCEVCDEC_H
+
+#include "config_components.h"
+
+#include <stdint.h>
+#if CONFIG_LIBLCEVC_DEC
+#include <LCEVC/lcevc_dec.h>
+#else
+typedef uintptr_t LCEVC_DecoderHandle;
+#endif
+#include "refstruct.h"
+
+typedef struct FFLCEVCContext {
+    LCEVC_DecoderHandle decoder;
+    int initialized;
+} FFLCEVCContext;
+
+struct AVFrame;
+
+int ff_lcevc_alloc(FFLCEVCContext **plcevc);
+int ff_lcevc_process(void *logctx, struct AVFrame *frame);
+void ff_lcevc_unref(void *opaque);
+#endif /* AVCODEC_LCEVCDEC_H */
diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index 019e33b7b2..1b1b96623f 100644
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -406,6 +406,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
         dst->hwaccel_flags = src->hwaccel_flags;
 
         ff_refstruct_replace(&dst->internal->pool, src->internal->pool);
+        ff_decode_internal_sync(dst, src);
     }
 
     if (for_user) {
@@ -782,6 +783,7 @@ void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
             ff_refstruct_unref(&ctx->internal->pool);
             av_packet_free(&ctx->internal->in_pkt);
             av_packet_free(&ctx->internal->last_pkt_props);
+            ff_decode_internal_uninit(ctx);
             av_freep(&ctx->internal);
             av_buffer_unref(&ctx->hw_frames_ctx);
             av_frame_side_data_free(&ctx->decoded_side_data,
@@ -845,6 +847,7 @@ static av_cold int init_thread(PerThreadContext *p, int *threads_to_free,
     copy->internal = ff_decode_internal_alloc();
     if (!copy->internal)
         return AVERROR(ENOMEM);
+    ff_decode_internal_sync(copy, avctx);
     copy->internal->thread_ctx = p;
     copy->internal->progress_frame_pool = avctx->internal->progress_frame_pool;
 



More information about the ffmpeg-cvslog mailing list