[FFmpeg-devel] Avoid a decoded frame copy in mmaldec for raspberrypi
Lluís Batlle i Rossell
viric at viric.name
Thu Feb 11 23:10:55 EET 2021
Hello,
I needed to acquire mjpeg from a v4l2 uvc webcam in a Raspberry pi,
and I saw mmaldec lacked mjpeg and also was doing an unnecessary copy.
I used two patches I found in the mailing list archives to enable mjpeg
mmaldec and then wrote one that avoids the copy. It works fine for me but
I have not run 'fate' yet. I'm in a cross-building situation and I have
yet to learn how that works.
In a Raspberry Pi 1 Model B now it can 640x480 30fps mjpeg->h264_omx with
25% of cpu (as "top" shows).
Before these three patches, 640x480 YUY2->h264_omx could do only 20fps and
"top" showed 50% of cpu.
A Raspberry Pi 2 can do easily 30fps 1280x720 mjpeg->h264_omx as well.
It'd be a lot easier for me if these patches were upstream so I'm
interested in the code getting in. I'm new in ffmpeg so I may have missed
customary details. I also thank the help #ffmpeg-devel that made my patch
(3rd) simpler than I originally thought.
Regards,
Lluís.
--
(Escriu-me xifrat si saps PGP / Write ciphered if you know PGP)
PGP key 7CBD1DA5 - https://emailselfdefense.fsf.org/
-------------- next part --------------
>From ac005288e1b8493dcf7457171820dcf1a2fa03ca Mon Sep 17 00:00:00 2001
From: Cosmin Gorgovan <cosmin at linux-geek.org>
Date: Thu, 20 Aug 2020 21:05:23 +0100
Subject: [PATCH 1/3] libavcodec/mmaldec: enable MJPEG decoding
---
configure | 1 +
libavcodec/allcodecs.c | 1 +
libavcodec/mmaldec.c | 4 ++++
3 files changed, 6 insertions(+)
diff --git a/configure b/configure
index a76c2ec4ae..048bedb589 100755
--- a/configure
+++ b/configure
@@ -3105,6 +3105,7 @@ hevc_v4l2m2m_decoder_deps="v4l2_m2m hevc_v4l2_m2m"
hevc_v4l2m2m_decoder_select="hevc_mp4toannexb_bsf"
hevc_v4l2m2m_encoder_deps="v4l2_m2m hevc_v4l2_m2m"
mjpeg_cuvid_decoder_deps="cuvid"
+mjpeg_mmal_decoder_deps="mmal"
mjpeg_qsv_decoder_select="qsvdec"
mjpeg_qsv_encoder_deps="libmfx"
mjpeg_qsv_encoder_select="qsvenc"
diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
index 16ec182a52..8d1908c6d5 100644
--- a/libavcodec/allcodecs.c
+++ b/libavcodec/allcodecs.c
@@ -187,6 +187,7 @@ extern AVCodec ff_mdec_decoder;
extern AVCodec ff_mimic_decoder;
extern AVCodec ff_mjpeg_encoder;
extern AVCodec ff_mjpeg_decoder;
+extern AVCodec ff_mjpeg_mmal_decoder;
extern AVCodec ff_mjpegb_decoder;
extern AVCodec ff_mmvideo_decoder;
extern AVCodec ff_mobiclip_decoder;
diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index cb15ac072a..df14b9fc95 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -375,6 +375,9 @@ static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
format_in = decoder->input[0]->format;
format_in->type = MMAL_ES_TYPE_VIDEO;
switch (avctx->codec_id) {
+ case AV_CODEC_ID_MJPEG:
+ format_in->encoding = MMAL_ENCODING_MJPEG;
+ break;
case AV_CODEC_ID_MPEG2VIDEO:
format_in->encoding = MMAL_ENCODING_MP2V;
break;
@@ -851,6 +854,7 @@ static const AVOption options[]={
};
FFMMAL_DEC(h264, AV_CODEC_ID_H264)
+FFMMAL_DEC(mjpeg, AV_CODEC_ID_MJPEG)
FFMMAL_DEC(mpeg2, AV_CODEC_ID_MPEG2VIDEO)
FFMMAL_DEC(mpeg4, AV_CODEC_ID_MPEG4)
FFMMAL_DEC(vc1, AV_CODEC_ID_VC1)
--
2.29.2
-------------- next part --------------
>From 62a2f1e7194405e004de9c2f0386b33c4b07446e Mon Sep 17 00:00:00 2001
From: Cosmin Gorgovan <cosmin at linux-geek.org>
Date: Thu, 20 Aug 2020 21:11:49 +0100
Subject: [PATCH 2/3] libavcodec/mmaldec: continue after receiving EOS without
having sent one
The previous logic in mmaldec was causing the MMAL MJPEG decoder
to stop if it received an invalid frame - which happened to be the
first frame received from a UVC camera via V4L2 in my application
---
libavcodec/mmaldec.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index df14b9fc95..4dfaacbb41 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -711,9 +711,13 @@ static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_fra
goto done;
}
- ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
- if (ctx->eos_received)
+ int eos = !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
+ if (eos) {
+ if (ctx->eos_sent) {
+ ctx->eos_received = 1;
+ }
goto done;
+ }
if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
MMAL_COMPONENT_T *decoder = ctx->decoder;
--
2.29.2
-------------- next part --------------
>From 6ae87ea2012444ff52dcff36718d27fb9262b14f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Llu=C3=ADs=20Batlle=20i=20Rossell?= <viric at viric.name>
Date: Wed, 10 Feb 2021 22:58:17 +0100
Subject: [PATCH 3/3] mmaldec with plain yuv420p without copy
---
libavcodec/mmaldec.c | 48 ++++++++++++++++++++------------------------
1 file changed, 22 insertions(+), 26 deletions(-)
diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index 4dfaacbb41..097b990f92 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -119,10 +119,11 @@ static void ffmmal_release_frame(void *opaque, uint8_t *data)
// Setup frame with a new reference to buffer. The buffer must have been
// allocated from the given pool.
-static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool,
- MMAL_BUFFER_HEADER_T *buffer)
+static int ffmmal_set_ref(AVCodecContext *avctx, AVFrame *frame,
+ FFPoolRef *pool, MMAL_BUFFER_HEADER_T *buffer)
{
FFBufferRef *ref = av_mallocz(sizeof(*ref));
+
if (!ref)
return AVERROR(ENOMEM);
@@ -140,8 +141,19 @@ static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool,
atomic_fetch_add_explicit(&ref->pool->refcount, 1, memory_order_relaxed);
mmal_buffer_header_acquire(buffer);
- frame->format = AV_PIX_FMT_MMAL;
- frame->data[3] = (uint8_t *)ref->buffer;
+ frame->format = avctx->pix_fmt;
+
+ if (avctx->pix_fmt == AV_PIX_FMT_YUV420P) {
+ int w = FFALIGN(avctx->width, 32);
+ int h = FFALIGN(avctx->height, 16);
+
+ av_image_fill_arrays(frame->data, frame->linesize,
+ buffer->data + buffer->type->video.offset[0],
+ avctx->pix_fmt, w, h, 1);
+ } else {
+ frame->data[3] = (uint8_t *)ref->buffer;
+ }
+
return 0;
}
@@ -633,30 +645,14 @@ static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame,
frame->interlaced_frame = ctx->interlaced_frame;
frame->top_field_first = ctx->top_field_first;
- if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
- if (!ctx->pool_out)
- return AVERROR_UNKNOWN; // format change code failed with OOM previously
-
- if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
- goto done;
-
- if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0)
- goto done;
- } else {
- int w = FFALIGN(avctx->width, 32);
- int h = FFALIGN(avctx->height, 16);
- uint8_t *src[4];
- int linesize[4];
+ if (!ctx->pool_out)
+ return AVERROR_UNKNOWN; // format change code failed with OOM previously
- if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
- goto done;
+ if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
+ goto done;
- av_image_fill_arrays(src, linesize,
- buffer->data + buffer->type->video.offset[0],
- avctx->pix_fmt, w, h, 1);
- av_image_copy(frame->data, frame->linesize, src, linesize,
- avctx->pix_fmt, avctx->width, avctx->height);
- }
+ if ((ret = ffmmal_set_ref(avctx, frame, ctx->pool_out, buffer)) < 0)
+ goto done;
frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
#if FF_API_PKT_PTS
--
2.29.2
More information about the ffmpeg-devel
mailing list