[FFmpeg-devel] [PATCH] avutil/hwcontext_videotoolbox: implement hwupload to convert AVFrame to CVPixelBuffer
Aman Karmani
ffmpeg at tmm1.net
Wed Jun 30 01:08:16 EEST 2021
From: Aman Karmani <aman at tmm1.net>
Teach AV_HWDEVICE_TYPE_VIDEOTOOLBOX to be able to create AVFrames of type
AV_PIX_FMT_VIDEOTOOLBOX. This can be used to hwupload a regular AVFrame
into its CVPixelBuffer equivalent.
ffmpeg -init_hw_device videotoolbox -f lavfi -i color=black:640x480 -vf hwupload -c:v h264_videotoolbox -f null -y /dev/null
Signed-off-by: Aman Karmani <aman at tmm1.net>
---
libavutil/hwcontext_videotoolbox.c | 418 ++++++++++++++++++++++++++++-
libavutil/hwcontext_videotoolbox.h | 7 +-
2 files changed, 420 insertions(+), 5 deletions(-)
diff --git a/libavutil/hwcontext_videotoolbox.c b/libavutil/hwcontext_videotoolbox.c
index bded9873fe..58095a1fc9 100644
--- a/libavutil/hwcontext_videotoolbox.c
+++ b/libavutil/hwcontext_videotoolbox.c
@@ -24,6 +24,7 @@
#include <VideoToolbox/VideoToolbox.h>
#include "buffer.h"
+#include "buffer_internal.h"
#include "common.h"
#include "hwcontext.h"
#include "hwcontext_internal.h"
@@ -32,6 +33,10 @@
#include "pixfmt.h"
#include "pixdesc.h"
+typedef struct VTFramesContext {
+ CVPixelBufferPoolRef pool;
+} VTFramesContext;
+
static const struct {
uint32_t cv_fmt;
bool full_range;
@@ -50,6 +55,39 @@ static const struct {
#endif
};
+static const enum AVPixelFormat supported_formats[] = {
+ AV_PIX_FMT_NV12,
+ AV_PIX_FMT_YUV420P,
+ AV_PIX_FMT_UYVY422,
+ AV_PIX_FMT_P010,
+ AV_PIX_FMT_BGRA,
+};
+
+static int vt_frames_get_constraints(AVHWDeviceContext *ctx,
+ const void *hwconfig,
+ AVHWFramesConstraints *constraints)
+{
+ int i;
+
+ constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
+ sizeof(*constraints->valid_sw_formats));
+ if (!constraints->valid_sw_formats)
+ return AVERROR(ENOMEM);
+
+ for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
+ constraints->valid_sw_formats[i] = supported_formats[i];
+ constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE;
+
+ constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
+ if (!constraints->valid_hw_formats)
+ return AVERROR(ENOMEM);
+
+ constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX;
+ constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
+
+ return 0;
+}
+
enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
{
int i;
@@ -75,11 +113,134 @@ uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo
return 0;
}
+static int vt_pool_alloc(AVHWFramesContext *ctx)
+{
+ VTFramesContext *fctx = ctx->internal->priv;
+ CVReturn err;
+ CFNumberRef w, h, pixfmt;
+ uint32_t cv_pixfmt;
+ CFMutableDictionaryRef attributes, iosurface_properties;
+
+ attributes = CFDictionaryCreateMutable(
+ NULL,
+ 2,
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+
+ cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format);
+ pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt);
+ CFDictionarySetValue(
+ attributes,
+ kCVPixelBufferPixelFormatTypeKey,
+ pixfmt);
+ CFRelease(pixfmt);
+
+ iosurface_properties = CFDictionaryCreateMutable(
+ NULL,
+ 0,
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+ CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties);
+ CFRelease(iosurface_properties);
+
+ w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width);
+ h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height);
+ CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w);
+ CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h);
+ CFRelease(w);
+ CFRelease(h);
+
+ err = CVPixelBufferPoolCreate(
+ NULL,
+ NULL,
+ attributes,
+ &fctx->pool);
+ CFRelease(attributes);
+
+ if (err == kCVReturnSuccess)
+ return 0;
+
+ av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err);
+ return AVERROR_EXTERNAL;
+}
+
+static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size)
+{
+ return NULL;
+}
+
+static void vt_frames_uninit(AVHWFramesContext *ctx)
+{
+ VTFramesContext *fctx = ctx->internal->priv;
+ if (fctx->pool) {
+ CVPixelBufferPoolRelease(fctx->pool);
+ fctx->pool = NULL;
+ }
+}
+
+static int vt_frames_init(AVHWFramesContext *ctx)
+{
+ int i, ret;
+
+ for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
+ if (ctx->sw_format == supported_formats[i])
+ break;
+ }
+ if (i == FF_ARRAY_ELEMS(supported_formats)) {
+ av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
+ av_get_pix_fmt_name(ctx->sw_format));
+ return AVERROR(ENOSYS);
+ }
+
+ // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL
+ if (!ctx->pool) {
+ ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, vt_dummy_pool_alloc, NULL);
+ if (!ctx->internal->pool_internal)
+ return AVERROR(ENOMEM);
+ }
+
+ ret = vt_pool_alloc(ctx);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
+{
+ CVPixelBufferRelease((CVPixelBufferRef)data);
+}
+
static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
{
- frame->buf[0] = av_buffer_pool_get(ctx->pool);
- if (!frame->buf[0])
- return AVERROR(ENOMEM);
+ VTFramesContext *fctx = ctx->internal->priv;
+
+ if (ctx->pool && ctx->pool->size != 0) {
+ frame->buf[0] = av_buffer_pool_get(ctx->pool);
+ if (!frame->buf[0])
+ return AVERROR(ENOMEM);
+ } else {
+ CVPixelBufferRef pixbuf;
+ AVBufferRef *buf = NULL;
+ CVReturn err;
+
+ err = CVPixelBufferPoolCreatePixelBuffer(
+ NULL,
+ fctx->pool,
+ &pixbuf
+ );
+ if (err != kCVReturnSuccess) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err);
+ return AVERROR_EXTERNAL;
+ }
+
+ buf = av_buffer_create((uint8_t *)pixbuf, 1, videotoolbox_buffer_release, NULL, 0);
+ if (!buf) {
+ CVPixelBufferRelease(pixbuf);
+ return AVERROR(ENOMEM);
+ }
+ frame->buf[0] = buf;
+ }
frame->data[3] = frame->buf[0]->data;
frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
@@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap)
CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv);
}
+static int vt_pixbuf_set_par(AVHWFramesContext *hwfc,
+ CVPixelBufferRef pixbuf, const AVFrame *src)
+{
+ CFMutableDictionaryRef par = NULL;
+ CFNumberRef num = NULL, den = NULL;
+ AVRational avpar = src->sample_aspect_ratio;
+
+ if (avpar.num == 0)
+ return 0;
+
+ av_reduce(&avpar.num, &avpar.den,
+ avpar.num, avpar.den,
+ 0xFFFFFFFF);
+
+ num = CFNumberCreate(kCFAllocatorDefault,
+ kCFNumberIntType,
+ &avpar.num);
+
+ den = CFNumberCreate(kCFAllocatorDefault,
+ kCFNumberIntType,
+ &avpar.den);
+
+ par = CFDictionaryCreateMutable(kCFAllocatorDefault,
+ 2,
+ &kCFCopyStringDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+
+ if (!par || !num || !den) {
+ if (par) CFRelease(par);
+ if (num) CFRelease(num);
+ if (den) CFRelease(den);
+ return AVERROR(ENOMEM);
+ }
+
+ CFDictionarySetValue(
+ par,
+ kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
+ num);
+ CFDictionarySetValue(
+ par,
+ kCVImageBufferPixelAspectRatioVerticalSpacingKey,
+ den);
+
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferPixelAspectRatioKey,
+ par,
+ kCVAttachmentMode_ShouldPropagate
+ );
+
+ CFRelease(par);
+ CFRelease(num);
+ CFRelease(den);
+
+ return 0;
+}
+
+static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc,
+ CVPixelBufferRef pixbuf, const AVFrame *src)
+{
+ CFStringRef loc = NULL;
+
+ switch (src->chroma_location) {
+ case AVCHROMA_LOC_LEFT:
+ loc = kCVImageBufferChromaLocation_Left;
+ break;
+ case AVCHROMA_LOC_CENTER:
+ loc = kCVImageBufferChromaLocation_Center;
+ break;
+ case AVCHROMA_LOC_TOP:
+ loc = kCVImageBufferChromaLocation_Top;
+ break;
+ case AVCHROMA_LOC_BOTTOM:
+ loc = kCVImageBufferChromaLocation_Bottom;
+ break;
+ case AVCHROMA_LOC_TOPLEFT:
+ loc = kCVImageBufferChromaLocation_TopLeft;
+ break;
+ case AVCHROMA_LOC_BOTTOMLEFT:
+ loc = kCVImageBufferChromaLocation_BottomLeft;
+ break;
+ }
+
+ if (loc) {
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferChromaLocationTopFieldKey,
+ loc,
+ kCVAttachmentMode_ShouldPropagate);
+ }
+
+ return 0;
+}
+
+static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc,
+ CVPixelBufferRef pixbuf, const AVFrame *src)
+{
+ CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL;
+ Float32 gamma = 0;
+
+ switch (src->colorspace) {
+ case AVCOL_SPC_BT2020_CL:
+ case AVCOL_SPC_BT2020_NCL:
+ if (__builtin_available(macOS 10.11, *))
+ colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
+ else
+ colormatrix = CFSTR("ITU_R_2020");
+ break;
+ case AVCOL_SPC_BT470BG:
+ case AVCOL_SPC_SMPTE170M:
+ colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
+ break;
+ case AVCOL_SPC_BT709:
+ colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
+ break;
+ case AVCOL_SPC_SMPTE240M:
+ colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
+ break;
+ case AVCOL_SPC_UNSPECIFIED:
+ break;
+ default:
+ av_log(hwfc, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace));
+ }
+
+ switch (src->color_primaries) {
+ case AVCOL_PRI_BT2020:
+ if (__builtin_available(macOS 10.11, *))
+ colorpri = kCVImageBufferColorPrimaries_ITU_R_2020;
+ else
+ colorpri = CFSTR("ITU_R_2020");
+ break;
+ case AVCOL_PRI_BT709:
+ colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2;
+ break;
+ case AVCOL_PRI_SMPTE170M:
+ colorpri = kCVImageBufferColorPrimaries_SMPTE_C;
+ break;
+ case AVCOL_PRI_BT470BG:
+ colorpri = kCVImageBufferColorPrimaries_EBU_3213;
+ break;
+ case AVCOL_PRI_UNSPECIFIED:
+ break;
+ default:
+ av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries));
+ }
+
+ switch (src->color_trc) {
+ case AVCOL_TRC_SMPTE2084:
+ if (__builtin_available(macOS 10.13, *))
+ colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
+ else
+ colortrc = CFSTR("SMPTE_ST_2084_PQ");
+ break;
+ case AVCOL_TRC_BT2020_10:
+ case AVCOL_TRC_BT2020_12:
+ if (__builtin_available(macOS 10.11, *))
+ colortrc = kCVImageBufferTransferFunction_ITU_R_2020;
+ else
+ colortrc = CFSTR("ITU_R_2020");
+ break;
+ case AVCOL_TRC_BT709:
+ colortrc = kCVImageBufferTransferFunction_ITU_R_709_2;
+ break;
+ case AVCOL_TRC_SMPTE240M:
+ colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
+ break;
+ case AVCOL_TRC_SMPTE428:
+ if (__builtin_available(macOS 10.12, *))
+ colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1;
+ else
+ colortrc = CFSTR("SMPTE_ST_428_1");
+ break;
+ case AVCOL_TRC_ARIB_STD_B67:
+ if (__builtin_available(macOS 10.13, *))
+ colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
+ else
+ colortrc = CFSTR("ITU_R_2100_HLG");
+ break;
+ case AVCOL_TRC_GAMMA22:
+ gamma = 2.2;
+ colortrc = kCVImageBufferTransferFunction_UseGamma;
+ break;
+ case AVCOL_TRC_GAMMA28:
+ gamma = 2.8;
+ colortrc = kCVImageBufferTransferFunction_UseGamma;
+ break;
+ case AVCOL_TRC_UNSPECIFIED:
+ break;
+ default:
+ av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc));
+ }
+
+ if (colormatrix) {
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferYCbCrMatrixKey,
+ colormatrix,
+ kCVAttachmentMode_ShouldPropagate);
+ }
+ if (colorpri) {
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferColorPrimariesKey,
+ colorpri,
+ kCVAttachmentMode_ShouldPropagate);
+ }
+ if (colortrc) {
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferTransferFunctionKey,
+ colortrc,
+ kCVAttachmentMode_ShouldPropagate);
+ }
+ if (gamma != 0) {
+ CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
+ CVBufferSetAttachment(
+ pixbuf,
+ kCVImageBufferGammaLevelKey,
+ gamma_level,
+ kCVAttachmentMode_ShouldPropagate);
+ CFRelease(gamma_level);
+ }
+
+ return 0;
+}
+
+static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc,
+ CVPixelBufferRef pixbuf, const AVFrame *src)
+{
+ int ret;
+ ret = vt_pixbuf_set_par(hwfc, pixbuf, src);
+ if (ret < 0)
+ return ret;
+ ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src);
+ if (ret < 0)
+ return ret;
+ ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src);
+ if (ret < 0)
+ return ret;
+ return 0;
+}
+
static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src,
int flags)
{
@@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext *hwfc,
if (err)
goto fail;
+ err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src);
+ if (err)
+ goto fail;
+
err = 0;
fail:
av_frame_free(&map);
@@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = {
.type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
.name = "videotoolbox",
+ .frames_priv_size = sizeof(VTFramesContext),
+
.device_create = vt_device_create,
+ .frames_init = vt_frames_init,
.frames_get_buffer = vt_get_buffer,
+ .frames_get_constraints = vt_frames_get_constraints,
+ .frames_uninit = vt_frames_uninit,
.transfer_get_formats = vt_transfer_get_formats,
.transfer_data_to = vt_transfer_data_to,
.transfer_data_from = vt_transfer_data_from,
diff --git a/libavutil/hwcontext_videotoolbox.h b/libavutil/hwcontext_videotoolbox.h
index 5074d79e68..62cde07c51 100644
--- a/libavutil/hwcontext_videotoolbox.h
+++ b/libavutil/hwcontext_videotoolbox.h
@@ -29,11 +29,14 @@
* @file
* An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
*
- * This API currently does not support frame allocation, as the raw VideoToolbox
- * API does allocation, and FFmpeg itself never has the need to allocate frames.
+ * This API supports frame allocation using a native CVPixelBufferPool
+ * instead of an AVBufferPool.
*
* If the API user sets a custom pool, AVHWFramesContext.pool must return
* AVBufferRefs whose data pointer is a CVImageBufferRef or CVPixelBufferRef.
+ * Note that the underlying CVPixelBuffer could be retained by OS frameworks
+ * depending on application usage, so it is preferable to let CoreVideo manage
+ * the pool using the default implementation.
*
* Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are always
* NULL.
--
2.29.2
More information about the ffmpeg-devel
mailing list