[FFmpeg-devel] [PATCH RFC v2 3/3] libavutil: Add thumbnail output to vaapi_h264 decoder
Zachary Zhou
zachary.zhou at intel.com
Mon Apr 8 11:53:34 EEST 2019
This is sample code for reference
HW support for decode+scaling in a single HW command (VDBOX+SFC).
The primary target usage is video analytics, but can be used playback,
transcoding, etc.
For VAAPI -
https://github.com/intel/libva
basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h).
VAAPI sample code -
https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
---
libavutil/frame.c | 1 +
libavutil/frame.h | 13 +++++++
libavutil/hwcontext_vaapi.c | 73 ++++++++++++++++++++++++++++++++++++-
libavutil/hwcontext_vaapi.h | 6 +++
4 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/libavutil/frame.c b/libavutil/frame.c
index dcf1fc3d17..14bcb456c9 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type)
#endif
case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic Metadata SMPTE2094-40 (HDR10+)";
case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest";
+ case AV_FRAME_DATA_THUMBNAIL_INFO: return "Thumbnail Information";
}
return NULL;
}
diff --git a/libavutil/frame.h b/libavutil/frame.h
index 8aa3e88367..2f3ac0cb23 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -179,6 +179,11 @@ enum AVFrameSideDataType {
* array element is implied by AVFrameSideData.size / AVRegionOfInterest.self_size.
*/
AV_FRAME_DATA_REGIONS_OF_INTEREST,
+
+ /**
+ * VDENC Thumbnail information.
+ */
+ AV_FRAME_DATA_THUMBNAIL_INFO,
};
enum AVActiveFormatDescription {
@@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest {
AVRational qoffset;
} AVRegionOfInterest;
+/**
+ * Structure to hold Information of Thumbnail.
+ *
+ */
+typedef struct AVThumbnailInfo {
+ uint8_t *data;
+} AVThumbnailInfo;
+
/**
* This structure describes decoded (raw) audio or video data.
*
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 8624369bb9..cacd352ebb 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -642,8 +642,74 @@ static void vaapi_frames_uninit(AVHWFramesContext *hwfc)
av_freep(&ctx->attributes);
}
+static int vaapi_get_thumbnail_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+ AVVAAPIFramesContext *avfc = hwfc->hwctx;
+ AVFrameSideData *frame_thumbnail_sd = av_frame_get_side_data(frame,
+ AV_FRAME_DATA_THUMBNAIL_INFO);
+ if (frame_thumbnail_sd) {
+ AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
+ VASurfaceID thumbnail_surface_id;
+ VASurfaceAttrib attrib;
+ unsigned int rt_format;
+ AVBufferRef *ref;
+ VAStatus vas;
+
+ memset(&attrib, 0, sizeof(attrib));
+ attrib.type = VASurfaceAttribPixelFormat;
+ attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
+ attrib.value.type = VAGenericValueTypeInteger;
+
+ switch (avfc->thumbnail_format) {
+ case AV_PIX_FMT_NV12:
+ attrib.value.value.i = VA_FOURCC_NV12;
+ rt_format = VA_RT_FORMAT_YUV420;
+ break;
+ case AV_PIX_FMT_ARGB:
+ attrib.value.value.i = VA_FOURCC_ARGB;
+ rt_format = VA_RT_FORMAT_RGB32;
+ break;
+ default:
+ av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc->thumbnail_format);
+ return AVERROR(EINVAL);
+ }
+
+ vas = vaCreateSurfaces(hwctx->display, rt_format,
+ avfc->thumbnail_width, avfc->thumbnail_height,
+ &thumbnail_surface_id, 1,
+ &attrib, 1);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ return AVERROR(EIO);
+ }
+
+ av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. thumbnail_format(%d), w(%d), h(%d)\n",
+ thumbnail_surface_id, avfc->thumbnail_format, avfc->thumbnail_width, avfc->thumbnail_height);
+
+ ref = av_buffer_create((uint8_t*)(uintptr_t)thumbnail_surface_id,
+ sizeof(thumbnail_surface_id), &vaapi_buffer_free,
+ hwfc, AV_BUFFER_FLAG_READONLY);
+ if (!ref) {
+ vaDestroySurfaces(hwctx->display, &thumbnail_surface_id, 1);
+ return AVERROR(EIO);
+ }
+
+ frame_thumbnail_sd->buf = ref;
+
+ av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n", ref);
+
+ } else {
+ av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_thumbnail_sd is null\n");
+ }
+
+ return 0;
+}
+
static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
{
+ int err;
+
frame->buf[0] = av_buffer_pool_get(hwfc->pool);
if (!frame->buf[0])
return AVERROR(ENOMEM);
@@ -653,6 +719,11 @@ static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
frame->width = hwfc->width;
frame->height = hwfc->height;
+ //create sfc surface, put it in side data
+ err = vaapi_get_thumbnail_buffer(hwfc, frame);
+ if (err)
+ return err;
+
return 0;
}
@@ -784,7 +855,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
// assume for now that the user is not aware of that and would therefore
// prefer not to be given direct-mapped memory if they request read access.
if (ctx->derive_works && dst->format == hwfc->sw_format &&
- ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags & AV_HWFRAME_MAP_READ))) {
+ ((flags & AV_HWFRAME_MAP_DIRECT) || (flags & AV_HWFRAME_MAP_READ))) {
vas = vaDeriveImage(hwctx->display, surface_id, &map->image);
if (vas != VA_STATUS_SUCCESS) {
av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from "
diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h
index 0b2e071cb3..5031211f2c 100644
--- a/libavutil/hwcontext_vaapi.h
+++ b/libavutil/hwcontext_vaapi.h
@@ -100,6 +100,12 @@ typedef struct AVVAAPIFramesContext {
*/
VASurfaceID *surface_ids;
int nb_surfaces;
+
+ // Thumbnail info
+ int thumbnail_flags;
+ int thumbnail_format;
+ int thumbnail_width;
+ int thumbnail_height;
} AVVAAPIFramesContext;
/**
--
2.17.1
More information about the ffmpeg-devel
mailing list