[Libav-user] get RGB values from ffmpeg frame
Malik Cissé
mc at enciris.com
Tue Nov 20 12:21:28 CET 2012
Hi Navin,
Here is an example with SDL and avcodec_decode_video2 (which is not deprecated).
int main(int argc, char **argv)
{
AVCodec *codec;
AVCodecContext *c= NULL;
int got_picture, len;
AVFrame *picture;
int out_size_y;
int out_size_cr;
AVPacket pkt;
int result;
unsigned char *pIdxBuffer;
int IdxBufferSize;
int IdxByteTransfered;
unsigned char *pHdrBuffer;
int HdrBufferSize;
int HdrByteTransfered;
unsigned long long fileSize;
FILE *pFileAsf;
int isFirstTime = 1;
int Status = LT_SUCCESS;
int FrameType = 0;
int LoopCnt = 0;
metaD_t metaD;
unsigned char *pVidData = (unsigned char *)malloc(MAX_FR_SZ);
int vidBytesXfered = 0;
unsigned char *pAudData = (unsigned char *)malloc(MAX_FR_SZ);
int audBytesXfered = 0;
int noSamples;//number of audio samples
unsigned char *pAsfData = (unsigned char *)malloc(MAX_FR_SZ);
int asfBytesXfered = 0;
char baseName[50] = "output_asf";
char fileName[50];
char strNumber[20];
char strHdr[5] = ".asf";
FILE *outfile = fopen("test_720x480.yuv", "wb");//output decompressed file
FILE *outfile1 = fopen("test_720x480.h264", "wb");//output decompressed file
int i,j;
//########### sdl stuff ################
SDL_Surface *screen;
SDL_Overlay *bmp;
SDL_Rect rect;
bool running;
SDL_Event event;
BOOL toggle = 1;
printf("Video decoding\n");
//init
init_user_data(&metaD);
LtHandle = 0;
pIdxBuffer = (unsigned char *)malloc(ASF_IDX_SZ);
IdxBufferSize = ASF_IDX_SZ;
pHdrBuffer = (unsigned char *)malloc(metaD.asf_packet_size);
HdrBufferSize = metaD.asf_packet_size;
//init board
//code has been removed...
//out_size_y = metaD.vc1_width * metaD.vc1_height;
//out_size_cr = (metaD.vc1_width * metaD.vc1_height)/4;
get_hdr_info(&metaD);
result = init_ffmpeg_lib();
if(result < 0)
{
fprintf(stderr, "Error while initializing FFMPEG lib\n");
exit(1);
}
/* find the video decoder */
if(IS_H264)
{
codec = avcodec_find_decoder(CODEC_ID_H264);
}
else
{
codec = avcodec_find_decoder(CODEC_ID_VC1);
}
if (!codec)
{
fprintf(stderr, "codec not found\n");
exit(1);
}
c= avcodec_alloc_context();
//###############################################
//###############################################
picture= (AVFrame *)avcodec_alloc_frame();
//h264
if(IS_H264)
{
//c->extradata = extra;
//c->extradata_size = 14;
}
else
{
//vc1
//this is important
c->width = metaD.vc1_width;
c->height = metaD.vc1_height;
c->extradata = metaD.pTopLevelHeader;
c->extradata_size = metaD.codec_data_len;
}
/* open it */
if (avcodec_open(c, codec) < 0) {
fprintf(stderr, "could not open codec\n");
exit(1);
}
if (!outfile)
{
av_free(c);
exit(1);
}
//######## SDL stuff ############################
if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
{
fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
exit(1);
}
screen = SDL_SetVideoMode(metaD.vc1_width, metaD.vc1_height, 0, 0);
if(!screen)
{
fprintf(stderr, "SDL: could not set video mode - exiting\n");
exit(1);
}
// Allocate a place to put our YUV image on that screen
bmp = SDL_CreateYUVOverlay(metaD.vc1_width,
metaD.vc1_height,
SDL_YV12_OVERLAY,
screen);
//######## end SDL stuff ############################
LT_StartVc1(LtHandle);
running = TRUE;
while(running)//audio video fetch loop
{
printf("LoopCnt: %d\r", LoopCnt++);
Status = LT_GetVc1Frame(LtHandle, pVidData, MAX_FR_SZ, &vidBytesXfered, &FrameType);
if( Status < 0 ) // is an error occured ?
{
printf("Vc1 fetch error\n");
LT_GetError(Status);
return Status;
}
fwrite(pVidData, sizeof(char), vidBytesXfered, outfile1);
//decode VC-1 image
av_init_packet(&pkt);
pkt.data = pVidData;
pkt.size = vidBytesXfered;
len = avcodec_decode_video2(c, picture, &got_picture, &pkt);
//len = avcodec_decode_video(c, picture, &got_picture, pVidData, vidBytesXfered);
if (len < 0)
{
fprintf(stderr, "Error while decoding frame\n");
exit(1);
}
#if 0
if (got_picture)
{
for(i=0; i<c->height; i++)
fwrite(picture->data[0] + i * picture->linesize[0], 1, c->width, outfile );
for(i=0; i<c->height/2; i++)
fwrite(picture->data[1] + i * picture->linesize[1], 1, c->width/2, outfile );
for(i=0; i<c->height/2; i++)
fwrite(picture->data[2] + i * picture->linesize[2], 1, c->width/2, outfile );
}
#endif
#if 1
// Did we get a video frame?
if(got_picture)
{
SDL_LockYUVOverlay(bmp);
AVPicture pict;
pict.data[0] = bmp->pixels[0];
pict.data[1] = bmp->pixels[2];
pict.data[2] = bmp->pixels[1];
pict.linesize[0] = bmp->pitches[0];
pict.linesize[1] = bmp->pitches[2];
pict.linesize[2] = bmp->pitches[1];
SwsContext * convert_ctx = sws_getContext ( c->width, c->height, c->pix_fmt,c->width, c->height, PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL );
sws_scale( convert_ctx, picture->data, picture->linesize, 0, c->height, pict.data, pict.linesize );
sws_freeContext ( convert_ctx );
/*
// Convert the image into YUV format that SDL uses
img_convert(&pict, PIX_FMT_YUV420P,
(AVPicture *)picture, c->pix_fmt,
c->width, c->height);
*/
SDL_UnlockYUVOverlay(bmp);
rect.x = 0;
rect.y = 0;
rect.w = c->width;
rect.h = c->height;
SDL_DisplayYUVOverlay(bmp, &rect);
//av_free_packet(&packet);
}
#endif
#if 0//only works with VC-1
//############ display ####################
//av_free_packet(&pkt);
// Did we get a video frame?
if(got_picture)
{
SDL_LockYUVOverlay(bmp);
memcpy(bmp->pixels[0], picture->data[0], c->width * c->height);//y
memcpy(bmp->pixels[2], picture->data[1], (c->width * c->height)/4);//v
memcpy(bmp->pixels[1], picture->data[2], (c->width * c->height)/4);//u
SDL_UnlockYUVOverlay(bmp);
rect.x = 0;
rect.y = 0;
rect.w = c->width;
rect.h = c->height;
SDL_DisplayYUVOverlay(bmp, &rect);//display this image
}
#endif
//should probably not be in the loop
SDL_PollEvent(&event);
switch(event.type)
{
//printf("");
case SDL_QUIT:
running = FALSE;
SDL_Quit();
exit(0);
break;
case SDL_KEYDOWN:
switch(event.key.keysym.sym)
{
case SDLK_ESCAPE:
case SDLK_q:
running = FALSE;
SDL_Quit();
exit(0);
break;
case SDLK_f:
toggle = !toggle;
break;
default:
break;
}
default:
break;
}
}
running = FALSE;
// Close dev handle
//......
FreeLtDll(m_ltDll);
printf("<-- CloseDevice\n\n");
fclose(outfile);
fclose(outfile1);
avcodec_close(c);
av_free(c);
av_free(picture);
SDL_FreeYUVOverlay( bmp );
SDL_FreeSurface( screen );
SDL_CloseAudio();
SDL_Quit();
printf("\n");
}
-----Original Message-----
From: libav-user-bounces at ffmpeg.org [mailto:libav-user-bounces at ffmpeg.org] On Behalf Of Navin
Sent: 20 November 2012 12:09
To: libav-user at ffmpeg.org
Subject: Re: [Libav-user] get RGB values from ffmpeg frame
Hey thanks everyone! Still trying/struggling with the code sent by Malik because avcodec_decode_video is deprecated.
Thought I'd post the entire code here for anyone who could help or for anyone who needs such code in future. This is the corrected version of ffmpeg's tutorial02, with SDL added to it too.
Problems are:
1. Since I was facing trouble with RGB, I thought I'd use SDL to output the frames as bitmaps, but although I could store the bitmap in an area in memory, it threw an exception when I tried extracting it out of memory.
2. Besides, when at least trying to output the bitmap to hard disk, I'm unable to output it unless SDL_DisplayYUVOverlay(bmp, &rect); is called (and I don't want to call it because I don't want the video to be displayed. I just want the bitmap to be output).
3. Main problem is still with extracting RGB, so would be grateful for any help. If not RGB, then at least with question 1 (how to store and extract the BMP from SDL memory).
The code:
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
}
#include<iostream>
#include <SDL.h>
#include <SDL_thread.h>
#include "SDL_opengl.h"
#include <gl/gl.h>
#include <gl/glu.h>
//#ifdef __MINGW32__
#undef main /* Prevents SDL from overriding main() */
//#endif
#include <stdio.h>
//------------------------
// These functions should not be used except from pointers in a RWops
static int myseekfunc(SDL_RWops *context, int offset, int whence) {
SDL_SetError("Can't seek in this kind of RWops"); return(-1); }
static int myreadfunc(SDL_RWops *context, void *ptr, int size, int
maxnum) { memset(ptr,0,size*maxnum); return(maxnum); }
static int mywritefunc(SDL_RWops *context, const void *ptr, int size,
int num) { return(num); }
static int myclosefunc(SDL_RWops *context)
{
if(context->type != 0xdeadbeef) { SDL_SetError("Wrong kind of RWops
for myclosefunc()"); return(-1); }
free(context->hidden.unknown.data1);
SDL_FreeRW(context);
return(0);
}
// Note that this function is NOT static -- we want it directly callable
from other source files
SDL_RWops *MyCustomRWop()
{
SDL_RWops *c=SDL_AllocRW();
if(c==NULL) return(NULL);
c->seek =myseekfunc;
c->read =myreadfunc;
c->write=mywritefunc;
c->close=myclosefunc;
c->type =0xdeadbeef;
printf("deadbeef=%d\n",c->type);
c->hidden.unknown.data1=malloc(100000);
return(c);
}
int main(int argc, char *argv[])
{
AVFormatContext *pFormatCtx = NULL;
int i, videoStream;
AVCodecContext *pCodecCtx = NULL;
AVCodec *pCodec = NULL;
AVFrame *pFrame = NULL;
AVPacket packet;
int frameFinished;
//float aspect_ratio;
AVDictionary *optionsDict = NULL;
struct SwsContext *sws_ctx = NULL;
SDL_Overlay *bmp = NULL;
SDL_Surface *screen = NULL;
SDL_Surface *screen2 = NULL;
SDL_Surface *rgbscreen = NULL;
SDL_Rect rect;
SDL_Event event;
if(argc < 2) { fprintf(stderr, "Usage: test.exe <videofile>\n");
exit(1); }
// Register all formats and codecs
av_register_all();
if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
exit(1); }
// Open video file
if(avformat_open_input(&pFormatCtx, argv[1], NULL, NULL)!=0)
return -1; // Couldn't open file
// Retrieve stream information
if(avformat_find_stream_info(pFormatCtx, NULL)<0) return -1; //
Couldn't find stream information
// Dump information about file onto standard error
//av_dump_format(pFormatCtx, 0, argv[1], 0);
// Find the first video stream
videoStream=-1;
for(i=0; i<pFormatCtx->nb_streams; i++)
if(pFormatCtx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO) {
videoStream=i; break; }
if(videoStream==-1) return -1; // Didn't find a video stream
// Get a pointer to the codec context for the video stream
pCodecCtx=pFormatCtx->streams[videoStream]->codec;
// Find the decoder for the video stream
pCodec=avcodec_find_decoder(pCodecCtx->codec_id);
if(pCodec==NULL) { fprintf(stderr, "Unsupported codec!\n");
return -1;} // Codec not found
// Open codec
if(avcodec_open2(pCodecCtx, pCodec, &optionsDict) < 0) return -1;
// Could not open codec
// Allocate video frame
pFrame = avcodec_alloc_frame();
#if defined(TRIAL)
int width1 = pCodecCtx->width, height1 = pCodecCtx->height, width2
= pCodecCtx->width, height2 = pCodecCtx->height;
struct SwsContext *resize = sws_getContext(width1, height1,
PIX_FMT_YUV420P, width2, height2, PIX_FMT_RGB24, SWS_BICUBIC, NULL,
NULL, NULL);
AVFrame* frame1 = avcodec_alloc_frame(); // this is your original frame
AVFrame* frame2 = avcodec_alloc_frame();
int num_bytes = avpicture_get_size(PIX_FMT_RGB24, width2, height2);
uint8_t* frame2_buffer = (uint8_t *) av_malloc(num_bytes *
sizeof(uint8_t));
#endif
// Make a screen to put our video
#ifndef __DARWIN__
screen = SDL_SetVideoMode(pCodecCtx->width, pCodecCtx->height,
0, 0);
#else
screen = SDL_SetVideoMode(pCodecCtx->width, pCodecCtx->height,
24, 0);
#endif
if(!screen) { fprintf(stderr, "SDL: could not set video mode -
exiting\n"); exit(1); }
// Allocate a place to put our YUV image on that screen
bmp = SDL_CreateYUVOverlay(pCodecCtx->width,
pCodecCtx->height, SDL_YV12_OVERLAY, screen);
sws_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,
pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,
PIX_FMT_YUV420P, SWS_BILINEAR, NULL, NULL, NULL );
// Read frames and save first five frames to disk
i=0;
while(av_read_frame(pFormatCtx, &packet) >= 0)
{
// Is this a packet from the video stream?
if(packet.stream_index==videoStream)
{
// Decode video frame
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
// Did we get a video frame?
if(frameFinished)
{
#if defined(TRIAL)
avpicture_fill((AVPicture*) frame2, frame2_buffer,
PIX_FMT_RGB24, width2, height2);
printf("data = %d \n",frame2->data[0]);
printf("linesize = %d %d %d\n",frame2->linesize[0],
frame2->linesize[1], frame2->linesize[2]);
printf("width = %d\n", pCodecCtx->width);
printf("height = %d\n", pCodecCtx->height);
std::cin.get();
int linesize = frame2->linesize[0];
for(int xx = 0; xx < (linesize * width1)-1; xx += 3)
{
int r = frame2->data[0][xx];//int r = frame2->data[0][xx];
int g = frame2->data[0][xx+1];
int b = frame2->data[0][xx+2];
printf("xx=%d r=%d, g=%d, b=%d \n",xx, r,
g, b);
}
printf("frame%d done----------------",i++);
//for(int xx = 0; xx < width1; xx = xx + 3)
//{
// for(int yy = 0; yy < height1; ++yy)
// {
// //int p = xx*3 + yy*frame2->linesize[0];
// //int p = xx * 3 + yy * linesize;
// printf("yy=%d xx=%d",yy,xx);
// int p = yy * linesize + xx;
// printf("p=%d\n",p);
// int r = frame2->data[0][p];
// int g = frame2->data[0][p+1];
// int b = frame2->data[0][p+2];
// printf("[r=%d, g=%d, b=%d ]\n", r, g, b);
// }//for
//}//for
// frame1 should be filled by now (eg using avcodec_decode_video)
sws_scale(resize, frame1->data, frame1->linesize, 0, height1,
frame2->data, frame2->linesize);
#endif
SDL_LockYUVOverlay(bmp);//-----------lock
AVPicture pict;
pict.data[0] = bmp->pixels[0];
pict.data[1] = bmp->pixels[2];
pict.data[2] = bmp->pixels[1];
pict.linesize[0] = bmp->pitches[0];
pict.linesize[1] = bmp->pitches[2];
pict.linesize[2] = bmp->pitches[1];
// Convert the image into YUV format that SDL uses
sws_scale( sws_ctx, (uint8_t const * const *)pFrame->data,
pFrame->linesize, 0, pCodecCtx->height, pict.data, pict.linesize );
SDL_UnlockYUVOverlay(bmp);//-----------unlock
rect.x = 0;
rect.y = 0;
rect.w = pCodecCtx->width;
rect.h = pCodecCtx->height;
SDL_DisplayYUVOverlay(bmp, &rect);
//printf("sizeof screen = %d\n",sizeof(*screen));
if (++i != 0)
{
char numberbuffer [33], str[80]; strcpy (str, "bitmap");
strcat (str, itoa(i, numberbuffer, 10)); strcat (str, ".bmp");//file for
storing image
//---------this saves it as a bitmap to filestream. But now
how to extract it?
//SDL_RWops *filestream = MyCustomRWop();//SDL_AllocRW();
//SDL_SaveBMP_RW (screen, filestream, i);
//screen2 = SDL_LoadBMP_RW(filestream,1);//LOADING IS THE
PROBLEM HERE. DON'T KNOW WHY
//filestream->close;
//SDL_SaveBMP(screen2, str);
SDL_SaveBMP(screen, str);//WORKS: saves frame to a file as
a bitmap
}
}//if(frameFinished)
}//if
// Free the packet that was allocated by av_read_frame
av_free_packet(&packet);
SDL_PollEvent(&event);
switch(event.type)
{
case SDL_QUIT:
SDL_Quit();
exit(0);
break;
default:
break;
}//switch
}//while
av_free(pFrame);// Free the YUV frame
avcodec_close(pCodecCtx);// Close the codec
avformat_close_input(&pFormatCtx);// Close the video file
return 0;
}//main
Nav
On 11/20/2012 3:38 PM, Carl Eugen Hoyos wrote:
> Navin<nkipe at ...> writes:
>
>> I have been through many websites, but they either use
>> img_convert (which doesn't work) or sws_scale, which
>> crashes when I try to use it with RGB.
> I may miss something, but I guess this is the important
> problem you should work on (see doc/examples).
>
> Possibly unrelated:
> In your source code, I did not find information which
> decoder you are using.
> Some decoders (everything mpeg-related) output yuv, and
> if you want the data in rgb, you have to convert it
> (using the sofware scaler), others (some lossless formats)
> output rgb, that may already be the data you want.
>
> Carl Eugen
>
> _______________________________________________
> Libav-user mailing list
> Libav-user at ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/libav-user
>
_______________________________________________
Libav-user mailing list
Libav-user at ffmpeg.org
http://ffmpeg.org/mailman/listinfo/libav-user
More information about the Libav-user
mailing list