[Libav-user] mpeg4 decoding with CODEC_FLAG_TRUNCATED: memleak and missing last image

[Gajdosik Johannes]

Hi,

In my code I use CODEC_FLAG_TRUNCATED for mpeg4 decoding.
But this gives me 2 bugs in my program: a memleak an a missing decoded image.
I get the misbehaviour in all versions I have tested with, especially in versions 3.0.2 and 2.8.7.

In order to reproduce this behaviour I have written a fully working test program below.

Could you please tell me if I use ffmpeg correctly and if the bugs are my own fault or part of ffmpeg?

Yours,
Johannes Gajdosik






--------------------start of program-----------------------
/*

Compile with:
gcc -g ffmpeg_mpeg4_decoding_bugs.c -o ffmpeg_mpeg4_decoding_bugs -lavcodec -lavutil

This program intends to demonstrate 2 bugs in the ffmpeg mpeg4 encoder:
a memleak and failure to flush when CODEC_FLAG_TRUNCATED is used.
It works like this:

1) Generate a certain number of mpeg4-frames with a unique image contents.
2) Decode the encoded frames and check the image contents.
   Step 2 can be done repeatedly in order to demonstrate the memleak.
3) release the encoded images

Basically the program works: The encoded images can be decoded
and the image contents is successfully checked.
But there are 2 bugs.

How to reproduce the bugs:

A) Call './ffmpeg_mpeg4_decoding_bugs -flag_truncated -verbosity 5'
You will see the msg "ERROR: only 4 pictures decoded, 1 picture(s) missing".
The last image was not decoded although the decoder was flushed
with a NULL-packet.

B) Call './ffmpeg_mpeg4_decoding_bugs -flag_truncated -verbosity 0 -nr_of_decoding_runs 1000000'
You can watch the memleak with 'top'.
Each time a the decoder is released you lose about 3k, that is approximately
the size of an I-Frame. Alternatively you can use valgrind:
valgrind --leak-check=full ./ffmpeg_mpeg4_decoding_bugs -flag_truncated

Note that without -flag_truncated these 2 errors do not show up.

*/


#include <libavcodec/avcodec.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

  /* program parameters: */
static int nr_of_images = 5;
static int nr_of_decoding_runs = 1;
static int flag_truncated = 0;
static int verbosity = 1;


  /* encoded images: */
struct EncodedImage {
  unsigned char *data;
  int size;
};
static struct EncodedImage *encoded_images = 0;



  /* in a grey image fill a rectangle with 'val' */
static
void FillRect(unsigned char *im,int width,
              int x,int y,int size_x,int size_y,
              unsigned char val) {
  im += y*width+x;
  while (--size_y >= 0) {
    memset(im,val,size_x);
    im += width;
  }
}

  /* get average gray value inside given rectangle */
unsigned char GetRectVal(const unsigned char *im,int width,
                         int x,int y,int size_x,int size_y) {
  unsigned int rval = 0;
    /* ignore rectangle borders: */
  x++;y++;
  size_x -= 2;
  size_y -= 2;
  im += y*width+x;
    /* sum up all pixel values: */
  for (y=0;y<size_y;y++,im+=width) {
    for (x=0;x<size_x;x++) {
      rval += im[x];
    }
  }
    /* and divide by number of pixels: */
  return (unsigned char)(rval / (size_y * size_x));
}

  /* fill yuv420 image with a pattern that encodes 'number' */
static
void EncodeBWpatternIntoYuvImage(unsigned char *im,
                                 int width,int height,
                                 unsigned int number) {
  const int w4 = width / 4;
  const int h4 = height / 4;
  int i,j;
    /* set u,v to neutral color */
  memset(im+width*height,128,width*height/2);
    /* fill image with a B/W pattern that encodes 'number' */
  for (j=3;j>=0;j--) {
    for (i=3;i>=0;i--) {
      FillRect(im,width,i*w4,j*h4,w4,h4,(number&1)?255:0);
      number >>= 1;
    }
  }
}

  /* decode the number that was encoded in EncodeBWpatternIntoYuvImage: */
static
unsigned int DecodeBWpatternFromYuv420(const unsigned char *im,
                                       int width,int height) {
  unsigned int image_val = 0;
    /* extract value of the B/W-pattern: */
  const int w4 = width / 4;
  const int h4 = height / 4;
  int i,j;
  for (j=0;j<4;j++) {
    for (i=0;i<4;i++) {
      const unsigned char val = GetRectVal(im,width,i*w4,j*h4,w4,h4);
      image_val <<= 1;
      if (val >= 128) image_val |= 1;
    }
  }
  return image_val;
}





  /* populate encoded_images: */
static
void EncodeImages(void) {

    /* ffmpeg data structures: */
  AVFrame *frame = av_frame_alloc();
  AVCodec *const codec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
  AVCodecContext *const codec_context = avcodec_alloc_context3(codec);

    /* image to encode: */
  const int width = 512;
  const int height = 512;
  unsigned char *const im = (unsigned char*)malloc(width*height*3/2);

  unsigned int image_nr;
  int got_output;
  

  if (frame == 0) abort();
  if (codec == 0) abort();
  if (codec_context == 0) abort();
  if (im == 0) abort();

  encoded_images = (struct EncodedImage*)
                     calloc(nr_of_images,sizeof(struct EncodedImage));
  if (encoded_images == 0) abort();

    /* encoding parameters: */
  codec_context->bit_rate_tolerance = 4000000;
  codec_context->bit_rate = 500000;
  codec_context->width = frame->width = width;
  codec_context->height = frame->height = height;
  codec_context->time_base.num = 1;
  codec_context->time_base.den = 25;
  codec_context->gop_size = 5;
  codec_context->max_b_frames = 0;
  frame->format = AV_PIX_FMT_YUV420P;
  codec_context->pix_fmt = (enum AVPixelFormat)frame->format;

  if (avcodec_open2(codec_context,codec,0) < 0) abort();

  for (image_nr=0;image_nr<nr_of_images;image_nr++) {
    AVPacket pkt;
    EncodeBWpatternIntoYuvImage(im,width,height,image_nr);
    frame->data[0] = im;
    frame->data[1] = frame->data[0] + (frame->width * frame->height);
    frame->data[2] = frame->data[1] + (frame->width * frame->height) / 4;
    frame->linesize[0] = width;
    frame->linesize[2] = frame->linesize[1] = frame->linesize[0]/2;
    frame->pts = image_nr;

    av_init_packet(&pkt);
    pkt.data = 0;
    pkt.size = 0;
    
    if (0 > avcodec_encode_video2(codec_context,&pkt,frame,
                                  &got_output)) abort();
    if (!got_output) abort();
    if (pkt.size <= 0 || pkt.size > 100000) abort();

    if (verbosity>2) printf("image %d encoded, size: %d\n",image_nr,pkt.size);

      /* save encoded image: */
    encoded_images[image_nr].data
      = (unsigned char*)malloc(pkt.size+AV_INPUT_BUFFER_PADDING_SIZE);
    if (encoded_images[image_nr].data == 0) abort();
    encoded_images[image_nr].size = pkt.size;
    memcpy(encoded_images[image_nr].data,pkt.data,pkt.size);
      /* pad with zeros: */
    memset(encoded_images[image_nr].data+pkt.size,0,
           AV_INPUT_BUFFER_PADDING_SIZE);

      /* cleanup */
    av_free_packet(&pkt);
  }
    /* cleanup */
  free(im);
  avcodec_close(codec_context);
  av_free(codec_context);
  av_frame_free(&frame);
  if (verbosity>1) printf("%d images encoded\n",nr_of_images);
}

  /* cleanup encoded_images */
static void FreeImages(void) {
  unsigned int image_nr;
  for (image_nr=0;image_nr<nr_of_images;image_nr++) {
    free(encoded_images[image_nr].data);
    if (verbosity>4) printf("image %d released\n",image_nr);
  }
  free(encoded_images);
  if (verbosity>1) printf("%d images released\n",nr_of_images);
}



void DecodeImages(void) {
    /* ffmpeg data structures: */
  AVFrame *picture = av_frame_alloc();
  AVCodec *const codec = avcodec_find_decoder(AV_CODEC_ID_MPEG4);
  AVCodecContext *const codec_context = avcodec_alloc_context3(codec);

  unsigned int frame_nr;
  unsigned int picture_nr = 0;
  unsigned int image_val;
  int got_pic;

  if (picture == 0) abort();
  if (codec == 0) abort();
  if (codec_context == 0) abort();

  codec_context->codec_type = codec->type;
  codec_context->codec_id = codec->id;
  if (flag_truncated) codec_context->flags |= CODEC_FLAG_TRUNCATED;

  if (avcodec_open2(codec_context,codec,0) < 0) abort();
  
  for (frame_nr=0;;frame_nr++) {
    AVPacket pkt;
    av_init_packet(&pkt);
    if (frame_nr<nr_of_images) {
      if (verbosity>4) printf("decoding frame %d\n",frame_nr);
      pkt.data = encoded_images[frame_nr].data;
      pkt.size = encoded_images[frame_nr].size;
    } else {
        /* decode NULL-packet in order to flush the decoder: */
      if (verbosity>2) printf("flushing decoder\n");
      pkt.data = 0;
      pkt.size = 0;
    }
    do {
      const int len = avcodec_decode_video2(codec_context,picture,
                                            &got_pic,&pkt);
      if (verbosity>3) printf("avcodec_decode_video2(%d) "
                              "returned %d, got_pict: %d\n",
                              pkt.size,len,got_pic);
      if (len < 0) break;
      if (got_pic) {
        if (picture->format != AV_PIX_FMT_YUV420P) abort();
        image_val = DecodeBWpatternFromYuv420(picture->data[0],
                                              picture->linesize[0],
                                              picture->height);
        if (image_val != picture_nr) {
          if (verbosity>0) printf("ERROR: picture %u has bad pattern %u\n",
                                  picture_nr,image_val);
        } else {
          if (verbosity>4) printf("picture %u: pattern ok\n",picture_nr);
        }
        picture_nr++;
      }
      if (pkt.data) {
        pkt.data += len;
        pkt.size -= len;
      }
    } while (pkt.size > 0);
    av_free_packet(&pkt);
      /* exit only after the flushing is complete: */
    if (got_pic == 0 && frame_nr >= nr_of_images) break;
  }
  if (picture_nr < nr_of_images) {
    if (verbosity>0) printf("ERROR: only %d pictures decoded,"
                            " %d picture(s) missing\n",
                            picture_nr,nr_of_images-picture_nr);
  }
  
    /* cleanup */
  avcodec_close(codec_context);
  av_free(codec_context);
  av_frame_free(&picture);
}


static
void PrintUsage(const char *progname) {
  printf("Usage: %s [-nr_of_images <nr>] [-nr_of_decoding_runs <nr>]"
         " [-flag_truncated] [-verbosity <0..5>]\n\n",
         progname);
}

int main(int argc,char *argv[]) {
  int i;
  for (i=1;i<argc;i++) {
    if (0 == strcmp(argv[i],"-nr_of_images")) {
      if (++i >= argc ||
          1 != sscanf(argv[i],"%d",&nr_of_images) ||
          nr_of_images <= 0) {
        fprintf(stderr,"nr_of_images: integer expected\n");
        return 1;
      }
    } else
    if (0 == strcmp(argv[i],"-nr_of_decoding_runs")) {
      if (++i >= argc ||
          1 != sscanf(argv[i],"%d",&nr_of_decoding_runs) ||
          nr_of_decoding_runs <= 0) {
        fprintf(stderr,"nr_of_decoding_runs: integer expected\n");
        return 1;
      }
    } else
    if (0 == strcmp(argv[i],"-flag_truncated")) {
      flag_truncated = 1;
    } else
    if (0 == strcmp(argv[i],"-verbosity")) {
      if (++i >= argc ||
          1 != sscanf(argv[i],"%d",&verbosity) ||
          verbosity < 0) {
        fprintf(stderr,"verbosity: integer expected\n");
        return 1;
      }
    } else
    {
      fprintf(stderr,"unknown option: \"%s\"\n",argv[i]);
      PrintUsage(argv[0]);
      return 1;
    }
  }

  printf("program parameters:\n"
         "  nr_of_images: %d\n"
         "  nr_of_decoding_runs: %d\n"
         "  flag_truncated: %d\n"
         "  verbosity: %d\n\n",
         nr_of_images,nr_of_decoding_runs,flag_truncated,verbosity);

  avcodec_register_all();
  EncodeImages();
  for (i=0;i<nr_of_decoding_runs;i++) {
    if (verbosity>0) printf("decoding test, run %d\n",i);
    DecodeImages();
  }
  FreeImages();
  printf("bye.\n");
  return 0;
}