[Libav-user] How to free an AVBuffer?

[Wladislav Artsimovich]

Dear FFmpeg and libav users,

I am using FFmpeg.AutoGen in my C# Program to write out frames encoded 
as a h.264 stream and inject per-frame metadata with an unregistered SEI 
Message. It works great, but I have a memory leak, which I don't know 
how to address. Source file attached.

Culprit is
```C#
fixed (byte* pMessageData = message)
{
     AVBufferRef* MetaDataBuffer = 
ffmpeg.av_buffer_alloc((ulong)message.Length);
     MetaDataBuffer->data = pMessageData;
     AVFrameSideData* sideData = 
ffmpeg.av_frame_new_side_data_from_buf(&frame, 
AVFrameSideDataType.AV_FRAME_DATA_SEI_UNREGISTERED, MetaDataBuffer);
}
```
I create an AVBuffer, as required by av_frame_new_side_data_from_buf(). 
But I cannot free it, resulting in a memory leak. I went through all of 
https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html and tried out 
different av_freep() av_free(), av_buffer_unref() functions, changing 
when to free etc. When I think I perform the free correctly, nothing 
happens. No error, no freeing just nothing. Clearly I misunderstand 
something.
To add insult to injury, the Visual Studio profiler cannot look inside 
the unmanaged memory of libav and reports, that the heap is fine and not 
growing, see attached screenshot.

How and when can I free this memory created allocated by 
av_buffer_alloc() in the attached source file?

I asked the same question in the FFmpeg.AutoGen Questions Repo. Some 
more context there: 
https://github.com/Ruslan-B/FFmpeg.AutoGen.Questions/issues/36

Best regards,

Vlad
-------------- next part --------------
using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Text;
using FFmpeg.AutoGen.Abstractions;

public sealed unsafe class H264VideoStreamEncoder : IDisposable
{
    private readonly Size _frameSize;
    private readonly int _width;
    private readonly int _height;
    private readonly int _linesizeY;
    private readonly AVCodec* _pCodec;
    private readonly AVCodecContext* _pCodecContext;
    private readonly Stream _stream;
    private readonly int _uSize;
    private readonly int _ySize;

    public H264VideoStreamEncoder(Stream stream, int fps, int width, int height)
    {
        _stream = stream;
        _width = width;
        _height = height;

        var codecId = AVCodecID.AV_CODEC_ID_H264;
        _pCodec = ffmpeg.avcodec_find_encoder(codecId);
        if (_pCodec == null) throw new InvalidOperationException("Codec not found.");

        _pCodecContext = ffmpeg.avcodec_alloc_context3(_pCodec);
        _pCodecContext->width = width;
        _pCodecContext->height = height;
        _pCodecContext->time_base = new AVRational { num = 1, den = fps };
        _pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_GRAY8;
        _pCodecContext->max_b_frames = 0;
        ffmpeg.av_opt_set(_pCodecContext->priv_data, "udu_sei", "1", 0);
        ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "veryfast", 0);
        ffmpeg.av_opt_set(_pCodecContext->priv_data, "crf", "25", 0);

        ffmpeg.avcodec_open2(_pCodecContext, _pCodec, null).ThrowExceptionIfError();


        _linesizeY = width;
    }

    public void Dispose()
    {
        ffmpeg.avcodec_close(_pCodecContext);
        ffmpeg.av_free(_pCodecContext);
    }

    public void Encode(AVFrame frame, MachineData machineData)
    {
        if (frame.format != (int)_pCodecContext->pix_fmt)
            throw new ArgumentException("Invalid pixel format.", nameof(frame));
        
        // Some Sanity checks
        if (frame.width != _width) throw new ArgumentException("Invalid width.", nameof(frame));
        if (frame.height != _height) throw new ArgumentException("Invalid height.", nameof(frame));
        if (frame.linesize[0] < _linesizeY) throw new ArgumentException("Invalid Y linesize.", nameof(frame));

        // The required `uuid_iso_iec_11578` as required by the H.264 spec, to
        // be recognized as a `User data unregistered` SEI message.
        string UUID = "139FB1A9446A4DEC8CBF65B1E12D2CFD";

        string custom_datapacket = string.Format(UUID +
            "{{" +
            "\"timestamp\":\" 
            ...
            <redacted>
            ...
            }}",
            <redacted>);
        byte[] message = Encoding.ASCII.GetBytes(custom_datapacket);

        fixed (byte* pMessageData = message)
        {
            AVBufferRef* MetaDataBuffer = ffmpeg.av_buffer_alloc((ulong)message.Length);
            MetaDataBuffer->data = pMessageData;
            AVFrameSideData* sideData = ffmpeg.av_frame_new_side_data_from_buf(&frame, AVFrameSideDataType.AV_FRAME_DATA_SEI_UNREGISTERED, MetaDataBuffer);
        }
        var pPacket = ffmpeg.av_packet_alloc();
        try
        {
            // Basic encoding loop explained: 
            // https://ffmpeg.org/doxygen/4.1/group__lavc__encdec.html

            // Give the encoder a frame to encode
            ffmpeg.avcodec_send_frame(_pCodecContext, &frame).ThrowExceptionIfError();

            // From https://ffmpeg.org/doxygen/4.1/group__lavc__encdec.html:
            // For encoding, call avcodec_receive_packet().  On success, it will return an AVPacket with a compressed frame.
            // Repeat this call until it returns AVERROR(EAGAIN) or an error.
            // The AVERROR(EAGAIN) return value means that new input data is required to return new output.
            // In this case, continue with sending input.
            // For each input frame/packet, the codec will typically return 1 output frame/packet, but it can also be 0 or more than 1.
            bool hasFinishedWithThisFrame;

            do
            {
                // Clear/wipe the receiving packet
                // (not sure if this is needed, since docs for avcoded_receive_packet say that it will call that first-thing
                ffmpeg.av_packet_unref(pPacket);

                // Receive back a packet; there might be 0, 1 or many packets to receive for an input frame.
                var response = ffmpeg.avcodec_receive_packet(_pCodecContext, pPacket);

                bool isPacketValid;

                if (response == 0)
                {
                    // 0 on success; as in, successfully retrieved a packet, and expecting us to retrieve another one.
                    isPacketValid = true;
                    hasFinishedWithThisFrame = false;
                }
                else if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // EAGAIN: there's no more output is available in the current state - user must try to send more input
                    isPacketValid = false;
                    hasFinishedWithThisFrame = true;
                }
                else if (response == ffmpeg.AVERROR(ffmpeg.AVERROR_EOF))
                {
                    // EOF: the encoder has been fully flushed, and there will be no more output packets
                    isPacketValid = false;
                    hasFinishedWithThisFrame = true;
                }
                else
                {
                    // AVERROR(EINVAL): codec not opened, or it is a decoder other errors: legitimate encoding errors
                    // , otherwise negative error code:
                    throw new InvalidOperationException($"error from avcodec_receive_packet: {response}");
                }

                if (isPacketValid)
                {
                    var packetStream = new UnmanagedMemoryStream(pPacket->data, pPacket->size);
                    packetStream.CopyTo(_stream);
                }
            } while (!hasFinishedWithThisFrame);
        }
        finally
        {
            ffmpeg.av_packet_free(&pPacket);
        }
    }

    public void Drain()
    {
        // From https://ffmpeg.org/doxygen/4.1/group__lavc__encdec.html:
        // End of stream situations. These require "flushing" (aka draining) the codec, as the codec might buffer multiple frames or packets internally for performance or out of necessity (consider B-frames). This is handled as follows:
        // Instead of valid input, send NULL to the avcodec_send_packet() (decoding) or avcodec_send_frame() (encoding) functions. This will enter draining mode.
        // 	Call avcodec_receive_frame() (decoding) or avcodec_receive_packet() (encoding) in a loop until AVERROR_EOF is returned. The functions will not return AVERROR(EAGAIN), unless you forgot to enter draining mode.

        var pPacket = ffmpeg.av_packet_alloc();

        try
        {
            // Send a null frame to enter draining mode
            ffmpeg.avcodec_send_frame(_pCodecContext, null).ThrowExceptionIfError();

            bool hasFinishedDraining;

            do
            {
                // Clear/wipe the receiving packet
                // (not sure if this is needed, since docs for avcoded_receive_packet say that it will call that first-thing
                ffmpeg.av_packet_unref(pPacket);

                var response = ffmpeg.avcodec_receive_packet(_pCodecContext, pPacket);

                bool isPacketValid;

                if (response == 0)
                {
                    // 0 on success; as in, successfully retrieved a packet, and expecting us to retrieve another one.
                    isPacketValid = true;
                    hasFinishedDraining = false;
                }
                else if (response == ffmpeg.AVERROR(ffmpeg.AVERROR_EOF))
                {
                    // EOF: the encoder has been fully flushed, and there will be no more output packets
                    isPacketValid = false;
                    hasFinishedDraining = true;
                }
                else
                {
                    // Some other error.
                    // Should probably throw here, but in testing we get error -541478725
                    isPacketValid = false;
                    hasFinishedDraining = true;
                }

                if (isPacketValid)
                {
                    var packetStream = new UnmanagedMemoryStream(pPacket->data, pPacket->size);
                    packetStream.CopyTo(_stream);
                }
            } while (!hasFinishedDraining);
        }
        finally
        {
            ffmpeg.av_packet_free(&pPacket);
        }
    }
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: profiler_screenshot.png
Type: image/png
Size: 44673 bytes
Desc: not available
URL: <https://ffmpeg.org/pipermail/libav-user/attachments/20230301/47f1b7a1/attachment.png>