[FFmpeg-devel] [PATCH] avformat/movenc: Support alpha channel for AVIF

[James Zern]

On Wed, May 18, 2022 at 2:56 PM Vignesh Venkatasubramanian
<vigneshv-at-google.com at ffmpeg.org> wrote:
>
> AVIF specification allows for alpha channel as an auxillary item (in

auxiliary

> case of still images) or as an auxillary track (in case of animated
> images).  Add support for both of these. The AVIF muxer will take
> exactly two streams (when alpha is present) as input (first one being
> the YUV planes and the second one being the alpha plane).
>
> The input has to come from two different images (one of it color and
> the other one being alpha), or it can come from a single file
> source with the alpha channel extracted using the "alphaextract"
> filter.
>
> Example using alphaextract:
> ffmpeg -i rgba.png -filter_complex "[0:v]alphaextract[a]" -map 0 -map "[a]" -still-picture 1 avif_with_alpha.avif
>
> Example using two sources (first source can be in any pixel format and
> the second source has to be in monochrome grey pixel format):
> ffmpeg -i color.avif -i grey.avif -map 0 -map 1 -c copy avif_with_alpha.avif
>
> The generated files pass the compliance checks in Compliance Warden:
> https://github.com/gpac/ComplianceWarden
>
> libavif (the reference avif library) is able to decode the files
> generated using this patch.
>
> They also play back properly (with transparent background) in:
> 1) Chrome
> 2) Firefox (only still AVIF, no animation support)
>
> Signed-off-by: Vignesh Venkatasubramanian <vigneshv at google.com>
> ---
>  libavformat/movenc.c | 185 +++++++++++++++++++++++++++++--------------
>  libavformat/movenc.h |   4 +-
>  2 files changed, 128 insertions(+), 61 deletions(-)
>
> diff --git a/libavformat/movenc.c b/libavformat/movenc.c
> index de971f94e8..00e42b7abb 100644
> --- a/libavformat/movenc.c
> +++ b/libavformat/movenc.c
> @@ -2852,7 +2852,7 @@ static int mov_write_hdlr_tag(AVFormatContext *s, AVIOContext *pb, MOVTrack *tra
>          hdlr = (track->mode == MODE_MOV) ? "mhlr" : "\0\0\0\0";
>          if (track->par->codec_type == AVMEDIA_TYPE_VIDEO) {
>              if (track->mode == MODE_AVIF) {
> -                hdlr_type = "pict";
> +                hdlr_type = (track == &mov->tracks[0]) ? "pict" : "auxv";
>                  descr     = "PictureHandler";
>              } else {
>                  hdlr_type = "vide";
> @@ -2940,57 +2940,83 @@ static int mov_write_iloc_tag(AVIOContext *pb, MOVMuxContext *mov, AVFormatConte
>      avio_wb32(pb, 0); /* Version & flags */
>      avio_w8(pb, (4 << 4) + 4); /* offset_size(4) and length_size(4) */
>      avio_w8(pb, 0); /* base_offset_size(4) and reserved(4) */
> -    avio_wb16(pb, 1); /* item_count */
> +    avio_wb16(pb, s->nb_streams); /* item_count */

This assumes at most 2 streams for AVIF, the use of nb_streams here and
elsewhere carry any risk for other formats?

> [...]
>
> @@ -6874,14 +6920,23 @@ static int mov_init(AVFormatContext *s)
>
>      /* AVIF output must have exactly one video stream */

This is no longer true.

>      if (mov->mode == MODE_AVIF) {
> -        if (s->nb_streams > 1) {
> -            av_log(s, AV_LOG_ERROR, "AVIF output requires exactly one stream\n");
> +        if (s->nb_streams > 2) {
> +            av_log(s, AV_LOG_ERROR, "AVIF output requires exactly one or two streams\n");
>              return AVERROR(EINVAL);
>          }
> -        if (s->streams[0]->codecpar->codec_type != AVMEDIA_TYPE_VIDEO) {
> -            av_log(s, AV_LOG_ERROR, "AVIF output requires one video stream\n");
> +        if (s->streams[0]->codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
> +            (s->nb_streams > 1 && s->streams[1]->codecpar->codec_type != AVMEDIA_TYPE_VIDEO)) {
> +            av_log(s, AV_LOG_ERROR, "AVIF output supports only video streams\n");
>              return AVERROR(EINVAL);
>          }
> +        if (s->nb_streams > 1) {
> +            const AVPixFmtDescriptor *pixdesc =
> +                av_pix_fmt_desc_get(s->streams[1]->codecpar->format);
> +            if (pixdesc->nb_components != 1) {
> +                av_log(s, AV_LOG_ERROR, "Second stream for AVIF (alpha) output must have exactly one plane\n");
> +                return AVERROR(EINVAL);
> +            }
> +        }
>          s->streams[0]->disposition |= AV_DISPOSITION_DEFAULT;
>      }
>
> @@ -7543,18 +7598,25 @@ static int avif_write_trailer(AVFormatContext *s)
>  {
>      AVIOContext *pb = s->pb;
>      MOVMuxContext *mov = s->priv_data;
> -    int64_t pos_backup, mdat_pos;
> +    int64_t pos_backup, extent_offsets[2];
>      uint8_t *buf;
> -    int buf_size, moov_size;
> +    int buf_size, moov_size, i;
>
>      if (mov->moov_written) return 0;
>
>      mov->is_animated_avif = s->streams[0]->nb_frames > 1;
> +    if (mov->is_animated_avif && s->nb_streams > 1) {
> +        // For animated avif with alpha channel, we need to write a the tref

...write a tref...