[FFmpeg-devel] [PATCH] mediacodecdec_h264: properly convert extradata to annex-b

[Benoit Fouet]

Hi,

On 04/07/2016 10:12, Matthieu Bouron wrote:
> From: Matthieu Bouron <matthieu.bouron at stupeflix.com>
>
> H264ParamSets has its SPS/PPS stored raw (SODB) and needs to be
> converted to NAL units before sending them to MediaCodec.
>
> This patch adds the missing convertion of the SPS/PPS from SOBP to RBSP
> which makes the resulting NAL units correct.
>
> Fixes codec initialization on Nexus 4 and Nexus 7.
> ---
>   libavcodec/mediacodecdec_h264.c | 69 +++++++++++++++++++++++++++++++++--------
>   1 file changed, 56 insertions(+), 13 deletions(-)
>
> diff --git a/libavcodec/mediacodecdec_h264.c b/libavcodec/mediacodecdec_h264.c
> index 0664e49..f07c7cc 100644
> --- a/libavcodec/mediacodecdec_h264.c
> +++ b/libavcodec/mediacodecdec_h264.c
> @@ -65,6 +65,54 @@ static av_cold int mediacodec_decode_close(AVCodecContext *avctx)
>       return 0;
>   }
>   
> +static int h264_ps_to_nalu(const uint8_t *src, int src_size, uint8_t **out, int *out_size)
> +{
> +    int i;
> +    int ret = 0;
> +    uint8_t *p = NULL;
> +    static const uint8_t nalu_header[] = { 0x00, 0x00, 0x00, 0x01 };
> +
> +    if (!out) {
> +        return AVERROR(EINVAL);
> +    }
> +

out_size should also be checked

> +    *out_size = sizeof(nalu_header) + src_size;
> +    *out = p = av_malloc(*out_size);
> +    if (!*out) {
> +        return AVERROR(ENOMEM);
> +    }
> +

nit: the size affectation could be done once the allocation is OK.

> +    memcpy(p, nalu_header, sizeof(nalu_header));
> +    memcpy(p + sizeof(nalu_header), src, src_size);
> +
> +    /* Escape 0x00, 0x00, 0x0{0-3} pattern */
> +    for (i = 4; i < *out_size; i++) {
> +        if (i < *out_size - 3 &&
> +            p[i + 0] == 0 &&
> +            p[i + 1] == 0 &&
> +            p[i + 2] <= 3) {
> +            uint8_t *new;
> +
> +            *out_size += 1;
> +            new = av_realloc(*out, *out_size);
> +            if (!new) {
> +                ret = AVERROR(ENOMEM);
> +                goto done;
> +            }
> +            *out = p = new;
> +
> +            i = i + 3;
> +            memmove(p + i, p + i - 1, *out_size - i);
> +            p[i - 1] = 0x03;
> +        }
> +    }
> +done:
> +    if (ret < 0)
> +        av_freep(out);
> +
> +    return ret;
> +}
> +
>   static av_cold int mediacodec_decode_init(AVCodecContext *avctx)
>   {
>       int i;
> @@ -112,24 +160,19 @@ static av_cold int mediacodec_decode_init(AVCodecContext *avctx)
>       }
>   
>       if (pps && sps) {
> -        static const uint8_t nal_headers[] = { 0x00, 0x00, 0x00, 0x01 };
> -
>           uint8_t *data = NULL;
> -        size_t data_size = sizeof(nal_headers) + FFMAX(sps->data_size, pps->data_size);
> +        size_t data_size = 0;
>   
> -        data = av_mallocz(data_size);
> -        if (!data) {
> -            ret = AVERROR(ENOMEM);
> +        if ((ret = h264_ps_to_nalu(sps->data, sps->data_size, &data, &data_size)) < 0) {
>               goto done;
>           }
> +        ff_AMediaFormat_setBuffer(format, "csd-0", (void*)data, data_size);
> +        av_freep(&data);
>   
> -        memcpy(data, nal_headers, sizeof(nal_headers));
> -        memcpy(data + sizeof(nal_headers), sps->data, sps->data_size);
> -        ff_AMediaFormat_setBuffer(format, "csd-0", (void*)data, sizeof(nal_headers) + sps->data_size);
> -
> -        memcpy(data + sizeof(nal_headers), pps->data, pps->data_size);
> -        ff_AMediaFormat_setBuffer(format, "csd-1", (void*)data, sizeof(nal_headers) + pps->data_size);
> -
> +        if ((ret = h264_ps_to_nalu(pps->data, pps->data_size, &data, &data_size)) < 0) {
> +            goto done;
> +        }
> +        ff_AMediaFormat_setBuffer(format, "csd-1", (void*)data, data_size);
>           av_freep(&data);
>       } else {
>           av_log(avctx, AV_LOG_ERROR, "Could not extract PPS/SPS from extradata");

LGTM otherwise

-- 
Ben