[Libav-user] Crash in ff_merge_formats

[David Yuan]

Hi,

I'm trying to transcode some H.264 video to HEVC, and it crashes in function ff_merge_formats()

AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
                                  enum AVMediaType type)
{
    AVFilterFormats *ret = NULL;
    int i, j;
    int alpha1=0, alpha2=0;
    int chroma1=0, chroma2=0;

    if (a == b)
        return a;

    /* Do not lose chroma or alpha in merging.
       It happens if both lists have formats with chroma (resp. alpha), but
       the only formats in common do not have it (e.g. YUV+gray vs.
       RGB+gray): in that case, the merging would select the gray format,
       possibly causing a lossy conversion elsewhere in the graph.
       To avoid that, pretend that there are no common formats to force the
       insertion of a conversion filter. */
    if (type == AVMEDIA_TYPE_VIDEO)
        for (i = 0; i < a->nb_formats; i++)
            for (j = 0; j < b->nb_formats; j++) {
                const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]);
                const AVPixFmtDescriptor *bdesc = av_pix_fmt_desc_get(b->formats[j]);
                alpha2 |= adesc->flags & bdesc->flags & AV_PIX_FMT_FLAG_ALPHA;
                chroma2|= adesc->nb_components > 1 && bdesc->nb_components > 1;  <<======= Access violation on this line.


I suspect I'm missing something when I setup the encoder's context, but couldn't figure out what. Could somebody point out what I'm doing wrong here? A snippet of sample code to do HEVC transcoding would also be appreciated!


           if (dec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
encoder = avcodec_find_encoder_by_name("hevc_nvenc");
if (!encoder) {
av_log(NULL, AV_LOG_FATAL, "HEVC_NVENC encoder not found\n");
return AVERROR_INVALIDDATA;
}

enc_ctx = avcodec_alloc_context3(encoder);
if (!enc_ctx) {
fprintf(stderr, "Could not allocate hevc_nvenc codec context\n");
return -1;
}

//use gloabal headers
enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

//set the profile and preset attributes
ret = av_opt_set(enc_ctx->priv_data, "profile", "main", AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set profile to be main\n");
return ret;
}

ret = av_opt_set(enc_ctx->priv_data, "preset", "hq", AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set preset to hq\n");
return ret;
}

enc_ctx->height = dec_ctx->height;
                enc_ctx->width = dec_ctx->width;
                enc_ctx->sample_aspect_ratio = dec_ctx->sample_aspect_ratio;
enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
enc_ctx->color_range = AVCOL_RANGE_JPEG;

                /* video time_base can be set to whatever is handy and supported by encoder */
                enc_ctx->time_base = dec_ctx->time_base;
out_stream->time_base = dec_ctx->time_base;


thanks!

David
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://ffmpeg.org/pipermail/libav-user/attachments/20170221/080ef475/attachment.html>