[FFmpeg-devel] [PATCH 3/4] avformat/rmdec: Use 64bit for intermediate for DEINT_ID_INT4

[James Almer]

On 4/16/2021 4:04 PM, Michael Niedermayer wrote:
> On Thu, Apr 15, 2021 at 06:22:10PM -0300, James Almer wrote:
>> On 4/15/2021 5:44 PM, Michael Niedermayer wrote:
>>> Fixes: runtime error: signed integer overflow: 65312 * 65535 cannot be represented in type 'int'
>>> Fixes: 32832/clusterfuzz-testcase-minimized-ffmpeg_dem_RM_fuzzer-4817710040088576
>>>
>>> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
>>> Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
>>> ---
>>>    libavformat/rmdec.c | 4 ++--
>>>    1 file changed, 2 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c
>>> index fc3bff4859..af032ed90a 100644
>>> --- a/libavformat/rmdec.c
>>> +++ b/libavformat/rmdec.c
>>> @@ -269,9 +269,9 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
>>>            case DEINT_ID_INT4:
>>>                if (ast->coded_framesize > ast->audio_framesize ||
>>>                    sub_packet_h <= 1 ||
>>> -                ast->coded_framesize * sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize)
>>> +                ast->coded_framesize * (uint64_t)sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize)
>>
>> This check seems superfluous with the one below right after it.
>> ast->coded_framesize * sub_packet_h must be equal to 2 *
>> ast->audio_framesize. It can be removed.
>>
>>>                    return AVERROR_INVALIDDATA;
>>> -            if (ast->coded_framesize * sub_packet_h != 2*ast->audio_framesize) {
>>> +            if (ast->coded_framesize * (uint64_t)sub_packet_h != 2*ast->audio_framesize) {
>>>                    avpriv_request_sample(s, "mismatching interleaver parameters");
>>>                    return AVERROR_INVALIDDATA;
>>>                }
>>
>> How about something like
>>
>>> diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c
>>> index fc3bff4859..09880ee3fe 100644
>>> --- a/libavformat/rmdec.c
>>> +++ b/libavformat/rmdec.c
>>> @@ -269,7 +269,7 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
>>>           case DEINT_ID_INT4:
>>>               if (ast->coded_framesize > ast->audio_framesize ||
>>>                   sub_packet_h <= 1 ||
>>> -                ast->coded_framesize * sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize)
>>> +                ast->audio_framesize > INT_MAX / sub_packet_h)
>>>                   return AVERROR_INVALIDDATA;
>>>               if (ast->coded_framesize * sub_packet_h != 2*ast->audio_framesize) {
>>>                   avpriv_request_sample(s, "mismatching interleaver parameters");
>>
>> Instead?
> 
> The 2 if() execute different things, the 2nd requests a sample, the first
> not. I think this suggestion would change when we request a sample

Why are we returning INVALIDDATA after requesting a sample, for that 
matter? If it's considered an invalid scenario, do we really need a sample?

In any case, if you don't want more files where "ast->coded_framesize * 
sub_packet_h != 2*ast->audio_framesize" would print a sample request, 
then maybe something like the following could be used instead?

> diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c
> index fc3bff4859..10c1699a81 100644
> --- a/libavformat/rmdec.c
> +++ b/libavformat/rmdec.c
> @@ -269,6 +269,7 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
>          case DEINT_ID_INT4:
>              if (ast->coded_framesize > ast->audio_framesize ||
>                  sub_packet_h <= 1 ||
> +                ast->audio_framesize > INT_MAX / sub_packet_h ||
>                  ast->coded_framesize * sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize)
>                  return AVERROR_INVALIDDATA;
>              if (ast->coded_framesize * sub_packet_h != 2*ast->audio_framesize) {
> @@ -278,12 +279,16 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
>              break;
>          case DEINT_ID_GENR:
>              if (ast->sub_packet_size <= 0 ||
> +                ast->audio_framesize > INT_MAX / sub_packet_h ||
>                  ast->sub_packet_size > ast->audio_framesize)
>                  return AVERROR_INVALIDDATA;
>              if (ast->audio_framesize % ast->sub_packet_size)
>                  return AVERROR_INVALIDDATA;
>              break;
>          case DEINT_ID_SIPR:
> +            if (ast->audio_framesize > INT_MAX / sub_packet_h)
> +                return AVERROR_INVALIDDATA;
> +            break;
>          case DEINT_ID_INT0:
>          case DEINT_ID_VBRS:
>          case DEINT_ID_VBRF:
> @@ -296,7 +301,6 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
>              ast->deint_id == DEINT_ID_GENR ||
>              ast->deint_id == DEINT_ID_SIPR) {
>              if (st->codecpar->block_align <= 0 ||
> -                ast->audio_framesize * (uint64_t)sub_packet_h > (unsigned)INT_MAX ||
>                  ast->audio_framesize * sub_packet_h < st->codecpar->block_align)
>                  return AVERROR_INVALIDDATA;
>              if (av_new_packet(&ast->pkt, ast->audio_framesize * sub_packet_h) < 0)

Same amount of checks for all three deint ids, and no integer casting to 
prevent overflows.