[FFmpeg-devel] [PATCH]Check ENOMEM in ff_interleave_add_packet()

[Carl Eugen Hoyos]

Hi!

Attached fixes a possible crash in ff_interleave_add_packet() on OOM.

Please comment, Carl Eugen
-------------- next part --------------
diff --git a/libavformat/audiointerleave.c b/libavformat/audiointerleave.c
index 844112f..922f4a5 100644
--- a/libavformat/audiointerleave.c
+++ b/libavformat/audiointerleave.c
@@ -113,10 +113,13 @@ int ff_audio_rechunk_interleave(AVFormatContext *s, AVPacket *out, AVPacket *pkt
             }
             av_fifo_generic_write(aic->fifo, pkt->data, pkt->size, NULL);
         } else {
+            int ret;
             // rewrite pts and dts to be decoded time line position
             pkt->pts = pkt->dts = aic->dts;
             aic->dts += pkt->duration;
-            ff_interleave_add_packet(s, pkt, compare_ts);
+            ret = ff_interleave_add_packet(s, pkt, compare_ts);
+            if (ret < 0)
+                return ret;
         }
         pkt = NULL;
     }
@@ -125,8 +128,12 @@ int ff_audio_rechunk_interleave(AVFormatContext *s, AVPacket *out, AVPacket *pkt
         AVStream *st = s->streams[i];
         if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
             AVPacket new_pkt;
-            while (ff_interleave_new_audio_packet(s, &new_pkt, i, flush))
-                ff_interleave_add_packet(s, &new_pkt, compare_ts);
+            int ret;
+            while (ff_interleave_new_audio_packet(s, &new_pkt, i, flush)) {
+                ret = ff_interleave_add_packet(s, &new_pkt, compare_ts);
+                if (ret < 0)
+                    return ret;
+            }
         }
     }
 
diff --git a/libavformat/internal.h b/libavformat/internal.h
index 582a2c8..f15940c 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -71,8 +71,9 @@ void ff_program_add_stream_index(AVFormatContext *ac, int progid, unsigned int i
 /**
  * Add packet to AVFormatContext->packet_buffer list, determining its
  * interleaved position using compare() function argument.
+ * @return 0, or < 0 on error
  */
-void ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
+int ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
                               int (*compare)(AVFormatContext *, AVPacket *, AVPacket *));
 
 void ff_read_frame_flush(AVFormatContext *s);
diff --git a/libavformat/utils.c b/libavformat/utils.c
index f52c93d..109fcd4 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -3262,12 +3262,14 @@ int av_write_frame(AVFormatContext *s, AVPacket *pkt)
     return ret;
 }
 
-void ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
+int ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
                               int (*compare)(AVFormatContext *, AVPacket *, AVPacket *))
 {
     AVPacketList **next_point, *this_pktl;
 
     this_pktl = av_mallocz(sizeof(AVPacketList));
+    if (!this_pktl)
+        return AVERROR(ENOMEM);
     this_pktl->pkt= *pkt;
     pkt->destruct= NULL;             // do not free original but only the copy
     av_dup_packet(&this_pktl->pkt);  // duplicate the packet if it uses non-alloced memory
@@ -3296,6 +3298,7 @@ next_non_null:
 
     s->streams[pkt->stream_index]->last_in_packet_buffer=
     *next_point= this_pktl;
+    return 0;
 }
 
 static int ff_interleave_compare_dts(AVFormatContext *s, AVPacket *next, AVPacket *pkt)
@@ -3314,10 +3317,12 @@ int av_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out, AVPacket *pk
     AVPacketList *pktl;
     int stream_count=0, noninterleaved_count=0;
     int64_t delta_dts_max = 0;
-    int i;
+    int i, ret;
 
     if(pkt){
-        ff_interleave_add_packet(s, pkt, ff_interleave_compare_dts);
+        ret = ff_interleave_add_packet(s, pkt, ff_interleave_compare_dts);
+        if (ret < 0)
+            return ret;
     }
 
     for(i=0; i < s->nb_streams; i++) {