[FFmpeg-cvslog] avcodec/diracdec: Check for hpel_base allocation failure

[Michael Niedermayer]

ffmpeg | branch: release/2.5 | Michael Niedermayer <michael at niedermayer.cc> | Sat Jul 18 17:55:19 2015 +0200| [c72c1b871602079f4d867bf5c04fa0a3f51ec7c2] | committer: Michael Niedermayer

avcodec/diracdec: Check for hpel_base allocation failure

Fixes null pointer dereference
Fixes: signal_sigsegv_b02a96_280_RL_420p_ffdirac.drc with memlimit of 67108864

Found-by: Samuel Groß, Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 1c5b712c0a643a039d6f34269b4102de313a050a)

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c72c1b871602079f4d867bf5c04fa0a3f51ec7c2
---

 libavcodec/diracdec.c |   16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c
index 1bfe6be..6b93d86 100644
--- a/libavcodec/diracdec.c
+++ b/libavcodec/diracdec.c
@@ -1561,7 +1561,7 @@ static void select_dsp_funcs(DiracContext *s, int width, int height, int xblen,
     }
 }
 
-static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, int width, int height)
+static int interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, int width, int height)
 {
     /* chroma allocates an edge of 8 when subsampled
        which for 4:2:2 means an h edge of 16 and v edge of 8
@@ -1573,11 +1573,14 @@ static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, in
 
     /* no need for hpel if we only have fpel vectors */
     if (!s->mv_precision)
-        return;
+        return 0;
 
     for (i = 1; i < 4; i++) {
         if (!ref->hpel_base[plane][i])
             ref->hpel_base[plane][i] = av_malloc((height+2*edge) * ref->avframe->linesize[plane] + 32);
+        if (!ref->hpel_base[plane][i]) {
+            return AVERROR(ENOMEM);
+        }
         /* we need to be 16-byte aligned even for chroma */
         ref->hpel[plane][i] = ref->hpel_base[plane][i] + edge*ref->avframe->linesize[plane] + 16;
     }
@@ -1591,6 +1594,8 @@ static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, in
         s->mpvencdsp.draw_edges(ref->hpel[plane][3], ref->avframe->linesize[plane], width, height, edge, edge, EDGE_TOP | EDGE_BOTTOM);
     }
     ref->interpolated[plane] = 1;
+
+    return 0;
 }
 
 /**
@@ -1640,8 +1645,11 @@ static int dirac_decode_frame_internal(DiracContext *s)
 
             select_dsp_funcs(s, p->width, p->height, p->xblen, p->yblen);
 
-            for (i = 0; i < s->num_refs; i++)
-                interpolate_refplane(s, s->ref_pics[i], comp, p->width, p->height);
+            for (i = 0; i < s->num_refs; i++) {
+                int ret = interpolate_refplane(s, s->ref_pics[i], comp, p->width, p->height);
+                if (ret < 0)
+                    return ret;
+            }
 
             memset(s->mctmp, 0, 4*p->yoffset*p->stride);