[FFmpeg-cvslog] Mark vectors as NAN instead of dereferencing NULL pointers on malloc failure
Michael Niedermayer
git at videolan.org
Fri Jun 12 15:29:58 CEST 2015
ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Fri Jun 12 15:03:49 2015 +0200| [80b5a1e2eee966d17a81a4af0b74865b019447e2] | committer: Michael Niedermayer
Mark vectors as NAN instead of dereferencing NULL pointers on malloc failure
Found-by: Daemon404
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=80b5a1e2eee966d17a81a4af0b74865b019447e2
---
libswscale/utils.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 0c78d75..de0c7f9 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -1654,6 +1654,22 @@ SwsContext *sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat,
return c;
}
+static int isnan_vec(SwsVector *a)
+{
+ int i;
+ for (i=0; i<a->length; i++)
+ if (isnan(a->coeff[i]))
+ return 1;
+ return 0;
+}
+
+static void makenan_vec(SwsVector *a)
+{
+ int i;
+ for (i=0; i<a->length; i++)
+ a->coeff[i] = NAN;
+}
+
SwsFilter *sws_getDefaultFilter(float lumaGBlur, float chromaGBlur,
float lumaSharpen, float chromaSharpen,
float chromaHShift, float chromaVShift,
@@ -1715,6 +1731,12 @@ SwsFilter *sws_getDefaultFilter(float lumaGBlur, float chromaGBlur,
sws_normalizeVec(filter->lumH, 1.0);
sws_normalizeVec(filter->lumV, 1.0);
+ if (isnan_vec(filter->chrH) ||
+ isnan_vec(filter->chrV) ||
+ isnan_vec(filter->lumH) ||
+ isnan_vec(filter->lumV))
+ goto fail;
+
if (verbose)
sws_printVec2(filter->chrH, NULL, AV_LOG_DEBUG);
if (verbose)
@@ -1890,6 +1912,10 @@ static SwsVector *sws_getShiftedVec(SwsVector *a, int shift)
void sws_shiftVec(SwsVector *a, int shift)
{
SwsVector *shifted = sws_getShiftedVec(a, shift);
+ if (!shifted) {
+ makenan_vec(a);
+ return;
+ }
av_free(a->coeff);
a->coeff = shifted->coeff;
a->length = shifted->length;
@@ -1899,6 +1925,10 @@ void sws_shiftVec(SwsVector *a, int shift)
void sws_addVec(SwsVector *a, SwsVector *b)
{
SwsVector *sum = sws_sumVec(a, b);
+ if (!sum) {
+ makenan_vec(a);
+ return;
+ }
av_free(a->coeff);
a->coeff = sum->coeff;
a->length = sum->length;
@@ -1908,6 +1938,10 @@ void sws_addVec(SwsVector *a, SwsVector *b)
void sws_subVec(SwsVector *a, SwsVector *b)
{
SwsVector *diff = sws_diffVec(a, b);
+ if (!diff) {
+ makenan_vec(a);
+ return;
+ }
av_free(a->coeff);
a->coeff = diff->coeff;
a->length = diff->length;
@@ -1917,6 +1951,10 @@ void sws_subVec(SwsVector *a, SwsVector *b)
void sws_convVec(SwsVector *a, SwsVector *b)
{
SwsVector *conv = sws_getConvVec(a, b);
+ if (!conv) {
+ makenan_vec(a);
+ return;
+ }
av_free(a->coeff);
a->coeff = conv->coeff;
a->length = conv->length;
More information about the ffmpeg-cvslog
mailing list