[FFmpeg-cvslog] avfilter/vf_convolution: fix undefined behaviour

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sun Oct 13 18:37:16 2019 +0200| [99f8d32129dd233d4eb2efa44678a0bc44869f23] | committer: Paul B Mahol

avfilter/vf_convolution: fix undefined behaviour

Fixes #8263

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=99f8d32129dd233d4eb2efa44678a0bc44869f23
---

 libavfilter/vf_convolution.c | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/libavfilter/vf_convolution.c b/libavfilter/vf_convolution.c
index f29df38a20..d022f1a04a 100644
--- a/libavfilter/vf_convolution.c
+++ b/libavfilter/vf_convolution.c
@@ -115,10 +115,10 @@ static void filter16_prewitt(uint8_t *dstp, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * -1 +
-                   AV_RN16A(&c[6][2 * x]) *  1 + AV_RN16A(&c[7][2 * x]) *  1 + AV_RN16A(&c[8][2 * x]) *  1;
-        int sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -1 +
-                   AV_RN16A(&c[5][2 * x]) *  1 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) *  1;
+        float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * -1 +
+                     AV_RN16A(&c[6][2 * x]) *  1 + AV_RN16A(&c[7][2 * x]) *  1 + AV_RN16A(&c[8][2 * x]) *  1;
+        float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -1 +
+                     AV_RN16A(&c[5][2 * x]) *  1 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) *  1;
 
         dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
     }
@@ -133,8 +133,8 @@ static void filter16_roberts(uint8_t *dstp, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = AV_RN16A(&c[0][2 * x]) *  1 + AV_RN16A(&c[1][2 * x]) * -1;
-        int sumb = AV_RN16A(&c[4][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -1;
+        float suma = AV_RN16A(&c[0][2 * x]) *  1 + AV_RN16A(&c[1][2 * x]) * -1;
+        float sumb = AV_RN16A(&c[4][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -1;
 
         dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
     }
@@ -149,10 +149,10 @@ static void filter16_sobel(uint8_t *dstp, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +
-                   AV_RN16A(&c[6][2 * x]) *  1 + AV_RN16A(&c[7][2 * x]) *  2 + AV_RN16A(&c[8][2 * x]) *  1;
-        int sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -2 +
-                   AV_RN16A(&c[5][2 * x]) *  2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) *  1;
+        float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +
+                     AV_RN16A(&c[6][2 * x]) *  1 + AV_RN16A(&c[7][2 * x]) *  2 + AV_RN16A(&c[8][2 * x]) *  1;
+        float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) *  1 + AV_RN16A(&c[3][2 * x]) * -2 +
+                     AV_RN16A(&c[5][2 * x]) *  2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) *  1;
 
         dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
     }
@@ -169,10 +169,10 @@ static void filter_prewitt(uint8_t *dst, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = c0[x] * -1 + c1[x] * -1 + c2[x] * -1 +
-                   c6[x] *  1 + c7[x] *  1 + c8[x] *  1;
-        int sumb = c0[x] * -1 + c2[x] *  1 + c3[x] * -1 +
-                   c5[x] *  1 + c6[x] * -1 + c8[x] *  1;
+        float suma = c0[x] * -1 + c1[x] * -1 + c2[x] * -1 +
+                     c6[x] *  1 + c7[x] *  1 + c8[x] *  1;
+        float sumb = c0[x] * -1 + c2[x] *  1 + c3[x] * -1 +
+                     c5[x] *  1 + c6[x] * -1 + c8[x] *  1;
 
         dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
     }
@@ -186,8 +186,8 @@ static void filter_roberts(uint8_t *dst, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = c[0][x] *  1 + c[1][x] * -1;
-        int sumb = c[4][x] *  1 + c[3][x] * -1;
+        float suma = c[0][x] *  1 + c[1][x] * -1;
+        float sumb = c[4][x] *  1 + c[3][x] * -1;
 
         dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
     }
@@ -204,10 +204,10 @@ static void filter_sobel(uint8_t *dst, int width,
     int x;
 
     for (x = 0; x < width; x++) {
-        int suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +
-                   c6[x] *  1 + c7[x] *  2 + c8[x] *  1;
-        int sumb = c0[x] * -1 + c2[x] *  1 + c3[x] * -2 +
-                   c5[x] *  2 + c6[x] * -1 + c8[x] *  1;
+        float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +
+                     c6[x] *  1 + c7[x] *  2 + c8[x] *  1;
+        float sumb = c0[x] * -1 + c2[x] *  1 + c3[x] * -2 +
+                     c5[x] *  2 + c6[x] * -1 + c8[x] *  1;
 
         dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
     }