[FFmpeg-cvslog] Avoid overflows when reading pgm files with maxval != 255 and != 65535.

[Carl Eugen Hoyos]

ffmpeg | branch: master | Carl Eugen Hoyos <cehoyos at ag.or.at> | Sun Jul 28 19:07:06 2013 +0200| [7651c0e49b1f6e66120ef32570a87e3f09224574] | committer: Carl Eugen Hoyos

Avoid overflows when reading pgm files with maxval != 255 and != 65535.

libjpeg v6b apparently does not initialize the most significant bits
to zero when writing 12bit pgm's with maxval 4095.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7651c0e49b1f6e66120ef32570a87e3f09224574
---

 libavcodec/pnmdec.c |   13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/libavcodec/pnmdec.c b/libavcodec/pnmdec.c
index d0c7295..96ed83e 100644
--- a/libavcodec/pnmdec.c
+++ b/libavcodec/pnmdec.c
@@ -35,6 +35,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
     int i, j, n, linesize, h, upgrade = 0, is_mono = 0;
     unsigned char *ptr;
     int components, sample_len, ret;
+    unsigned int maskval = 0;
 
     s->bytestream_start =
     s->bytestream       = (uint8_t *)buf;
@@ -75,8 +76,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
         n = avctx->width;
         components=1;
         sample_len=8;
-        if (s->maxval < 255)
+        if (s->maxval < 255) {
             upgrade = 1;
+            maskval = (2 << av_log2(s->maxval)) - 1;
+        }
         goto do_read;
     case AV_PIX_FMT_GRAY8A:
         n = avctx->width * 2;
@@ -88,8 +91,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
         n = avctx->width * 2;
         components=1;
         sample_len=16;
-        if (s->maxval < 65535)
+        if (s->maxval < 65535) {
             upgrade = 2;
+            maskval = (2 << av_log2(s->maxval)) - 1;
+        }
         goto do_read;
     case AV_PIX_FMT_MONOWHITE:
     case AV_PIX_FMT_MONOBLACK:
@@ -136,11 +141,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
             else if (upgrade == 1) {
                 unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval;
                 for (j = 0; j < n; j++)
-                    ptr[j] = (s->bytestream[j] * f + 64) >> 7;
+                    ptr[j] = ((s->bytestream[j] & maskval) * f + 64) >> 7;
             } else if (upgrade == 2) {
                 unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval;
                 for (j = 0; j < n / 2; j++) {
-                    v = av_be2ne16(((uint16_t *)s->bytestream)[j]);
+                    v = av_be2ne16(((uint16_t *)s->bytestream)[j]) & maskval;
                     ((uint16_t *)ptr)[j] = (v * f + 16384) >> 15;
                 }
             }