[FFmpeg-devel] [PATCH] libvpxenc: avoid rollover in settings conversion

[James Zern]

When converting bitrate values to percentages or buffer sizes to
milliseconds as required for libvpx avoid rolling over integer values
by using 64-bit math.
In these cases output quality would have been adversely effected. As
an example '-vb 2.2M -maxrate 22M' will roll
rc_2pass_vbr_maxsection_pct (visible with -loglevel debug).
---
 libavcodec/libvpxenc.c |    8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)
-------------- next part --------------
diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
index 4e16c72..d0838ba 100644
--- a/libavcodec/libvpxenc.c
+++ b/libavcodec/libvpxenc.c
@@ -244,17 +244,17 @@ static av_cold int vp8_init(AVCodecContext *avctx)
     //0-100 (0 => CBR, 100 => VBR)
     enccfg.rc_2pass_vbr_bias_pct           = round(avctx->qcompress * 100);
     enccfg.rc_2pass_vbr_minsection_pct     =
-        avctx->rc_min_rate * 100 / avctx->bit_rate;
+        avctx->rc_min_rate * 100LL / avctx->bit_rate;
     if (avctx->rc_max_rate)
         enccfg.rc_2pass_vbr_maxsection_pct =
-            avctx->rc_max_rate * 100 / avctx->bit_rate;
+            avctx->rc_max_rate * 100LL / avctx->bit_rate;
 
     if (avctx->rc_buffer_size)
         enccfg.rc_buf_sz         =
-            avctx->rc_buffer_size * 1000 / avctx->bit_rate;
+            avctx->rc_buffer_size * 1000LL / avctx->bit_rate;
     if (avctx->rc_initial_buffer_occupancy)
         enccfg.rc_buf_initial_sz =
-            avctx->rc_initial_buffer_occupancy * 1000 / avctx->bit_rate;
+            avctx->rc_initial_buffer_occupancy * 1000LL / avctx->bit_rate;
     enccfg.rc_buf_optimal_sz     = enccfg.rc_buf_sz * 5 / 6;
 
     //_enc_init() will balk if kf_min_dist differs from max w/VPX_KF_AUTO