[FFmpeg-cvslog] rtmpdh: Create sufficiently long private keys for gcrypt/nettle

[Martin Storsjö]

ffmpeg | branch: master | Martin Storsjö <martin at martin.st> | Thu May 28 11:42:44 2015 +0300| [e9e86d9ef637f5a600c76b352ffe5a82b71b25d1] | committer: Martin Storsjö

rtmpdh: Create sufficiently long private keys for gcrypt/nettle

There was a misunderstanding betewen bits and bytes for the parameter
value for generating random big numbers.

Signed-off-by: Martin Storsjö <martin at martin.st>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e9e86d9ef637f5a600c76b352ffe5a82b71b25d1
---

 libavformat/rtmpdh.c |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libavformat/rtmpdh.c b/libavformat/rtmpdh.c
index b73d987..5cc66c9 100644
--- a/libavformat/rtmpdh.c
+++ b/libavformat/rtmpdh.c
@@ -81,12 +81,12 @@
             ret = 1;                                \
     } while (0)
 #define bn_modexp(bn, y, q, p)      mpz_powm(bn, y, q, p)
-#define bn_random(bn, num_bytes)                    \
+#define bn_random(bn, num_bits)                     \
     do {                                            \
         gmp_randstate_t rs;                         \
         gmp_randinit_mt(rs);                        \
         gmp_randseed_ui(rs, av_get_random_seed());  \
-        mpz_urandomb(bn, rs, num_bytes);            \
+        mpz_urandomb(bn, rs, num_bits);             \
         gmp_randclear(rs);                          \
     } while (0)
 #elif CONFIG_GCRYPT
@@ -102,7 +102,7 @@
 #define bn_bin2bn(bn, buf, len)     gcry_mpi_scan(&bn, GCRYMPI_FMT_USG, buf, len, NULL)
 #define bn_hex2bn(bn, buf, ret)     ret = (gcry_mpi_scan(&bn, GCRYMPI_FMT_HEX, buf, 0, 0) == 0)
 #define bn_modexp(bn, y, q, p)      gcry_mpi_powm(bn, y, q, p)
-#define bn_random(bn, num_bytes)    gcry_mpi_randomize(bn, num_bytes, GCRY_WEAK_RANDOM)
+#define bn_random(bn, num_bits)     gcry_mpi_randomize(bn, num_bits, GCRY_WEAK_RANDOM)
 #endif
 
 #define MAX_BYTES 18000
@@ -120,7 +120,7 @@ static FFBigNum dh_generate_key(FF_DH *dh)
     bn_new(dh->priv_key);
     if (!dh->priv_key)
         return NULL;
-    bn_random(dh->priv_key, num_bytes);
+    bn_random(dh->priv_key, 8 * num_bytes);
 
     bn_new(dh->pub_key);
     if (!dh->pub_key) {