[rtmpdump] [PATCH] Add support for building with gnutls with nettle as backend

Martin Storsjo martin at martin.st
Sun Aug 7 10:45:58 CEST 2011 

---
Updated after some initial testing.

 Makefile            |    1 +
 librtmp/Makefile    |    3 +++
 librtmp/dh.h        |   20 +++++++++++++++++++-
 librtmp/handshake.h |   20 ++++++++++++++++++++
 librtmp/hashswf.c   |   11 +++++++++++
 librtmp/rtmp.c      |    4 ++--
 librtmp/rtmp_sys.h  |    2 +-
 7 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6ef5742..0cf41be 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ CRYPTO=OPENSSL
 #CRYPTO=GNUTLS
 LIBZ=-lz
 LIB_GNUTLS=-lgnutls -lgcrypt $(LIBZ)
+LIB_GNUTLS_NETTLE=-lgnutls -lhogweed -lnettle -lgmp $(LIBZ)
 LIB_OPENSSL=-lssl -lcrypto $(LIBZ)
 LIB_POLARSSL=-lpolarssl $(LIBZ)
 CRYPTO_LIB=$(LIB_$(CRYPTO))
diff --git a/librtmp/Makefile b/librtmp/Makefile
index a0125f1..353c6c8 100644
--- a/librtmp/Makefile
+++ b/librtmp/Makefile
@@ -21,14 +21,17 @@ CRYPTO=OPENSSL
 DEF_POLARSSL=-DUSE_POLARSSL
 DEF_OPENSSL=-DUSE_OPENSSL
 DEF_GNUTLS=-DUSE_GNUTLS
+DEF_GNUTLS_NETTLE=-DUSE_GNUTLS_NETTLE
 DEF_=-DNO_CRYPTO
 REQ_GNUTLS=gnutls
+REQ_GNUTLS_NETTLE=gnutls
 REQ_OPENSSL=libssl,libcrypto
 LIBZ=-lz
 LIBS_posix=
 LIBS_darwin=
 LIBS_mingw=-lws2_32 -lwinmm -lgdi32
 LIB_GNUTLS=-lgnutls -lgcrypt $(LIBZ)
+LIB_GNUTLS_NETTLE=-lgnutls -lhogweed -lnettle -lgmp $(LIBZ)
 LIB_OPENSSL=-lssl -lcrypto $(LIBZ)
 LIB_POLARSSL=-lpolarssl $(LIBZ)
 PRIVATE_LIBS=$(LIBS_$(SYS))
diff --git a/librtmp/dh.h b/librtmp/dh.h
index efef0fd..fd60f31 100644
--- a/librtmp/dh.h
+++ b/librtmp/dh.h
@@ -77,7 +77,8 @@ static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
   return 0;
 }
 
-#elif defined(USE_GNUTLS)
+#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE)
+#ifdef USE_GNUTLS
 #include <gcrypt.h>
 typedef gcry_mpi_t MP_t;
 #define MP_new(m)	m = gcry_mpi_new(1)
@@ -92,6 +93,23 @@ typedef gcry_mpi_t MP_t;
 #define MP_bytes(u)	(gcry_mpi_get_nbits(u) + 7) / 8
 #define MP_setbin(u,buf,len)	gcry_mpi_print(GCRYMPI_FMT_USG,buf,len,NULL,u)
 #define MP_getbin(u,buf,len)	gcry_mpi_scan(&u,GCRYMPI_FMT_USG,buf,len,NULL)
+#else
+#include <gmp.h>
+#include <nettle/bignum.h>
+typedef mpz_ptr MP_t;
+#define MP_new(m)	m = malloc(sizeof(*m)); mpz_init2(m, 1)
+#define MP_set_w(mpi, w)	mpz_set_ui(mpi, w)
+#define MP_cmp(u, v)	mpz_cmp(u, v)
+#define MP_set(u, v)	mpz_set(u, v)
+#define MP_sub_w(mpi, w)	mpz_sub_ui(mpi, mpi, w)
+#define MP_cmp_1(mpi)	mpz_cmp_ui(mpi, 1)
+#define MP_modexp(r, y, q, p)	mpz_powm(r, y, q, p)
+#define MP_free(mpi)	mpz_clear(mpi); free(mpi)
+#define MP_gethex(u, hex, res)	u = malloc(sizeof(*u)); mpz_init2(u, 1); res = (mpz_set_str(u, hex, 16) == 0)
+#define MP_bytes(u)	(mpz_sizeinbase(u, 2) + 7) / 8
+#define MP_setbin(u,buf,len)	nettle_mpz_get_str_256(len,buf,u)
+#define MP_getbin(u,buf,len)	u = malloc(sizeof(*u)); mpz_init2(u, 1); nettle_mpz_set_str_256_u(u,len,buf)
+#endif
 
 typedef struct MDH {
   MP_t p;
diff --git a/librtmp/handshake.h b/librtmp/handshake.h
index 98bf3c8..4c2ea7f 100644
--- a/librtmp/handshake.h
+++ b/librtmp/handshake.h
@@ -59,6 +59,26 @@ typedef gcry_cipher_hd_t	RC4_handle;
 #define RC4_encrypt2(h,l,s,d)	gcry_cipher_encrypt(h,(void *)d,l,(void *)s,l)
 #define RC4_free(h)	gcry_cipher_close(h)
 
+#elif defined(USE_GNUTLS_NETTLE)
+#include <nettle/hmac.h>
+#include <nettle/arcfour.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH    32
+#endif
+#undef HMAC_CTX
+#define HMAC_CTX	struct hmac_sha256_ctx
+#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
+#define HMAC_close(ctx)
+
+typedef struct arcfour_ctx*	RC4_handle;
+#define RC4_alloc(h)	*h = malloc(sizeof(struct arcfour_ctx))
+#define RC4_setkey(h,l,k)	arcfour_set_key(h, l, k)
+#define RC4_encrypt(h,l,d)	arcfour_crypt(h,l,(uint8_t *)d,(uint8_t *)d)
+#define RC4_encrypt2(h,l,s,d)	arcfour_crypt(h,l,(uint8_t *)d,(uint8_t *)s)
+#define RC4_free(h)	free(h)
+
 #else	/* USE_OPENSSL */
 #include <openssl/sha.h>
 #include <openssl/hmac.h>
diff --git a/librtmp/hashswf.c b/librtmp/hashswf.c
index 3c56b69..ca2084c 100644
--- a/librtmp/hashswf.c
+++ b/librtmp/hashswf.c
@@ -52,6 +52,17 @@
 #define HMAC_crunch(ctx, buf, len)	gcry_md_write(ctx, buf, len)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; memcpy(dig, gcry_md_read(ctx, 0), dlen)
 #define HMAC_close(ctx)	gcry_md_close(ctx)
+#elif defined(USE_GNUTLS_NETTLE)
+#include <nettle/hmac.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH	32
+#endif
+#undef HMAC_CTX
+#define HMAC_CTX	struct hmac_sha256_ctx
+#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
+#define HMAC_close(ctx)
 #else	/* USE_OPENSSL */
 #include <openssl/ssl.h>
 #include <openssl/sha.h>
diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
index df2cb27..c9d5c83 100644
--- a/librtmp/rtmp.c
+++ b/librtmp/rtmp.c
@@ -34,7 +34,7 @@
 #ifdef CRYPTO
 #ifdef USE_POLARSSL
 #include <polarssl/havege.h>
-#elif defined(USE_GNUTLS)
+#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE)
 #include <gnutls/gnutls.h>
 #else	/* USE_OPENSSL */
 #include <openssl/ssl.h>
@@ -204,7 +204,7 @@ RTMP_TLS_Init()
   /* Do this regardless of NO_SSL, we use havege for rtmpe too */
   RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx));
   havege_init(&RTMP_TLS_ctx->hs);
-#elif defined(USE_GNUTLS) && !defined(NO_SSL)
+#elif (defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE)) && !defined(NO_SSL)
   /* Technically we need to initialize libgcrypt ourselves if
    * we're not going to call gnutls_global_init(). Ignoring this
    * for now.
diff --git a/librtmp/rtmp_sys.h b/librtmp/rtmp_sys.h
index 6a3f215..4958736 100644
--- a/librtmp/rtmp_sys.h
+++ b/librtmp/rtmp_sys.h
@@ -80,7 +80,7 @@ typedef struct tls_ctx {
 #define TLS_shutdown(s)	ssl_close_notify(s)
 #define TLS_close(s)	ssl_free(s); free(s)
 
-#elif defined(USE_GNUTLS)
+#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE)
 #include <gnutls/gnutls.h>
 typedef struct tls_ctx {
 	gnutls_certificate_credentials_t cred;
-- 
1.7.3.1

More information about the rtmpdump mailing list