January 2012 - rtmpdump - lists.mplayerhq.hu

Re: [rtmpdump] Proposal: dynamically loadable plugins
by Antti Ajanki 05 Mar '12

05 Mar '12

Hi, These patches implement the plugin API I proposed in my previous mail. There is also a slightly modified version of the callback patch included. Thanks for Adam for the comments. The same patches are also available here: git clone http://users.tkk.fi/~aajanki/rtmpdump-yle/rtmpdump.git This has been tested only on Linux. It also cross-compiles on mingw, but I haven't tried running it on Windows. The patches add two new dependencies: libltdl for cross-platform dynamic library loading support and pthreads for synchronizing the loading of the plugins. Here is a plugin (together with fully patches rtmpdump sources) that uses the API: http://users.tkk.fi/~aajanki/rtmpdump-yle/yle-dl-1.99.1.tar.gz The plugin does custom authentication for http://areena-beta.yle.fi/ng/areena site. An example command for testing the plugin: rtmpdump -r "rtmp://flashu.yle.fi/mediakanta//x swfUrl=http://areena-beta.yle.fi/static/player/1.1/flowplayer/flowplayer.cluster-3.2.3.swf tcUrl=rtmp://flashu.yle.fi/mediakanta/ pageUrl=http://areena-beta.yle.fi/ng/areena/tv/1370427/#/play yle=2f4d1d102a34462696ad23573adfd296" -o video.flv Antti

4 5

[PATCH] Add support for building with gnutls with nettle as backend
by Martin Storsjo 27 Feb '12

27 Feb '12

--- Since gnutls 2.12, they default to nettle instead of gcrypt as default crypto backend. Since librtmp uses some of the crypto functions directly, this needs updates here. Implemented this by adding another CRYPTO option, GNUTLS_NETTLE. This is totally untested, since I don't really have access to any such rtmp streams that require use of the crypto libraries. Makefile | 1 + librtmp/Makefile | 3 +++ librtmp/dh.h | 20 +++++++++++++++++++- librtmp/handshake.h | 20 ++++++++++++++++++++ librtmp/hashswf.c | 11 +++++++++++ librtmp/rtmp.c | 4 ++-- librtmp/rtmp_sys.h | 2 +- 7 files changed, 57 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6ef5742..0cf41be 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ CRYPTO=OPENSSL #CRYPTO=GNUTLS LIBZ=-lz LIB_GNUTLS=-lgnutls -lgcrypt $(LIBZ) +LIB_GNUTLS_NETTLE=-lgnutls -lhogweed -lnettle -lgmp $(LIBZ) LIB_OPENSSL=-lssl -lcrypto $(LIBZ) LIB_POLARSSL=-lpolarssl $(LIBZ) CRYPTO_LIB=$(LIB_$(CRYPTO)) diff --git a/librtmp/Makefile b/librtmp/Makefile index a0125f1..353c6c8 100644 --- a/librtmp/Makefile +++ b/librtmp/Makefile @@ -21,14 +21,17 @@ CRYPTO=OPENSSL DEF_POLARSSL=-DUSE_POLARSSL DEF_OPENSSL=-DUSE_OPENSSL DEF_GNUTLS=-DUSE_GNUTLS +DEF_GNUTLS_NETTLE=-DUSE_GNUTLS_NETTLE DEF_=-DNO_CRYPTO REQ_GNUTLS=gnutls +REQ_GNUTLS_NETTLE=gnutls REQ_OPENSSL=libssl,libcrypto LIBZ=-lz LIBS_posix= LIBS_darwin= LIBS_mingw=-lws2_32 -lwinmm -lgdi32 LIB_GNUTLS=-lgnutls -lgcrypt $(LIBZ) +LIB_GNUTLS_NETTLE=-lgnutls -lhogweed -lnettle -lgmp $(LIBZ) LIB_OPENSSL=-lssl -lcrypto $(LIBZ) LIB_POLARSSL=-lpolarssl $(LIBZ) PRIVATE_LIBS=$(LIBS_$(SYS)) diff --git a/librtmp/dh.h b/librtmp/dh.h index efef0fd..5d0739f 100644 --- a/librtmp/dh.h +++ b/librtmp/dh.h @@ -77,7 +77,8 @@ static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh) return 0; } -#elif defined(USE_GNUTLS) +#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE) +#ifdef USE_GNUTLS #include <gcrypt.h> typedef gcry_mpi_t MP_t; #define MP_new(m) m = gcry_mpi_new(1) @@ -92,6 +93,23 @@ typedef gcry_mpi_t MP_t; #define MP_bytes(u) (gcry_mpi_get_nbits(u) + 7) / 8 #define MP_setbin(u,buf,len) gcry_mpi_print(GCRYMPI_FMT_USG,buf,len,NULL,u) #define MP_getbin(u,buf,len) gcry_mpi_scan(&u,GCRYMPI_FMT_USG,buf,len,NULL) +#else +#include <gmp.h> +#include <nettle/bignum.h> +typedef mpz_ptr MP_t; +#define MP_new(m) m = malloc(sizeof(*m)); mpz_init2(m, 1) +#define MP_set_w(mpi, w) mpz_set_ui(mpi, w) +#define MP_cmp(u, v) mpz_cmp(u, v) +#define MP_set(u, v) mpz_set(u, v) +#define MP_sub_w(mpi, w) mpz_sub_ui(mpi, mpi, w) +#define MP_cmp_1(mpi) mpz_cmp_ui(mpi, 1) +#define MP_modexp(r, y, q, p) mpz_powm(r, y, q, p) +#define MP_free(mpi) free(mpi) +#define MP_gethex(u, hex, res) res = (mpz_set_str(u, hex, 16) == 0) +#define MP_bytes(u) mpz_sizeinbase(u, 8) +#define MP_setbin(u,buf,len) nettle_mpz_get_str_256(len,buf,u) +#define MP_getbin(u,buf,len) nettle_mpz_set_str_256_u(u,len,buf) +#endif typedef struct MDH { MP_t p; diff --git a/librtmp/handshake.h b/librtmp/handshake.h index 98bf3c8..354c419 100644 --- a/librtmp/handshake.h +++ b/librtmp/handshake.h @@ -59,6 +59,26 @@ typedef gcry_cipher_hd_t RC4_handle; #define RC4_encrypt2(h,l,s,d) gcry_cipher_encrypt(h,(void *)d,l,(void *)s,l) #define RC4_free(h) gcry_cipher_close(h) +#elif defined(USE_GNUTLS_NETTLE) +#include <nettle/hmac.h> +#include <nettle/arcfour.h> +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif +#undef HMAC_CTX +#define HMAC_CTX struct hmac_sha256_ctx +#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) +#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_close(ctx) + +typedef struct arcfour_ctx* RC4_handle; +#define RC4_alloc(h) *h = malloc(sizeof(struct arcfour_ctx)) +#define RC4_setkey(h,l,k) arcfour_set_key(h, l, k) +#define RC4_encrypt(h,l,d) arcfour_crypt(h,l,(uint8_t *)d,(uint8_t *)d) +#define RC4_encrypt2(h,l,s,d) arcfour_crypt(h,l,(uint8_t *)s,(uint8_t *)d) +#define RC4_free(h) free(h) + #else /* USE_OPENSSL */ #include <openssl/sha.h> #include <openssl/hmac.h> diff --git a/librtmp/hashswf.c b/librtmp/hashswf.c index 3c56b69..ca2084c 100644 --- a/librtmp/hashswf.c +++ b/librtmp/hashswf.c @@ -52,6 +52,17 @@ #define HMAC_crunch(ctx, buf, len) gcry_md_write(ctx, buf, len) #define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; memcpy(dig, gcry_md_read(ctx, 0), dlen) #define HMAC_close(ctx) gcry_md_close(ctx) +#elif defined(USE_GNUTLS_NETTLE) +#include <nettle/hmac.h> +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif +#undef HMAC_CTX +#define HMAC_CTX struct hmac_sha256_ctx +#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) +#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_close(ctx) #else /* USE_OPENSSL */ #include <openssl/ssl.h> #include <openssl/sha.h> diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c index df2cb27..c9d5c83 100644 --- a/librtmp/rtmp.c +++ b/librtmp/rtmp.c @@ -34,7 +34,7 @@ #ifdef CRYPTO #ifdef USE_POLARSSL #include <polarssl/havege.h> -#elif defined(USE_GNUTLS) +#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE) #include <gnutls/gnutls.h> #else /* USE_OPENSSL */ #include <openssl/ssl.h> @@ -204,7 +204,7 @@ RTMP_TLS_Init() /* Do this regardless of NO_SSL, we use havege for rtmpe too */ RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx)); havege_init(&RTMP_TLS_ctx->hs); -#elif defined(USE_GNUTLS) && !defined(NO_SSL) +#elif (defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE)) && !defined(NO_SSL) /* Technically we need to initialize libgcrypt ourselves if * we're not going to call gnutls_global_init(). Ignoring this * for now. diff --git a/librtmp/rtmp_sys.h b/librtmp/rtmp_sys.h index 6a3f215..4958736 100644 --- a/librtmp/rtmp_sys.h +++ b/librtmp/rtmp_sys.h @@ -80,7 +80,7 @@ typedef struct tls_ctx { #define TLS_shutdown(s) ssl_close_notify(s) #define TLS_close(s) ssl_free(s); free(s) -#elif defined(USE_GNUTLS) +#elif defined(USE_GNUTLS) || defined(USE_GNUTLS_NETTLE) #include <gnutls/gnutls.h> typedef struct tls_ctx { gnutls_certificate_credentials_t cred; -- 1.7.3.1

2 8

Re: [rtmpdump] rtmpdump not working with dailymotion
by chapun67＠gmail.com 27 Jan '12

27 Jan '12

> But there is another problem now: the transfer keeps being interrupted > after > a few percents, and restarts at an earlier point. That makes the overall > progress slow as hell: the SD version could be finished, but the HD720p > version seems to make no progress at all. try use option (-v) E:\Мои документы\STREAM\rtmpdump-2.3-windows\rtmpdump-2.3\rtmpexplorer>rtmpdump -r "rtmpe://vod-02.dailymotion.com:1935/video" -a "video?auth=1327106757-fdb09df 9ce042f56745891d07125e25d" -f "WIN 10,3,183,10" -W "http://static1.dmcdn.net/fla sh/dmplayerv4/dmplayer-prod.swf.vc5e6c6332d98bade2" -p "http://www.dailymotion.c om/video/xnqokq_nisemonogatari-ep01-remplacement_shortfilms" -y "mp4:470/678/398 76074:mp4_h264_aac_hq.mp4?auth=1327106757-fdb09df9ce042f56745891d07125e25d" -o 3 9876074_mp4_h264_aac_hq.flv -v RTMPDump v2.4 GIT-2011-12-22 (Compiled by KSV) (c) 2010 Andrej Stepanchuk, Howard Chu, The Flvstreamer Team; license: GPL Connecting ... WARNING: Trying different position for server digest! INFO: Connected... Starting Live Stream INFO: Metadata: INFO: trackinfo: INFO: timescale 24.00 INFO: length 35045.00 INFO: language eng INFO: sampledescription: INFO: sampletype avc1 INFO: timescale 44100.00 INFO: length 64391168.00 INFO: language eng INFO: sampledescription: INFO: sampletype mp4a INFO: audiochannels 2.00 INFO: audiosamplerate 44100.00 INFO: videoframerate 24.00 INFO: aacaot 2.00 INFO: avclevel 31.00 INFO: avcprofile 100.00 INFO: audiocodecid mp4a INFO: videocodecid avc1 INFO: width 848.00 INFO: height 480.00 INFO: frameWidth 848.00 INFO: frameHeight 480.00 INFO: displayWidth 848.00 INFO: displayHeight 480.00 INFO: framerate 24.00 INFO: moovposition 40.00 INFO: duration 1460.21 9279.450 kB / 96.83 sec (6.6%)

5 5

Re: [rtmpdump] rtmpdump not working with dailymotion
by chapun67＠gmail.com 21 Jan '12

21 Jan '12

try use option -v -r "rtmpe://vod-02.dailymotion.com:1935/video" -a "video?auth=1327107928-5b1d8ce 124456909db9ac1eb6b66841d" -f "WIN 10,3,183,10" -W "http://static1.dmcdn.net/fla sh/dmplayerv4/dmplayer-prod.swf.vc5e6c6332d98bade2" -p "http://www.dailymotion.c om/video/xnqokq_nisemonogatari-ep01-remplacement_shortfilms" -y "mp4:470/678/398 76074:mp4_h264_aac_hq.mp4?auth=1327107928-5b1d8ce124456909db9ac1eb6b66841d" -o 3 9876074_mp4_h264_aac_hq.flv -v RTMPDump v2.4 (c) 2010 Andrej Stepanchuk, Howard Chu, The Flvstreamer Team; license: GPL Connecting ... WARNING: Trying different position for server digest! INFO: Connected... Starting Live Stream INFO: Metadata: INFO: trackinfo: INFO: timescale 24.00 INFO: length 35045.00 INFO: language eng INFO: sampledescription: INFO: sampletype avc1 INFO: timescale 44100.00 INFO: length 64391168.00 INFO: language eng INFO: sampledescription: INFO: sampletype mp4a INFO: audiochannels 2.00 INFO: audiosamplerate 44100.00 INFO: videoframerate 24.00 INFO: aacaot 2.00 INFO: avclevel 31.00 INFO: avcprofile 100.00 INFO: audiocodecid mp4a INFO: videocodecid avc1 INFO: width 848.00 INFO: height 480.00 INFO: frameWidth 848.00 INFO: frameHeight 480.00 INFO: displayWidth 848.00 INFO: displayHeight 480.00 INFO: framerate 24.00 INFO: moovposition 40.00 INFO: duration 1460.21 8926.402 kB / 92.58 sec (6.3%)

1 0

rtmpdump not working with dailymotion
by Nicolas George 19 Jan '12

19 Jan '12

Hi. Since about last week, dailymotion gives rtmpe:// links instead of HTTP links, but rtmpdump seems unable to handle them. Steps to reproduce: 1. Get a dailymotion page, and grep for the URL: curl -s http://www.dailymotion.com/video/xnqokq_nisemonogatari-ep01-remplacement_sh… | grep 'addVariable.*hqURL' > hqurl 2. Edit hqurl, keep only the part between hqURL and hd720URL, unescape. The reslut looks like that: http://www.dailymotion.com/cdn/H264-848x480/video/xnqokq.mp4?cell=secure-vo… 3. Get the file, it is a simple text file holding an URL: curl http://www.dailymotion.com/cdn/H264-848x480/video/xnqokq.mp4[...] rtmpe://vod-02.dailymotion.com/video/mp4:470/678/39876074:mp4_h264_aac_hq.m… 4. Use rtmpdump: rtmpdump -o blah.mp4 -r rtmpe://vod-02... Result: RTMPDump v2.4 (c) 2010 Andrej Stepanchuk, Howard Chu, The Flvstreamer Team; license: GPL Connecting ... WARNING: Trying different position for server digest! INFO: Connected... ERROR: Closing connection: NetStream.Play.Failed And blah.mp4 is empty. Any idea on how to fix that? Regards, -- Nicolas George

2 4

[PATCH] Callback API
by Antti Ajanki 09 Jan '12

09 Jan '12

The attached patch adds a callback interface for overriding (parts of) the RTMP packet processing. This is based on Chris Larsen's and Adam Malcontenti-Wilson's patches from last Fall. This is useful for implementing custom CDN authentication. My other motivation is implementing remote procedure calls over RTMP in the open source Flash player Lightspark, where a callback could be used to capture incoming RPC packets and execute the corresponding functions on Lightspark's virtual machine. The patch actually adds different callbacks on various levels of generality. RTMP_CALLBACK_PACKET callbacks can be used to completely override RTMP packet handling. Then there are more restricted callbacks, which only apply to some particular types of packets but, on the other hand, avoid much of the RTMP body parsing because that can be left to librtmp. See the comments in rtmp.h for details on different types of callbacks. Antti

3 2

RTMPSUCK
by chapun67＠gmail.com 09 Jan '12

09 Jan '12

is it possible to make more transparent RTMPSUCK?

2 1

Proposal: dynamically loadable plugins
by Antti Ajanki 08 Jan '12

08 Jan '12

Hi, I'm designing a plugin interface for librtmp for extending the functionality through dynamically loadable libraries. For example, a custom CDN authentication could be implemented as a plugin. Would something like this be accepted into librtmp? I'm designing this according to the needs of one particular CDN (the Finnish Yle Areena website) but trying to make the interface general. I'm asking for feedback on how to make the interface generally useful. I propose that plugins register new URL options (similar to the ones that RTMP_SetOpt() currently handles) and hooks that will be called if the registered options are present in the URL. The plugin will need to be explicitly activated by appending the correct URL option (this way one misbehaving plugin won't break all RTMP downloads). If none of the registered options are present, none of plugin's hooks will be called. A plugin has following hooks: void *create(RTMP *r) hook will the first hook to be called if a registered URL option is present. This will be called only once for each plugin even if there are multiple matching URL options. The return value is a pointer to plugin's private data. It will be passed to other hooks but librtmp won't use it otherwise. This function is meant for allocating plugin's private data structures and setting callbacks (using RTMP_AttachCallback(), see my previous mail) that implement the authetication or whatever work the plugin is meant to do. void parseOption(const AVal *name, const AVal *val, void *ctx) hook will be called for each URL option, which the plugin has registered and is present in the URL. The last parameter is the value returned by the create hook. This function processes the option values and setups callbacks. void delete(RTMP *r, void *ctx) hook will be called by RTMP_Free(). The last parameter is the value returned by the create hook. This should free the memory allocated by create and parseOption hooks. To make this more concrete, a plugin is a shared object that exposes the following entry point: const RTMP_Plugin *rtmp_init_plugin(); This returns a pointer to a struct containing the hooks and other information about the plugin: typedef struct RTMP_Plugin { uint32_t requiredAPIVersion; const char *name; const char *version; const char *author; const char *homepage; RTMPPluginOption *options; void *(*create)(RTMP *r); void (*delete)(RTMP *r, void *ctx); } RTMP_Plugin; requiredAPIVersion is the plugin API version (librtmp soname) for which this plugin is compiled for. If the the librtmp instance, which is trying to load this plugin, has different API version the plugin will not be loaded. name, version, author and homepage are strings that will be shown in the --help message. create and delete are the allocation and deallocation hooks as described above. options is an array of URL options that this plugin recognizes. The last item must be { 0 }. RTMPPluginOption is the following struct: typedef struct RTMPPluginOption { const char *name; /* Name of the option in the SetupURL string */ const char *type; /* Type name to be shown in the help screen */ const char *usage; /* A short description in the help screen */ void (*parseOption)(const AVal *name, const AVal *val, void *ctx); /* Function that will be called if the SetupURL string contains * option called name. ctx is the user data pointer returned by * the create hook. */ } RTMPPluginOption; librtmp will scan for plugins in ~/.librtmp/plugins and in $(LIBDIR)/librtmp/plugins when the first RTMP struct is allocated. RTMP_SetupURL() will loop through all options in the input URL and call the corresponding hooks, if one of the loaded plugins has registered a handler for that option. I'm hoping to be able to post a prototype implementation soon. Any comments are welcome even before that. Antti

1 0

spike in memory usage
by Leontie Eugen 04 Jan '12

04 Jan '12

When using ffmpeg compiled and linked with librtmp to decode a ustream radio, I have experienced some very large memory spikes after a couple of hours of usage. Running it in valgrind with the heap profiler shows me that the memory usage is fairly low for a while ( 2-3MB in total) then after a couple of hours ( 4-5 on average ) it spikes to 500-800MB. Left running the application is finaly exhausting the memory on the computer after a couple of days and the decoder crashes. All this extra memory is allocated in RTMPPacket_Alloc, I am not sure if it is a memory leak or a loose packet queue that never stops growing. Before I fetch more debugging details, let me ask if this sounds familiar to anyone. The command: ffmpeg -i "rtmp://flash64.ustream.tv app=ustreamVideo/5210553 playpath=streams/live live=true" -acodec pcm_s16le -f s16le -ac 1 -ar 8000 - > /dev/null Running it in valgrind: valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out.RADIO.%p ~/audionow/work/ffmpeg/bin/ffmpeg -i " rtmp://flash64.ustream.tv app=ustreamVideo/5210553 playpath=streams/live live=true" -acodec pcm_s16le -f s16le -ac 1 -ar 8000 - > /dev/null Reported by massif : 94.10% (986,617,021B) 0x9F7C92: RTMPPacket_Alloc | ->94.10% (986,617,021B) 0x9F9C39: RTMP_ReadPacket | ->94.10% (986,617,021B) 0x9FC3A5: RTMP_GetNextMediaPacket | | ->94.10% (986,617,021B) 0x9FC4B5: Read_1_Packet | | ->94.10% (986,617,021B) 0x9FD089: RTMP_Read | | | ->94.10% (986,617,021B) 0x46C7AA: ffurl_read (avio.c:271) | | | ->94.10% (986,617,021B) 0x46F6D8: fill_buffer (aviobuf.c:568) | | | | ->73.83% (774,147,504B) 0x46F833: avio_r8 (aviobuf.c:612) | | | | | ->73.83% (774,147,504B) 0x4820F9: flv_read_packet (flvdec.c:407) | | | | | | ->73.83% (774,147,504B) 0x4F1F42: av_read_packet (utils.c:746) | | | | | | ->73.83% (774,147,504B) 0x4F3D37: read_frame_internal (utils.c:1216) Thanks, Eugen

1 0