[FFmpeg-devel] [PATCH 02/14] avformat/tls_openssl: force dtls handshake to be blocking

[Jack Lau]

> On Jul 14, 2025, at 03:24, Timo Rothenpieler <timo at rothenpieler.org> wrote:
> 
> There is no sensible way to handle this otherwise anyway, one just has
> to loop over this function until it succeeds.
> ---
> libavformat/tls_openssl.c | 18 ++++++++++++------
> 1 file changed, 12 insertions(+), 6 deletions(-)
> 
> diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
> index f6826222f9..54213c4090 100644
> --- a/libavformat/tls_openssl.c
> +++ b/libavformat/tls_openssl.c
> @@ -685,27 +685,33 @@ static int openssl_dtls_verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
> 
> static int dtls_handshake(URLContext *h)
> {
> -    int ret = 0, r0, r1;
> +    int ret = 1, r0, r1;
>     TLSContext *p = h->priv_data;
> 
> +    int was_nonblock = h->flags & AVIO_FLAG_NONBLOCK;
> +    h->flags &= ~AVIO_FLAG_NONBLOCK;
> +
>     r0 = SSL_do_handshake(p->ssl);
> -    r1 = SSL_get_error(p->ssl, r0);
>     if (r0 <= 0) {
> +        r1 = SSL_get_error(p->ssl, r0);
> +
>         if (r1 != SSL_ERROR_WANT_READ && r1 != SSL_ERROR_WANT_WRITE && r1 != SSL_ERROR_ZERO_RETURN) {
> -            av_log(p, AV_LOG_ERROR, "TLS: Read failed, r0=%d, r1=%d %s\n", r0, r1, openssl_get_error(p));
> -            ret = AVERROR(EIO);
> +            av_log(p, AV_LOG_ERROR, "Handshake failed, r0=%d, r1=%d\n", r0, r1);
> +            ret = print_ssl_error(h, r0);
>             goto end;
>         }
>     } else {
> -        av_log(p, AV_LOG_TRACE, "TLS: Read %d bytes, r0=%d, r1=%d\n", r0, r0, r1);
> +        av_log(p, AV_LOG_TRACE, "Handshake success, r0=%d\n", r0);
>     }
> 
> -    /* Check whether the DTLS is completed. */
>     if (SSL_is_init_finished(p->ssl) != 1)
>         goto end;
> 
> +    ret = 0;
>     p->tls_shared.state = DTLS_STATE_FINISHED;
> end:
> +    if (was_nonblock)
> +        h->flags |= AVIO_FLAG_NONBLOCK;
>     return ret;
> }
> 
> -- 
> 2.49.0
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe”.

The flag of DTLS didn’t pass into udp, so maybe you should add this diff:

diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index 07d1af40d8..62f0df2202 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -701,9 +701,10 @@ static int dtls_handshake(URLContext *h)
 {
     int ret = 1, r0, r1;
     TLSContext *p = h->priv_data;
-
+    TLSShared *c = &p->tls_shared;
+    URLContext *uc = c->is_dtls ? c->udp : c->tcp;
     int was_nonblock = h->flags & AVIO_FLAG_NONBLOCK;
-    h->flags &= ~AVIO_FLAG_NONBLOCK;
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
 
     r0 = SSL_do_handshake(p->ssl);
     if (r0 <= 0) {
@@ -725,7 +726,7 @@ static int dtls_handshake(URLContext *h)
     p->tls_shared.state = DTLS_STATE_FINISHED;
 end:
     if (was_nonblock)
-        h->flags |= AVIO_FLAG_NONBLOCK;
+        uc->flags |= AVIO_FLAG_NONBLOCK;
     return ret;
 }

Thanks
Jack