[FFmpeg-devel] [PATCH v2] avformat/tls_schannel: add check for Windows 10 only types and defines

[James Almer]

Old Mingw-w64 releases provided by some distros seemingly don't have them, so
check for them and disable the dtls protocol if unavailable.

Signed-off-by: James Almer <jamrial at gmail.com>
---
 configure                  |  4 ++++
 libavformat/tls_schannel.c | 29 +++++++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 6df8fa4deb..fc082d5467 100755
--- a/configure
+++ b/configure
@@ -2498,6 +2498,7 @@ TYPES_LIST="
     kCVImageBufferTransferFunction_ITU_R_2020
     kCVImageBufferTransferFunction_SMPTE_ST_428_1
     kVTQPModulationLevel_Default
+    SecPkgContext_KeyingMaterialInfo
     socklen_t
     struct_addrinfo
     struct_group_source_req
@@ -6822,6 +6823,7 @@ check_type "windows.h d3d12video.h" "ID3D12VideoEncoder"
 test_code cc "windows.h d3d12video.h" "D3D12_FEATURE_VIDEO feature = D3D12_FEATURE_VIDEO_ENCODER_CODEC" && \
 test_code cc "windows.h d3d12video.h" "D3D12_FEATURE_DATA_VIDEO_ENCODER_RESOURCE_REQUIREMENTS req" && enable d3d12_encoder_feature
 check_type "windows.h" "DPI_AWARENESS_CONTEXT" -D_WIN32_WINNT=0x0A00
+check_type "windows.h security.h schnlsp.h" SecPkgContext_KeyingMaterialInfo -DSECURITY_WIN32
 check_type "d3d9.h dxva2api.h" DXVA2_ConfigPictureDecode -D_WIN32_WINNT=0x0602
 check_func_headers mfapi.h MFCreateAlignedMemoryBuffer -lmfplat
 
@@ -7271,6 +7273,8 @@ enabled schannel &&
     schannel_extralibs="-lsecur32 -lncrypt -lcrypt32" ||
         disable schannel
 
+enabled schannel && check_cc dtls_protocol "windows.h security.h schnlsp.h" "int i = SP_PROT_DTLS1_X_CLIENT;" -DSECURITY_WIN32
+
 makeinfo --version > /dev/null 2>&1 && enable makeinfo  || disable makeinfo
 enabled makeinfo \
     && [ 0$(makeinfo --version | grep "texinfo" | sed 's/.*texinfo[^0-9]*\([0-9]*\)\..*/\1/') -ge 5 ] \
diff --git a/libavformat/tls_schannel.c b/libavformat/tls_schannel.c
index da6a284376..55bfe08977 100644
--- a/libavformat/tls_schannel.c
+++ b/libavformat/tls_schannel.c
@@ -20,6 +20,9 @@
 
 /** Based on the CURL SChannel module */
 
+#include "config.h"
+#include "config_components.h"
+
 #include "libavutil/mem.h"
 #include "avformat.h"
 #include "internal.h"
@@ -634,6 +637,7 @@ int ff_tls_set_external_socket(URLContext *h, URLContext *sock)
 
 int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz)
 {
+#if HAVE_SECPKGCONTEXT_KEYINGMATERIALINFO
     TLSContext *c = h->priv_data;
 
     SecPkgContext_KeyingMaterialInfo keying_info = { 0 };
@@ -672,6 +676,9 @@ int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t ma
     }
 
     return 0;
+#else
+    return AVERROR(ENOSYS);
+#endif
 }
 
 int ff_dtls_state(URLContext *h)
@@ -773,7 +780,11 @@ static int tls_shutdown_client(URLContext *h)
                 }
                 FreeContextBuffer(outbuf.pvBuffer);
             }
-        } while(sspi_ret == SEC_I_MESSAGE_FRAGMENT || sspi_ret == SEC_I_CONTINUE_NEEDED);
+        } while(
+#ifdef SEC_I_MESSAGE_FRAGMENT
+                sspi_ret == SEC_I_MESSAGE_FRAGMENT ||
+#endif
+                sspi_ret == SEC_I_CONTINUE_NEEDED);
 
         av_log(h, AV_LOG_DEBUG, "Close session result: 0x%lx\n", sspi_ret);
 
@@ -928,7 +939,11 @@ static int tls_handshake_loop(URLContext *h, int initial)
         }
 
         /* continue handshake */
-        if (sspi_ret == SEC_I_CONTINUE_NEEDED || sspi_ret == SEC_I_MESSAGE_FRAGMENT || sspi_ret == SEC_E_OK) {
+        if (sspi_ret == SEC_I_CONTINUE_NEEDED ||
+#ifdef SEC_I_MESSAGE_FRAGMENT
+            sspi_ret == SEC_I_MESSAGE_FRAGMENT ||
+#endif
+            sspi_ret == SEC_E_OK) {
             for (i = 0; i < 3; i++) {
                 if (outbuf[i].BufferType == SECBUFFER_TOKEN && outbuf[i].cbBuffer > 0) {
                     ret = ffurl_write(uc, outbuf[i].pvBuffer, outbuf[i].cbBuffer);
@@ -1080,6 +1095,7 @@ static int tls_handshake(URLContext *h)
     if (ret < 0)
         goto fail;
 
+#if CONFIG_DTLS_PROTOCOL
     if (s->is_dtls && s->mtu > 0) {
         ULONG mtu = s->mtu;
         sspi_ret = SetContextAttributes(&c->ctxt_handle, SECPKG_ATTR_DTLS_MTU, &mtu, sizeof(mtu));
@@ -1090,6 +1106,7 @@ static int tls_handshake(URLContext *h)
         }
         av_log(h, AV_LOG_VERBOSE, "Set DTLS MTU to %d\n", s->mtu);
     }
+#endif
 
     c->connected = 1;
     s->state = DTLS_STATE_FINISHED;
@@ -1136,8 +1153,10 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
 
         schannel_cred.dwFlags = SCH_CRED_NO_SYSTEM_MAPPER | SCH_CRED_MANUAL_CRED_VALIDATION;
 
+#if CONFIG_DTLS_PROTOCOL
         if (s->is_dtls)
             schannel_cred.grbitEnabledProtocols = SP_PROT_DTLS1_X_SERVER;
+#endif
     } else {
         if (s->verify)
             schannel_cred.dwFlags = SCH_CRED_AUTO_CRED_VALIDATION |
@@ -1147,8 +1166,10 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
                                     SCH_CRED_IGNORE_NO_REVOCATION_CHECK |
                                     SCH_CRED_IGNORE_REVOCATION_OFFLINE;
 
+#if CONFIG_DTLS_PROTOCOL
         if (s->is_dtls)
             schannel_cred.grbitEnabledProtocols = SP_PROT_DTLS1_X_CLIENT;
+#endif
     }
 
     /* Get credential handle */
@@ -1439,6 +1460,7 @@ static const AVOption options[] = {
     { NULL }
 };
 
+#if CONFIG_TLS_PROTOCOL
 static const AVClass tls_class = {
     .class_name = "tls",
     .item_name  = av_default_item_name,
@@ -1458,7 +1480,9 @@ const URLProtocol ff_tls_protocol = {
     .flags          = URL_PROTOCOL_FLAG_NETWORK,
     .priv_data_class = &tls_class,
 };
+#endif
 
+#if CONFIG_DTLS_PROTOCOL
 static const AVClass dtls_class = {
     .class_name = "dtls",
     .item_name  = av_default_item_name,
@@ -1479,3 +1503,4 @@ const URLProtocol ff_dtls_protocol = {
     .flags          = URL_PROTOCOL_FLAG_NETWORK,
     .priv_data_class = &dtls_class,
 };
+#endif
-- 
2.50.1