[FFmpeg-devel] [PATCH] configure: improve logic and checks for nvenc

[James Almer]

On 8/30/2016 8:00 AM, Timo Rothenpieler wrote:
> ---
>  configure | 37 +++++++++++++++++++++++++------------
>  1 file changed, 25 insertions(+), 12 deletions(-)
> 
> diff --git a/configure b/configure
> index 52931c3..bcfc9a8 100755
> --- a/configure
> +++ b/configure
> @@ -5992,20 +5992,33 @@ enabled vdpau && enabled xlib &&
>      check_lib2 "vdpau/vdpau.h vdpau/vdpau_x11.h" vdp_device_create_x11 -lvdpau &&
>      enable vdpau_x11
>  
> -case $target_os in
> -    mingw32*|mingw64*|win32|win64|linux|cygwin*)
> -        disabled nvenc || enable nvenc
> -        ;;
> -    *)
> -        disable nvenc
> -        ;;
> -esac
> -
> -if enabled nvenc; then
> +if ! enabled x86; then
> +    enabled nvenc && die "NVENC is only supported on x86"
> +    disable nvenc
> +elif ! disabled nvenc; then
>      {
>          echo '#include "compat/nvenc/nvEncodeAPI.h"'
> -        echo 'int main(void) { return 0; }'
> -    } | check_cc -I$source_path || disable nvenc
> +        echo 'NV_ENCODE_API_FUNCTION_LIST flist;'
> +        echo 'void f(void) { struct { const GUID guid; } s[] = { { NV_ENC_PRESET_HQ_GUID } }; }'

This will most likely prevent nvenc from being enabled for msvc 2012, but not old
mingw32, which is failing with the error:

src/libavcodec/nvenc.c:115:52: error: 'ENOBUFS' undeclared here (not in a function)
     { NV_ENC_ERR_NOT_ENOUGH_BUFFER,        AVERROR(ENOBUFS), "not enough buffer"        },

I think the easiest solution would be using AVERROR_BUFFER_TOO_SMALL if ENOBUFS is
not defined.
That or just disable nvenc if using mingw32 toolchains by checking "enabled
libc_mingw32", since disabling for target-os == mingw32 would also affect mingw-w64.

gcc-asan fails with

/usr/bin/ld: libavcodec/libavcodec.a(nvenc.o): undefined reference to symbol 'dlsym@@GLIBC_2.2.5'
/usr/lib/../lib/libdl.so.2: error adding symbols: DSO missing from command line
collect2: error: ld returned 1 exit status

I have no idea how to deal with this.

> +        echo 'int main(void) { f(); return 0; }'
> +    } | check_cc -I$source_path
> +    nvenc_check_res=$?
> +
> +    if [ $nvenc_check_res != 0 ] && enabled nvenc; then
> +        die "NVENC enabled but test-compile failed"
> +    fi
> +
> +    case $target_os in
> +        mingw32*|mingw64*|win32|win64|linux|cygwin*)
> +            [ $nvenc_check_res = 0 ] && enable nvenc
> +            ;;
> +        *)
> +            enabled nvenc && die "NVENC is only supported on Windows and Linux"
> +            disable nvenc
> +            ;;
> +    esac
> +
> +    unset nvenc_check_res

This test is different from other automatically detected features, and also
unnecessarily complex.
You should force enable nvenc earlier in the script like with other similar
features (including hardware codecs and accelerators), then disable it on
unsupported platforms and old/broken compilers with the corresponding checks
and tests.

Something like this:

------

diff --git a/configure b/configure
index 52931c3..a09aa6e 100755
--- a/configure
+++ b/configure
@@ -3205,7 +3205,7 @@ enable audiotoolbox
 enable d3d11va dxva2 vaapi vda vdpau videotoolbox_hwaccel xvmc
 enable xlib

-enable vda_framework videotoolbox videotoolbox_encoder
+enable nvenc vda_framework videotoolbox videotoolbox_encoder

 # build settings
 SHFLAGS='-shared -Wl,-soname,$$(@F)'
@@ -5992,22 +5992,25 @@ enabled vdpau && enabled xlib &&
     check_lib2 "vdpau/vdpau.h vdpau/vdpau_x11.h" vdp_device_create_x11 -lvdpau &&
     enable vdpau_x11

-case $target_os in
-    mingw32*|mingw64*|win32|win64|linux|cygwin*)
-        disabled nvenc || enable nvenc
-        ;;
-    *)
-        disable nvenc
-        ;;
-esac
-
-if enabled nvenc; then
-    {
-        echo '#include "compat/nvenc/nvEncodeAPI.h"'
-        echo 'int main(void) { return 0; }'
-    } | check_cc -I$source_path || disable nvenc
+if enabled x86; then
+    case $target_os in
+        mingw32*|mingw64*|win32|win64|linux|cygwin*)
+            ;;
+        *)
+            disable nvenc
+            ;;
+    esac
+else
+    disable nvenc
 fi

+enabled nvenc &&
+    check_cc -I$source_path <<EOF || disable nvenc
+#include "compat/nvenc/nvEncodeAPI.h"
+NV_ENCODE_API_FUNCTION_LIST flist;
+void f(void) { struct { const GUID guid; } s[] = { { NV_ENC_PRESET_HQ_GUID } }; }
+EOF
+
 # Funny iconv installations are not unusual, so check it after all flags have been set
 disabled iconv || check_func_headers iconv.h iconv || check_lib2 iconv.h iconv -liconv || disable iconv

------

Is IMO much simpler.