autoconf: proper extra CFLAG check/handling e.g. -f* and inter-library-dependency checks, snprintf m4 should not fail if xcompiling

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

3 files changed, 55 insertions, 24 deletions

diff --git a/configure.ac b/configure.ac
index 5b3e353..b5f7649 100644
--- a/configure.ac
+++ b/configure.ac
@@ -20,7 +20,7 @@ AC_CHECK_TOOL([PKGCONFIG], [pkg-config], [:])
 AS_IF([test "x${PKGCONFIG}" = x], [ AC_MSG_ERROR([pkg-config not found]) ])
 
 dnl check for -std=gnu99
-CFLAGS="-std=gnu99 $CFLAGS"
+CFLAGS="-pedantic -Wall -std=gnu99 -D_GNU_SOURCE=1 $CFLAGS"
 AC_MSG_CHECKING([if ${CC} supports gnu99])
 AC_COMPILE_IFELSE([AC_LANG_SOURCE([char foo;])],
       AC_MSG_RESULT([yes]),
@@ -33,22 +33,27 @@ saved_CFLAGS="$CFLAGS"
 CFLAGS="$CFLAGS -mindirect-branch=thunk"
 AC_MSG_CHECKING([if ${CC} supports -mindirect-branch=thunk spectre mitigation])
 AC_COMPILE_IFELSE([AC_LANG_SOURCE([char foo;])],
-      [ AC_MSG_RESULT([yes])
-        SPECTRE_MIT="-mindirect-branch=thunk" ],
-        AC_MSG_RESULT([no]))
-CFLAGS="$saved_CFLAGS"
-AC_SUBST([SPECTRE_MIT])
+      [ AC_MSG_RESULT([yes]) ],
+      [ AC_MSG_RESULT([no])
+        CFLAGS="$saved_CFLAGS" ])
+
+dnl check for -fstrict-aliasing compilter support
+saved_CFLAGS="$CFLAGS"
+CFLAGS="$CFLAGS -fstrict-aliasing"
+AC_MSG_CHECKING([if ${CC} supports -fstrict-aliasing])
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([char foo;])],
+      [ AC_MSG_RESULT([yes]) ],
+      [ AC_MSG_RESULT([no])
+        CFLAGS="$saved_CFLAGS" ])
 
 dnl check for -fvisibility=hidden compiler support (GCC >= 4)
 saved_CFLAGS="$CFLAGS"
-CFLAGS="$CFLAGS -fvisibility=hidden -fvisibility-inlines-hidden"
-AC_MSG_CHECKING([if ${CC} supports -fvisibility=hidden -fvisibility-inlines-hidden])
+CFLAGS="$CFLAGS -fvisibility=hidden"
+AC_MSG_CHECKING([if ${CC} supports -fvisibility=hidden])
 AC_COMPILE_IFELSE([AC_LANG_SOURCE([char foo;])],
-      [ AC_MSG_RESULT([yes])
-        SYMBOL_VISIBILITY="-fvisibility=hidden" ],
-        AC_MSG_RESULT([no]))
-CFLAGS="$saved_CFLAGS"
-AC_SUBST([SYMBOL_VISIBILITY])
+      [ AC_MSG_RESULT([yes]) ],
+      [ AC_MSG_RESULT([no])
+        CFLAGS="$saved_CFLAGS" ])
 
 AX_PTHREAD([],[ AC_MSG_ERROR([pthread required and not found]) ])
 LIBS="$PTHREAD_LIBS $LIBS"
@@ -89,7 +94,15 @@ AS_IF([test "x${libcrypto_require_zlib}" = xyes],
         LIBS="$LIBS -lz"
         AC_CHECK_LIB([z], [inflate], [],
             [ AC_MSG_ERROR([libz not found]) ])
-        AC_CHECK_LIB([crypto], [RSA_new],
+        AC_CHECK_LIB([crypto], [RSA_free], [ libcrypto_require_libdl="no" ],
+                                           [ libcrypto_require_libdl="yes"])
+      ])
+AS_IF([test "x${libcrypto_require_libdl}" = xyes],
+      [ additional_libssh_libs="${additional_libssh_libs} -ldl"
+        LIBS="$LIBS -ldl"
+        AC_CHECK_LIB([dl], [dlopen], [],
+            [ AC_MSG_ERROR([libdl not found]) ])
+        AC_CHECK_LIB([crypto], [RSA_generate_key], [],
             [ AC_MSG_ERROR([libcrypto not found]) ])
       ])
 
@@ -101,10 +114,19 @@ AC_MSG_RESULT([${libssh_require_libssl}])
 AS_IF([test "x${libssh_require_libssl}" = xyes],
       [ additional_libssh_libs="${additional_libssh_libs} -lssl"
         LIBS="$LIBS -lssl"
-        AC_CHECK_LIB([ssl], [SSL_new], [],
-            [AC_MSG_ERROR([libssl not found])])
+        AC_CHECK_LIB([ssl], [SSL_new], [ libssl_require_libdl="no" ],
+                                       [ libssl_require_libdl="yes"])
 	  ])
 
+AS_IF([test "x${libssl_require_libdl}" = xyes],
+      [ additional_libssh_libs="${additional_libssh_libs} -ldl"
+        LIBS="$LIBS -ldl"
+        AC_CHECK_LIB([dl], [dlopen], [],
+            [ AC_MSG_ERROR([libdl not found]) ])
+        AC_CHECK_LIB([ssl], [SSL_free], [],
+            [ AC_MSG_ERROR([libssl not found]) ])
+      ])
+
 AC_MSG_CHECKING([if libssh requires -lz])
 AC_TRY_LINK([#include <libssh/libssh.h>],
       [ return ssh_init(); ], [ libssh_require_libz="no" ],
@@ -118,7 +140,18 @@ AS_IF([test "x${libssh_require_libz}" = xyes -a "x${libcrypto_require_zlib}" !=
       ])
 
 LIBS="$saved_LIBS $additional_libssh_libs"
-AC_CHECK_LIB([ssh], [ssh_init], [], [AC_MSG_ERROR([final link against libssh failed])])
+AC_CHECK_LIB([ssh], [ssh_init], [ libssh_require_gssapi="no" ],
+                                [ libssh_require_gssapi="yes"])
+AS_IF([test "x${libssh_require_gssapi}" = xyes],
+      [ saved_LIBS="$LIBS"
+        LIBS="$LIBS -lgssapi"
+        AC_CHECK_LIB([gssapi], [gss_init_sec_context], [],
+            [ AC_MSG_ERROR([no usable gssapi found]) ])
+        AC_CHECK_LIB([ssh], [ssh_free],
+                     [ AC_MSG_ERROR([final link against libssh failed]) ])
+      ])
+
+
 AC_CHECK_LIB([seccomp], [seccomp_init], [], [AC_MSG_ERROR([final link against libseccomp failed])])
 LIBS="$saved_LIBS $additional_libssh_libs"
 
@@ -286,6 +319,4 @@ potd_netns_run_dir="/var/run/potd-netns"
 AC_DEFINE_UNQUOTED([POTD_NETNS_RUN_DIR], ["$potd_netns_run_dir"],
       [default path to network namespace run directory])
 
-CFLAGS="-pedantic -Wall -fstrict-aliasing -D_GNU_SOURCE=1 $CFLAGS"
-
 AC_OUTPUT(Makefile src/Makefile)
diff --git a/m4/snprintf.m4 b/m4/snprintf.m4
index e66b1c3..b1dda79 100644
--- a/m4/snprintf.m4
+++ b/m4/snprintf.m4
@@ -50,7 +50,7 @@ AC_DEFUN([HW_FUNC_VA_COPY],
         [[va_list ap, aq; va_copy(aq, ap);]])],
       [hw_cv_func_va_copy=yes],
       [hw_cv_func_va_copy=no],
-      [hw_cv_func_va_copy=no])])
+      [hw_cv_func_va_copy=yes])])
   AS_IF([test "$hw_cv_func_va_copy" = yes],
     [AC_DEFINE([HAVE_VA_COPY], [1],
       [Define to 1 if you have the `va_copy' function or macro.])])
@@ -76,7 +76,7 @@ AC_DEFUN([HW_FUNC___VA_COPY],
         [[va_list ap, aq; __va_copy(aq, ap);]])],
       [hw_cv_func___va_copy=yes],
       [hw_cv_func___va_copy=no],
-      [hw_cv_func___va_copy=no])])
+      [hw_cv_func___va_copy=yes])])
   AS_IF([test "$hw_cv_func___va_copy" = yes],
     [AC_DEFINE([HAVE___VA_COPY], [1],
       [Define to 1 if you have the `__va_copy' function or macro.])])
@@ -124,7 +124,7 @@ AC_DEFUN([HW_FUNC_VSNPRINTF],
 				return 1;]])],
 			[hw_cv_func_vsnprintf_c99=yes],
 			[hw_cv_func_vsnprintf_c99=no],
-			[hw_cv_func_vsnprintf_c99=no])])],
+			[hw_cv_func_vsnprintf_c99=yes])])],
 		[hw_cv_func_snprintf_c99=no])
 	  AS_IF([test "$hw_cv_func_vsnprintf_c99" = yes],
 		[AC_DEFINE([HAVE_VSNPRINTF], [1],
@@ -179,7 +179,7 @@ AC_DEFUN([HW_FUNC_SNPRINTF],
 				return 1;]])],
 			[hw_cv_func_snprintf_c99=yes],
 			[hw_cv_func_snprintf_c99=no],
-			[hw_cv_func_snprintf_c99=no])])],
+			[hw_cv_func_snprintf_c99=yes])])],
 		[hw_cv_func_snprintf_c99=no])
   else
       hw_cv_func_snprintf_c99=no
diff --git a/src/Makefile.am b/src/Makefile.am
index d5d11d9..c51a60c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,4 +1,4 @@
-AM_CFLAGS = $(libssh_CFLAGS) $(libseccomp_CFLAGS) $(valgrind_CFLAGS) $(SPECTRE_MIT) $(SYMBOL_VISIBILITY)
+AM_CFLAGS = $(libssh_CFLAGS) $(libseccomp_CFLAGS) $(valgrind_CFLAGS)
 
 sbin_PROGRAMS = potd
 potd_SOURCES = utils.c options.c log.c log_colored.c log_file.c socket.c pevent.c capabilities.c pseccomp.c jail.c forward.c redirector.c protocol.c protocol_ssh.c main.c