bpo-43466: Add --with-openssl-rpath configure option (GH-24820)

Doc/using/unix.rst

@@ -134,3 +134,53 @@ some Unices may not have the :program:`env` command, so you may need to hardcode
 ``/usr/bin/python3`` as the interpreter path.
 
 To use shell commands in your Python scripts, look at the :mod:`subprocess` module.
+
+
+Custom OpenSSL
+==============
+
+1. To use your vendor's OpenSSL configuration and system trust store, locate
+   the directory with ``openssl.cnf`` file or symlink in ``/etc``. On most
+   distribution the file is either in ``/etc/ssl`` or ``/etc/pki/tls``. The
+   directory should also contain a ``cert.pem`` file and/or a ``certs``
+   directory.
+
+   .. code-block:: shell-session
+
+      $ find /etc/ -name openssl.cnf -printf "%h\n"
+      /etc/ssl
+
+2. Download, build, and install OpenSSL. Make sure you use ``install_sw`` and
+   not ``install``. The ``install_sw`` target does not override
+   ``openssl.cnf``.
+
+   .. code-block:: shell-session
+
+      $ curl -O https://www.openssl.org/source/openssl-VERSION.tar.gz
+         $ tar xzf openssl-VERSION
+         $ pushd openssl-VERSION
+         $ ./config \
+              --prefix=/usr/local/custom-openssl \
+              --openssldir=/etc/ssl
+         $ make -j1 depend
+         $ make -j8
+         $ make install_sw
+         $ popd
+
+3. Build Python with custom OpenSSL
+
+   .. code-block:: shell-session
+
+      $ pushd python-3.x.x
+      $ ./configure -C \
+          --with-openssl=/usr/local/custom-openssl \
+          --with-openssl-rpath=auto \
+          --prefix=/usr/local/python-3.x.x
+      $ make -j8
+      $ make altinstall
+
+.. note::
+
+   Patch releases of OpenSSL have a backwards compatible ABI. You don't need
+   to recompile Python to update OpenSSL. It's sufficient to replace the
+   custom OpenSSL installation with a newer version.

Doc/whatsnew/3.10.rst

@@ -1181,6 +1181,12 @@ Build Changes
   and ``--with-tcltk-libs`` configuration options.
   (Contributed by Manolis Stamatogiannakis in :issue:`42603`.)
 
+* Add ``--with-openssl-rpath`` option to ``configure`` script. The option
+  simplifies building Python with a custom OpenSSL installation, e.g.
+  ``./configure --with-openssl=/path/to/openssl --with-openssl-rpath=auto``.
+  (Contributed by Christian Heimes in :issue:`43466`.)
+
+
 
 C API Changes
 =============

Makefile.pre.in

@@ -202,6 +202,7 @@ ENSUREPIP=      @ENSUREPIP@
 OPENSSL_INCLUDES=@OPENSSL_INCLUDES@
 OPENSSL_LIBS=@OPENSSL_LIBS@
 OPENSSL_LDFLAGS=@OPENSSL_LDFLAGS@
+OPENSSL_RPATH=@OPENSSL_RPATH@
 
 # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var
 TZPATH=@TZPATH@

Misc/NEWS.d/next/Build/2021-03-11-00-14-47.bpo-43466.N861Z5.rst

@@ -0,0 +1 @@
+The ``configure`` script now supports ``--with-openssl-rpath`` option.

Tools/ssl/multissltests.py

@@ -48,8 +48,8 @@ OPENSSL_OLD_VERSIONS = [
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-    "1.1.1g",
-    # "3.0.0-alpha2"
+    "1.1.1j",
+    # "3.0.0-alpha12"
 ]
 
 LIBRESSL_OLD_VERSIONS = [
@@ -57,7 +57,7 @@ LIBRESSL_OLD_VERSIONS = [
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
-    "3.1.0",
+    "3.2.4",
 ]
 
 # store files in ../multissl
@@ -169,7 +169,9 @@ class AbstractBuilder(object):
     url_templates = None
     src_template = None
     build_template = None
+    depend_target = None
     install_target = 'install'
+    jobs = os.cpu_count()
 
     module_files = ("Modules/_ssl.c",
                     "Modules/_hashopenssl.c")
@@ -321,8 +323,11 @@ class AbstractBuilder(object):
         if self.system:
             env['SYSTEM'] = self.system
         self._subprocess_call(cmd, cwd=cwd, env=env)
-        # Old OpenSSL versions do not support parallel builds.
-        self._subprocess_call(["make", "-j1"], cwd=cwd, env=env)
+        if self.depend_target:
+            self._subprocess_call(
+                ["make", "-j1", self.depend_target], cwd=cwd, env=env
+            )
+        self._subprocess_call(["make", f"-j{self.jobs}"], cwd=cwd, env=env)
 
     def _make_install(self):
         self._subprocess_call(
@@ -409,6 +414,7 @@ class BuildOpenSSL(AbstractBuilder):
     build_template = "openssl-{}"
     # only install software, skip docs
     install_target = 'install_sw'
+    depend_target = 'depend'
 
     def _post_install(self):
         if self.version.startswith("3.0"):
@@ -434,11 +440,11 @@ class BuildOpenSSL(AbstractBuilder):
                 self.openssl_cli, "fipsinstall",
                 "-out", fipsinstall_cnf,
                 "-module", fips_mod,
-                "-provider_name", "fips",
-                "-mac_name", "HMAC",
-                "-macopt", "digest:SHA256",
-                "-macopt", "hexkey:00",
-                "-section_name", "fips_sect"
+                # "-provider_name", "fips",
+                # "-mac_name", "HMAC",
+                # "-macopt", "digest:SHA256",
+                # "-macopt", "hexkey:00",
+                # "-section_name", "fips_sect"
             ]
         )
         with open(openssl_fips_cnf, "w") as f:

aclocal.m4

@@ -67,7 +67,7 @@ AS_VAR_POPDEF([CACHEVAR])dnl
 ])dnl AX_CHECK_COMPILE_FLAGS
 
 # pkg.m4 - Macros to locate and utilise pkg-config.   -*- Autoconf -*-
-# serial 12 (pkg-config-0.29.2)
+# serial 11 (pkg-config-0.29.1)
 
 dnl Copyright © 2004 Scott James Remnant <scott@netsplit.com>.
 dnl Copyright © 2012-2015 Dan Nicholson <dbn.lists@gmail.com>
@@ -109,7 +109,7 @@ dnl
 dnl See the "Since" comment for each macro you use to see what version
 dnl of the macros you require.
 m4_defun([PKG_PREREQ],
-[m4_define([PKG_MACROS_VERSION], [0.29.2])
+[m4_define([PKG_MACROS_VERSION], [0.29.1])
 m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1,
     [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])])
 ])dnl PKG_PREREQ
@@ -210,7 +210,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl
 AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl
 
 pkg_failed=no
-AC_MSG_CHECKING([for $2])
+AC_MSG_CHECKING([for $1])
 
 _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2])
 _PKG_CONFIG([$1][_LIBS], [libs], [$2])
@@ -342,5 +342,73 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
 AS_VAR_IF([$1], [""], [$5], [$4])dnl
 ])dnl PKG_CHECK_VAR
 
+dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl   [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
+dnl   [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------
+dnl
+dnl Prepare a "--with-" configure option using the lowercase
+dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
+dnl PKG_CHECK_MODULES in a single macro.
+AC_DEFUN([PKG_WITH_MODULES],
+[
+m4_pushdef([with_arg], m4_tolower([$1]))
+
+m4_pushdef([description],
+           [m4_default([$5], [build with ]with_arg[ support])])
+
+m4_pushdef([def_arg], [m4_default([$6], [auto])])
+m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
+m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
+
+m4_case(def_arg,
+            [yes],[m4_pushdef([with_without], [--without-]with_arg)],
+            [m4_pushdef([with_without],[--with-]with_arg)])
+
+AC_ARG_WITH(with_arg,
+     AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
+    [AS_TR_SH([with_]with_arg)=def_arg])
+
+AS_CASE([$AS_TR_SH([with_]with_arg)],
+            [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
+            [auto],[PKG_CHECK_MODULES([$1],[$2],
+                                        [m4_n([def_action_if_found]) $3],
+                                        [m4_n([def_action_if_not_found]) $4])])
+
+m4_popdef([with_arg])
+m4_popdef([description])
+m4_popdef([def_arg])
+
+])dnl PKG_WITH_MODULES
+
+dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl   [DESCRIPTION], [DEFAULT])
+dnl -----------------------------------------------
+dnl
+dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
+dnl check._[VARIABLE-PREFIX] is exported as make variable.
+AC_DEFUN([PKG_HAVE_WITH_MODULES],
+[
+PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
+
+AM_CONDITIONAL([HAVE_][$1],
+               [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
+])dnl PKG_HAVE_WITH_MODULES
+
+dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl   [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------------------
+dnl
+dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
+dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
+dnl and preprocessor variable.
+AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
+[
+PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
+
+AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
+        [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
+])dnl PKG_HAVE_DEFINE_WITH_MODULES
+
 m4_include([m4/ax_c_float_words_bigendian.m4])
 m4_include([m4/ax_check_openssl.m4])

configure.ac

@@ -5808,6 +5808,30 @@ if test "$have_openssl" = yes; then
     LIBS="$save_LIBS"
 fi
 
+# rpath to libssl and libcrypto
+AC_MSG_CHECKING(for --with-openssl-rpath)
+AC_ARG_WITH(openssl-rpath,
+    AS_HELP_STRING([--with-openssl-rpath=@<:@DIR|auto|no@:>@],
+                   [Set runtime library directory (rpath) for OpenSSL libraries,
+                    no (default): don't set rpath,
+                    auto: auto-detect rpath from --with-openssl and pkg-config,
+                    DIR: set an explicit rpath
+                   ]),
+    [],
+    [with_openssl_rpath=no]
+)
+AS_CASE($with_openssl_rpath,
+    [auto|yes],[OPENSSL_RPATH=auto],
+    [no],[OPENSSL_RPATH=],
+    [AS_IF(
+        [test -d "$with_openssl_rpath"],
+        [OPENSSL_RPATH="$with_openssl_rpath"],
+        AC_MSG_ERROR([--with-openssl-rpath "$with_openssl_rpath" is not a directory]))
+    ]
+)
+AC_MSG_RESULT($OPENSSL_RPATH)
+AC_SUBST([OPENSSL_RPATH])
+
 # ssl module default cipher suite string
 AH_TEMPLATE(PY_SSL_DEFAULT_CIPHERS,
   [Default cipher suites list for ssl module.

setup.py

@@ -549,6 +549,8 @@ class PyBuildExt(build_ext):
                   "libssl with X509_VERIFY_PARAM_set1_host().")
             print("LibreSSL 2.6.4 and earlier do not provide the necessary "
                   "APIs, https://github.com/libressl-portable/portable/issues/381")
+            if sysconfig.get_config_var("OPENSSL_LDFLAGS"):
+                print("Custom linker flags may require --with-openssl-rpath=auto")
             print()
 
     def build_extension(self, ext):
@@ -2416,6 +2418,7 @@ class PyBuildExt(build_ext):
         openssl_includes = split_var('OPENSSL_INCLUDES', '-I')
         openssl_libdirs = split_var('OPENSSL_LDFLAGS', '-L')
         openssl_libs = split_var('OPENSSL_LIBS', '-l')
+        openssl_rpath = config_vars.get('OPENSSL_RPATH')
         if not openssl_libs:
             # libssl and libcrypto not found
             self.missing.extend(['_ssl', '_hashlib'])
@@ -2437,12 +2440,20 @@ class PyBuildExt(build_ext):
         if krb5_h:
             ssl_incs.extend(krb5_h)
 
+        if openssl_rpath == 'auto':
+            runtime_library_dirs = openssl_libdirs[:]
+        elif not openssl_rpath:
+            runtime_library_dirs = []
+        else:
+            runtime_library_dirs = [openssl_rpath]
+
         if config_vars.get("HAVE_X509_VERIFY_PARAM_SET1_HOST"):
             self.add(Extension(
                 '_ssl', ['_ssl.c'],
                 include_dirs=openssl_includes,
                 library_dirs=openssl_libdirs,
                 libraries=openssl_libs,
+                runtime_library_dirs=runtime_library_dirs,
                 depends=['socketmodule.h', '_ssl/debughelpers.c'])
             )
         else:
@@ -2452,6 +2463,7 @@ class PyBuildExt(build_ext):
                            depends=['hashlib.h'],
                            include_dirs=openssl_includes,
                            library_dirs=openssl_libdirs,
+                           runtime_library_dirs=runtime_library_dirs,
                            libraries=openssl_libs))
 
     def detect_hash_builtins(self):