From 387109364e66468736000a57e071cdfff8328ff6 Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Tue, 29 Nov 2022 05:39:41 +1000 Subject: [PATCH] Python3: replace `distutils` with `sysconfig` - `distutils` module is long time deprecated and already deleted from the CPython mainline. - To remain compatible with Debian/Ubuntu Python3 packaging style, try `distutils.sysconfig.get_python_path(0,0)` first with fallback on `sysconfig.get_path('purelib')` - pyzfs_unittest suite is run unconditionally as a part of ZTS. - Add pyzfs_unittest suite to sanity tests. Reviewed-by: Brian Behlendorf Signed-off-by: szubersk Closes #12833 Closes #13280 Closes #14177 --- config/ax_python_devel.m4 | 51 +++++++------------ rpm/generic/zfs.spec.in | 2 +- tests/runfiles/sanity.run | 4 ++ tests/test-runner/bin/zts-report.py.in | 9 ---- .../functional/pyzfs/pyzfs_unittest.ksh.in | 16 +----- 5 files changed, 26 insertions(+), 56 deletions(-) diff --git a/config/ax_python_devel.m4 b/config/ax_python_devel.m4 index 7adcf01a04..e6cee04288 100644 --- a/config/ax_python_devel.m4 +++ b/config/ax_python_devel.m4 @@ -165,31 +165,16 @@ variable to configure. See ``configure --help'' for reference. fi fi - # - # Check if you have distutils, else fail - # - AC_MSG_CHECKING([for the distutils Python package]) - if ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`; then - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - m4_ifvaln([$2],[$2],[ - AC_MSG_ERROR([cannot import Python module "distutils". -Please check your Python installation. The error was: -$ac_distutils_result]) - PYTHON_VERSION="" - ]) - fi - # # Check for Python include path # + # AC_MSG_CHECKING([for Python include path]) if test -z "$PYTHON_CPPFLAGS"; then - python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc ());"` - plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc (plat_specific=1));"` + python_path=`$PYTHON -c "import sysconfig; \ + print (sysconfig.get_path('include'));"` + plat_python_path=`$PYTHON -c "import sysconfig; \ + print (sysconfig.get_path('platinclude'));"` if test -n "${python_path}"; then if test "${plat_python_path}" != "${python_path}"; then python_path="-I$python_path -I$plat_python_path" @@ -213,7 +198,7 @@ $ac_distutils_result]) # join all versioning strings, on some systems # major/minor numbers could be in different list elements -from distutils.sysconfig import * +from sysconfig import * e = get_config_var('VERSION') if e is not None: print(e) @@ -236,8 +221,8 @@ EOD` ac_python_libdir=`cat</dev/null || \ + $PYTHON -c "import sysconfig; \ + print (sysconfig.get_path('purelib'));"` fi AC_MSG_RESULT([$PYTHON_SITE_PKG]) AC_SUBST([PYTHON_SITE_PKG]) @@ -299,8 +286,8 @@ EOD` # AC_MSG_CHECKING(python extra libraries) if test -z "$PYTHON_EXTRA_LIBS"; then - PYTHON_EXTRA_LIBS=`$PYTHON -c "import distutils.sysconfig; \ - conf = distutils.sysconfig.get_config_var; \ + PYTHON_EXTRA_LIBS=`$PYTHON -c "import sysconfig; \ + conf = sysconfig.get_config_var; \ print (conf('LIBS') + ' ' + conf('SYSLIBS'))"` fi AC_MSG_RESULT([$PYTHON_EXTRA_LIBS]) @@ -311,8 +298,8 @@ EOD` # AC_MSG_CHECKING(python extra linking flags) if test -z "$PYTHON_EXTRA_LDFLAGS"; then - PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import distutils.sysconfig; \ - conf = distutils.sysconfig.get_config_var; \ + PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import sysconfig; \ + conf = sysconfig.get_config_var; \ print (conf('LINKFORSHARED'))"` fi AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS]) diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index aea82d2417..166411d91a 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -78,7 +78,7 @@ %define __python %{__use_python} %define __python_pkg_version %{__use_python_pkg_version} %endif -%define __python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +%define __python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())" 2>/dev/null || %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") Name: @PACKAGE@ Version: @VERSION@ diff --git a/tests/runfiles/sanity.run b/tests/runfiles/sanity.run index f115f0b578..5142396620 100644 --- a/tests/runfiles/sanity.run +++ b/tests/runfiles/sanity.run @@ -624,3 +624,7 @@ tags = ['functional', 'zvol', 'zvol_swap'] [tests/functional/zpool_influxdb] tests = ['zpool_influxdb'] tags = ['functional', 'zpool_influxdb'] + +[tests/functional/pyzfs] +tests = ['pyzfs_unittest'] +tags = ['functional', 'pyzfs'] diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index e5b1ae0f65..a2324d4b52 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -61,14 +61,6 @@ known_reason = 'Known issue' # exec_reason = 'Test user execute permissions required for utilities' -# -# Some tests require a minimum python version of 3.6 and will be skipped when -# the default system version is too old. There may also be tests which require -# additional python modules be installed, for example python3-cffi is required -# by the pyzfs tests. -# -python_deps_reason = 'Python modules missing: python3-cffi' - # # Some tests require that the kernel supports renameat2 syscall. # @@ -232,7 +224,6 @@ maybe = { 'io/mmap': ['SKIP', fio_reason], 'largest_pool/largest_pool_001_pos': ['FAIL', known_reason], 'mmp/mmp_on_uberblocks': ['FAIL', known_reason], - 'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason], 'pool_checkpoint/checkpoint_discard_busy': ['FAIL', 11946], 'projectquota/setup': ['SKIP', exec_reason], 'removal/removal_condense_export': ['FAIL', known_reason], diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in index d55c4f3270..7fa14161a5 100755 --- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in +++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in @@ -18,6 +18,8 @@ if [ -n "$ASAN_OPTIONS" ]; then export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}') + # ASan reports leaks in CPython 3.10 + ASAN_OPTIONS="$ASAN_OPTIONS:detect_leaks=false" fi # @@ -30,20 +32,6 @@ fi # verify_runnable "global" - -# Verify that the required dependencies for testing are installed. -@PYTHON@ -c "import cffi" 2>/dev/null || - log_unsupported "python3-cffi not found by Python" - -# We don't just try to "import libzfs_core" because we want to skip these tests -# only if pyzfs was not installed due to missing, build-time, dependencies; if -# we cannot load "libzfs_core" due to other reasons, for instance an API/ABI -# mismatch, we want to report it. -@PYTHON@ -c ' -import pkgutil, sys -sys.exit(pkgutil.find_loader("libzfs_core") is None)' || - log_unsupported "libzfs_core not found by Python" - log_assert "Verify the nvlist and libzfs_core Python unittest run successfully" # log_must buffers stderr, which interacts badly with