diff --git a/.gitignore b/.gitignore index 3a8cb2e867..6367ebf778 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ *.swp *.gcno *.gcda +*.pyc +*.pyo .deps .libs .dirstamp diff --git a/cmd/arc_summary/Makefile.am b/cmd/arc_summary/Makefile.am index ac7b0d48d0..a83edffadc 100644 --- a/cmd/arc_summary/Makefile.am +++ b/cmd/arc_summary/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = arc_summary.py arc_summary3.py +EXTRA_DIST = arc_summary2 arc_summary3 + +if USING_PYTHON_2 +dist_bin_SCRIPTS = arc_summary2 +install-exec-hook: + mv $(DESTDIR)$(bindir)/arc_summary2 $(DESTDIR)$(bindir)/arc_summary +endif + +if USING_PYTHON_3 +dist_bin_SCRIPTS = arc_summary3 +install-exec-hook: + mv $(DESTDIR)$(bindir)/arc_summary3 $(DESTDIR)$(bindir)/arc_summary +endif diff --git a/cmd/arc_summary/arc_summary.py b/cmd/arc_summary/arc_summary2 similarity index 99% rename from cmd/arc_summary/arc_summary.py rename to cmd/arc_summary/arc_summary2 index 642c94b690..ab4a3c574a 100755 --- a/cmd/arc_summary/arc_summary.py +++ b/cmd/arc_summary/arc_summary2 @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # $Id: arc_summary.pl,v 388:e27800740aa2 2011-07-08 02:53:29Z jhell $ # @@ -35,6 +35,8 @@ # Note some of this code uses older code (eg getopt instead of argparse, # subprocess.Popen() instead of subprocess.run()) because we need to support # some very old versions of Python. +# + """Print statistics on the ZFS Adjustable Replacement Cache (ARC) Provides basic information on the ARC, its efficiency, the L2ARC (if present), @@ -1005,7 +1007,7 @@ def zfs_header(): def usage(): """Print usage information""" - sys.stdout.write("Usage: arc_summary.py [-h] [-a] [-d] [-p PAGE]\n\n") + sys.stdout.write("Usage: arc_summary [-h] [-a] [-d] [-p PAGE]\n\n") sys.stdout.write("\t -h, --help : " "Print this help message and exit\n") sys.stdout.write("\t -a, --alternate : " @@ -1018,10 +1020,10 @@ def usage(): "should be an integer between 1 and " + str(len(unSub)) + "\n\n") sys.stdout.write("Examples:\n") - sys.stdout.write("\tarc_summary.py -a\n") - sys.stdout.write("\tarc_summary.py -p 4\n") - sys.stdout.write("\tarc_summary.py -ad\n") - sys.stdout.write("\tarc_summary.py --page=2\n") + sys.stdout.write("\tarc_summary -a\n") + sys.stdout.write("\tarc_summary -p 4\n") + sys.stdout.write("\tarc_summary -ad\n") + sys.stdout.write("\tarc_summary --page=2\n") def main(): diff --git a/cmd/arc_summary/arc_summary3.py b/cmd/arc_summary/arc_summary3 similarity index 99% rename from cmd/arc_summary/arc_summary3.py rename to cmd/arc_summary/arc_summary3 index e70f2a35e6..e67cd90f71 100755 --- a/cmd/arc_summary/arc_summary3.py +++ b/cmd/arc_summary/arc_summary3 @@ -346,7 +346,7 @@ def get_version(request): error_msg = '(ERROR: "{0}" requested)'.format(request) return error_msg - # The original arc_summary.py called /sbin/modinfo/{spl,zfs} to get + # The original arc_summary called /sbin/modinfo/{spl,zfs} to get # the version information. We switch to /sys/module/{spl,zfs}/version # to make sure we get what is really loaded in the kernel command = ["cat", "/sys/module/{0}/version".format(request)] @@ -374,7 +374,7 @@ def print_header(): """ # datetime is now recommended over time but we keep the exact formatting - # from the older version of arc_summary.py in case there are scripts + # from the older version of arc_summary in case there are scripts # that expect it in this way daydate = time.strftime(DATE_FORMAT) spc_date = LINE_LENGTH-len(daydate) @@ -586,7 +586,7 @@ def section_archits(kstats_dict): # For some reason, anon_hits can turn negative, which is weird. Until we # have figured out why this happens, we just hide the problem, following - # the behavior of the original arc_summary.py + # the behavior of the original arc_summary. if anon_hits >= 0: prt_i2('Anonymously used:', f_perc(anon_hits, arc_stats['hits']), f_hits(anon_hits)) diff --git a/cmd/arcstat/Makefile.am b/cmd/arcstat/Makefile.am index 8987b2414c..462e9a6197 100644 --- a/cmd/arcstat/Makefile.am +++ b/cmd/arcstat/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = arcstat.py +dist_bin_SCRIPTS = arcstat + +# +# The arcstat script is compatibile with both Python 2.6 and 3.4. +# As such the python 3 shebang can be replaced at install time when +# targeting a python 2 system. This allows us to maintain a single +# version of the source. +# +if USING_PYTHON_2 +install-exec-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(bindir)/arcstat +endif diff --git a/cmd/arcstat/arcstat.py b/cmd/arcstat/arcstat similarity index 97% rename from cmd/arcstat/arcstat.py rename to cmd/arcstat/arcstat index a2c52ddb34..57a2d621f3 100755 --- a/cmd/arcstat/arcstat.py +++ b/cmd/arcstat/arcstat @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Print out ZFS ARC Statistics exported via kstat(1) # For a definition of fields, or usage, use arctstat.pl -v @@ -42,7 +42,8 @@ # @hdr is the array of fields that needs to be printed, so we # just iterate over this array and print the values using our pretty printer. # - +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import sys import time @@ -109,7 +110,7 @@ opfile = None sep = " " # Default separator is 2 spaces version = "0.4" l2exist = False -cmd = ("Usage: arcstat.py [-hvx] [-f fields] [-o file] [-s string] [interval " +cmd = ("Usage: arcstat [-hvx] [-f fields] [-o file] [-s string] [interval " "[count]]\n") cur = {} d = {} @@ -138,10 +139,10 @@ def usage(): sys.stderr.write("\t -s : Override default field separator with custom " "character or string\n") sys.stderr.write("\nExamples:\n") - sys.stderr.write("\tarcstat.py -o /tmp/a.log 2 10\n") - sys.stderr.write("\tarcstat.py -s \",\" -o /tmp/a.log 2 10\n") - sys.stderr.write("\tarcstat.py -v\n") - sys.stderr.write("\tarcstat.py -f time,hit%,dh%,ph%,mh% 1\n") + sys.stderr.write("\tarcstat -o /tmp/a.log 2 10\n") + sys.stderr.write("\tarcstat -s \",\" -o /tmp/a.log 2 10\n") + sys.stderr.write("\tarcstat -v\n") + sys.stderr.write("\tarcstat -f time,hit%,dh%,ph%,mh% 1\n") sys.stderr.write("\n") sys.exit(1) diff --git a/cmd/dbufstat/Makefile.am b/cmd/dbufstat/Makefile.am index 19bffb0200..968a760779 100644 --- a/cmd/dbufstat/Makefile.am +++ b/cmd/dbufstat/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = dbufstat.py +dist_bin_SCRIPTS = dbufstat + +# +# The dbufstat script is compatibile with both Python 2.6 and 3.4. +# As such the python 3 shebang can be replaced at install time when +# targeting a python 2 system. This allows us to maintain a single +# version of the source. +# +if USING_PYTHON_2 +install-exec-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(bindir)/dbufstat +endif diff --git a/cmd/dbufstat/dbufstat.py b/cmd/dbufstat/dbufstat similarity index 97% rename from cmd/dbufstat/dbufstat.py rename to cmd/dbufstat/dbufstat index 5e2217a545..e6c947fbcb 100755 --- a/cmd/dbufstat/dbufstat.py +++ b/cmd/dbufstat/dbufstat @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Print out statistics for all cached dmu buffers. This information # is available through the dbufs kstat and may be post-processed as @@ -27,6 +27,8 @@ # Copyright (C) 2013 Lawrence Livermore National Security, LLC. # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import sys import getopt @@ -106,7 +108,7 @@ cols = { hdr = None xhdr = None sep = " " # Default separator is 2 spaces -cmd = ("Usage: dbufstat.py [-bdhnrtvx] [-i file] [-f fields] [-o file] " +cmd = ("Usage: dbufstat [-bdhnrtvx] [-i file] [-f fields] [-o file] " "[-s string] [-F filter]\n") raw = 0 @@ -167,11 +169,11 @@ def usage(): "character or string\n") sys.stderr.write("\t -F : Filter output by value or regex\n") sys.stderr.write("\nExamples:\n") - sys.stderr.write("\tdbufstat.py -d -o /tmp/d.log\n") - sys.stderr.write("\tdbufstat.py -t -s \",\" -o /tmp/t.log\n") - sys.stderr.write("\tdbufstat.py -v\n") - sys.stderr.write("\tdbufstat.py -d -f pool,object,objset,dsize,cached\n") - sys.stderr.write("\tdbufstat.py -bx -F dbc=1,objset=54,pool=testpool\n") + sys.stderr.write("\tdbufstat -d -o /tmp/d.log\n") + sys.stderr.write("\tdbufstat -t -s \",\" -o /tmp/t.log\n") + sys.stderr.write("\tdbufstat -v\n") + sys.stderr.write("\tdbufstat -d -f pool,object,objset,dsize,cached\n") + sys.stderr.write("\tdbufstat -bx -F dbc=1,objset=54,pool=testpool\n") sys.stderr.write("\n") sys.exit(1) diff --git a/config/always-python.m4 b/config/always-python.m4 new file mode 100644 index 0000000000..858ab7b015 --- /dev/null +++ b/config/always-python.m4 @@ -0,0 +1,102 @@ +dnl # +dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false]) +dnl # +dnl # Verify Python version +dnl # +AC_DEFUN([ZFS_AC_PYTHON_VERSION], [ + ver_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"` + AS_IF([test "$ver_check" = "True"], [ + m4_ifvaln([$2], [$2]) + ], [ + m4_ifvaln([$3], [$3]) + ]) +]) + +dnl # +dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false]) +dnl # +dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE +dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html +dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS. +dnl # +AC_DEFUN([ZFS_AC_PYTHON_MODULE], [ + PYTHON_NAME=`basename $PYTHON` + AC_MSG_CHECKING([for $PYTHON_NAME module: $1]) + AS_IF([$PYTHON -c "import $1" 2>/dev/null], [ + AC_MSG_RESULT(yes) + m4_ifvaln([$2], [$2]) + ], [ + AC_MSG_RESULT(no) + m4_ifvaln([$3], [$3]) + ]) +]) + +dnl # +dnl # The majority of the python scripts are written to be compatible +dnl # with Python 2.6 and Python 3.4. Therefore, they may be installed +dnl # and used with either interpreter. This option is intended to +dnl # to provide a method to specify the default system version, and +dnl # set the PYTHON environment variable accordingly. +dnl # +AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [ + AC_ARG_WITH([python], + AC_HELP_STRING([--with-python[=VERSION]], + [default system python version @<:@default=check@:>@]), + [with_python=$withval], + [with_python=check]) + + AS_CASE([$with_python], + [check], + [AS_IF([test -x /usr/bin/python3], + [PYTHON="python3"], + [AS_IF([test -x /usr/bin/python2], + [PYTHON="python2"], + [PYTHON=""] + )] + )], + [2*], [PYTHON="python${with_python}"], + [*python2*], [PYTHON="${with_python}"], + [3*], [PYTHON="python${with_python}"], + [*python3*], [PYTHON="${with_python}"], + [no], [PYTHON=""], + [AC_MSG_ERROR([Unknown --with-python value '$with_python'])] + ) + + AS_IF([$PYTHON --version >/dev/null 2>&1], [ /bin/true ], [ + AC_MSG_ERROR([Cannot find $PYTHON in your system path]) + ]) + + AM_PATH_PYTHON([2.6], [], [:]) + AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :]) + AM_CONDITIONAL([USING_PYTHON_2], [test "${PYTHON_VERSION:0:2}" = "2."]) + AM_CONDITIONAL([USING_PYTHON_3], [test "${PYTHON_VERSION:0:2}" = "3."]) + + dnl # + dnl # Minimum supported Python versions for utilities: + dnl # Python 2.6.x, or Python 3.4.x + dnl # + AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [ + ZFS_AC_PYTHON_VERSION([>= '2.6'], [ /bin/true ], + [AC_MSG_ERROR("Python >= 2.6.x is not available")]) + ]) + + AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [ + ZFS_AC_PYTHON_VERSION([>= '3.4'], [ /bin/true ], + [AC_MSG_ERROR("Python >= 3.4.x is not available")]) + ]) + + dnl # + dnl # Request that packages be built for a specific Python version. + dnl # + AS_IF([test $with_python != check], [ + PYTHON_PKG_VERSION=`echo ${PYTHON} | tr -d 'a-zA-Z.'` + DEFINE_PYTHON_PKG_VERSION='--define "__use_python_pkg_version '${PYTHON_PKG_VERSION}'"' + DEFINE_PYTHON_VERSION='--define "__use_python '${PYTHON}'"' + ], [ + DEFINE_PYTHON_VERSION='' + DEFINE_PYTHON_PKG_VERSION='' + ]) + + AC_SUBST(DEFINE_PYTHON_VERSION) + AC_SUBST(DEFINE_PYTHON_PKG_VERSION) +]) diff --git a/config/always-pyzfs.m4 b/config/always-pyzfs.m4 index c50acb0996..d74d6f1a75 100644 --- a/config/always-pyzfs.m4 +++ b/config/always-pyzfs.m4 @@ -1,80 +1,44 @@ dnl # -dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false]) +dnl # Determines if pyzfs can be built, requires Python 2.7 or latter. dnl # -dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE -dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html -dnl # -AC_DEFUN([ZFS_AC_PYTHON_MODULE],[ - PYTHON_NAME=`basename $PYTHON` - AC_MSG_CHECKING([for $PYTHON_NAME module: $1]) - $PYTHON -c "import $1" 2>/dev/null - if test $? -eq 0; - then - AC_MSG_RESULT(yes) - m4_ifvaln([$2], [$2]) - else - AC_MSG_RESULT(no) - m4_ifvaln([$3], [$3]) - fi -]) - -dnl # -dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false]) -dnl # -dnl # Verify Python version -dnl # -AC_DEFUN([ZFS_AC_PYTHON_VERSION], [ - AC_MSG_CHECKING([for a version of Python $1]) - version_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"` - if test "$version_check" = "True"; - then - AC_MSG_RESULT(yes) - m4_ifvaln([$2], [$2]) - else - AC_MSG_RESULT(no) - m4_ifvaln([$3], [$3]) - fi - -]) - AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ - PYTHON_REQUIRED_VERSION="<= '2.7.x'" - AC_ARG_ENABLE([pyzfs], AC_HELP_STRING([--enable-pyzfs], [install libzfs_core python bindings @<:@default=check@:>@]), [enable_pyzfs=$enableval], [enable_pyzfs=check]) - AM_PATH_PYTHON([2.7], [], [ + dnl # + dnl # Packages for pyzfs specifically enabled/disabled. + dnl # + AS_IF([test "x$enable_pyzfs" != xcheck], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python >= 2.7 is not installed") - ], [test ! "x$enable_pyzfs" = xno], [ - enable_pyzfs=no - ]) - ]) - AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != :]) - - dnl # - dnl # Python 2.7.x is supported, other versions (3.5) are not yet - dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ - ZFS_AC_PYTHON_VERSION([$PYTHON_REQUIRED_VERSION], [], [ - AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION is not available") - ], [test ! "x$enable_pyzfs" = xno], [ - enable_pyzfs=no - ]) + DEFINE_PYZFS='--with pyzfs' + ], [ + DEFINE_PYZFS='--without pyzfs' ]) + ], [ + DEFINE_PYZFS='' ]) + AC_SUBST(DEFINE_PYZFS) dnl # dnl # Require python-devel libraries dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ + AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [ + PYTHON_REQUIRED_VERSION=">= '2.7.0'" + ], [ + AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [ + PYTHON_REQUIRED_VERSION=">= '3.4.0'" + ], [ + AC_MSG_ERROR("Python $PYTHON_VERSION unknown") + ]) + ]) + AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("Python development library is not available") + AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION development library is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -84,10 +48,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ dnl # dnl # Python "setuptools" module is required to build and install pyzfs dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ ZFS_AC_PYTHON_MODULE([setuptools], [], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python-setuptools is not installed") + AC_MSG_ERROR("Python $PYTHON_VERSION setuptools is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -97,10 +61,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ dnl # dnl # Python "cffi" module is required to run pyzfs dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ ZFS_AC_PYTHON_MODULE([cffi], [], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python-cffi is not installed") + AC_MSG_ERROR("Python $PYTHON_VERSION cffi is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -114,12 +78,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ AM_CONDITIONAL([PYZFS_ENABLED], [test x$enable_pyzfs = xyes]) AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs]) - - AS_IF([test "x$enable_pyzfs" = xyes], [ - DEFINE_PYZFS='--define "_pyzfs 1"' - ],[ - DEFINE_PYZFS='' - ]) - AC_SUBST(DEFINE_PYZFS) AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG]) + + AC_MSG_CHECKING([whether to enable pyzfs: ]) + AC_MSG_RESULT($enable_pyzfs) ]) diff --git a/config/deb.am b/config/deb.am index eb4e5bbdae..e405547aa9 100644 --- a/config/deb.am +++ b/config/deb.am @@ -47,7 +47,7 @@ deb-utils: deb-local rpm-utils pkg7=$${name}-test-$${version}.$${arch}.rpm; \ pkg8=$${name}-dracut-$${version}.$${arch}.rpm; \ pkg9=$${name}-initramfs-$${version}.$${arch}.rpm; \ - pkg10=pyzfs-$${version}.noarch.rpm; \ + pkg10=`ls python*-pyzfs-$${version}* | tail -1`; \ ## Arguments need to be passed to dh_shlibdeps. Alien provides no mechanism ## to do this, so we install a shim onto the path which calls the real ## dh_shlibdeps with the required arguments. diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 1d47b03848..6e305996e5 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -160,6 +160,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [ ZFS_AC_CONFIG_ALWAYS_CC_ASAN ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD ZFS_AC_CONFIG_ALWAYS_ARCH + ZFS_AC_CONFIG_ALWAYS_PYTHON ZFS_AC_CONFIG_ALWAYS_PYZFS ]) @@ -264,10 +265,13 @@ AC_DEFUN([ZFS_AC_RPM], [ RPM_DEFINE_UTIL+=' $(DEFINE_INITRAMFS)' RPM_DEFINE_UTIL+=' $(DEFINE_SYSTEMD)' RPM_DEFINE_UTIL+=' $(DEFINE_PYZFS)' + RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_VERSION)' + RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_PKG_VERSION)' - dnl # Override default lib directory on Debian/Ubuntu systems. The provided - dnl # /usr/lib/rpm/platform//macros files do not specify the correct - dnl # path for multiarch systems as described by the packaging guidelines. + dnl # Override default lib directory on Debian/Ubuntu systems. The + dnl # provided /usr/lib/rpm/platform//macros files do not + dnl # specify the correct path for multiarch systems as described + dnl # by the packaging guidelines. dnl # dnl # https://wiki.ubuntu.com/MultiarchSpec dnl # https://wiki.debian.org/Multiarch/Implementation diff --git a/contrib/pyzfs/Makefile.am b/contrib/pyzfs/Makefile.am index f27216a77e..36290661f7 100644 --- a/contrib/pyzfs/Makefile.am +++ b/contrib/pyzfs/Makefile.am @@ -27,7 +27,7 @@ install-exec-local: $(PYTHON) $(srcdir)/setup.py install \ --prefix $(prefix) \ --root $(DESTDIR)/ \ - --install-lib $(pythondir) \ + --install-lib $(pythonsitedir) \ --single-version-externally-managed \ --verbose diff --git a/contrib/pyzfs/setup.py b/contrib/pyzfs/setup.py index e76ffbf821..3ff6c04c69 100644 --- a/contrib/pyzfs/setup.py +++ b/contrib/pyzfs/setup.py @@ -29,8 +29,13 @@ setup( "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 2 :: Only", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: System :: Filesystems", "Topic :: Software Development :: Libraries", ], @@ -48,7 +53,7 @@ setup( setup_requires=[ "cffi", ], - python_requires='>=2.7,<3', + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<4', zip_safe=False, test_suite="libzfs_core.test", ) diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index 55edbc83fa..533792989b 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -53,16 +53,6 @@ %bcond_with asan %bcond_with systemd -# Python permits the !/usr/bin/python shebang for scripts that are cross -# compatible between python2 and python3, but Fedora 28 does not. Fedora -# wants us to choose python3 for cross-compatible scripts. Since we want -# to support python2 and python3 users, exclude our scripts from Fedora 28's -# RPM build check, so that we don't get a bunch of build warnings. -# -# Details: https://github.com/zfsonlinux/zfs/issues/7360 -# -%global __brp_mangle_shebangs_exclude_from arc_summary.py|arcstat.py|dbufstat.py|test-runner.py|zts-report.py - # Generic enable switch for systemd %if %{with systemd} %define _systemd 1 @@ -85,6 +75,32 @@ %define _systemd 1 %endif +# When not specified default to distribution provided version. This +# is normally Python 3, but for RHEL <= 7 only Python 2 is provided. +%if %{undefined __use_python} +%if 0%{?rhel} && 0%{?rhel} <= 7 +%define __python /usr/bin/python2 +%define __python_pkg_version 2 +%define __python_cffi_pkg python-cffi +%else +%define __python /usr/bin/python3 +%define __python_pkg_version 3 +%define __python_cffi_pkg python3-cffi +%endif +%else +%define __python %{__use_python} +%define __python_pkg_version %{__use_python_pkg_version} +%define __python_cffi_pkg python%{__python_pkg_version}-cffi +%endif + +# By default python-pyzfs is enabled, with the exception of +# RHEL 6 which by default uses Python 2.6 which is too old. +%if 0%{?rhel} == 6 +%bcond_with pyzfs +%else +%bcond_without pyzfs +%endif + Name: @PACKAGE@ Version: @VERSION@ Release: @RELEASE@%{?dist} @@ -135,7 +151,7 @@ Requires: util-linux Requires: sysstat %description -This package contains the ZFS command line utilities. +This package contains the core ZFS command line utilities. %package -n libzpool2 Summary: Native ZFS pool library for Linux @@ -219,6 +235,7 @@ Requires: acl Requires: sudo Requires: sysstat Requires: libaio +Requires: python%{__python_pkg_version} %if 0%{?rhel}%{?fedora}%{?suse_version} BuildRequires: libaio-devel %endif @@ -240,23 +257,23 @@ Requires: grep This package contains a dracut module used to construct an initramfs image which is ZFS aware. -%if 0%{?_pyzfs} -%package -n pyzfs -Summary: Python wrapper for libzfs_core +%if %{with pyzfs} +%package -n python%{__python_pkg_version}-pyzfs +Summary: Python %{python_version} wrapper for libzfs_core Group: Development/Languages/Python License: Apache-2.0 BuildArch: noarch Requires: libzfs2 = %{version} Requires: libnvpair1 = %{version} Requires: libffi -Requires: python >= 2.7 -Requires: python-cffi +Requires: python%{__python_pkg_version} +Requires: %{__python_cffi_pkg} %if 0%{?rhel}%{?fedora}%{?suse_version} -BuildRequires: python-devel +BuildRequires: python%{__python_pkg_version}-devel BuildRequires: libffi-devel %endif -%description -n pyzfs +%description -n python%{__python_pkg_version}-pyzfs This package provides a python wrapper for the libzfs_core C library. %endif @@ -299,6 +316,12 @@ image which is ZFS aware. %define systemd --enable-sysvinit --disable-systemd %endif +%if %{with pyzfs} + %define pyzfs --enable-pyzfs +%else + %define pyzfs --disable-pyzfs +%endif + %setup -q %build @@ -307,11 +330,13 @@ image which is ZFS aware. --with-udevdir=%{_udevdir} \ --with-udevruledir=%{_udevruledir} \ --with-dracutdir=%{_dracutdir} \ + --with-python=%{__python} \ --disable-static \ %{debug} \ %{debuginfo} \ %{asan} \ - %{systemd} + %{systemd}\ + %{pyzfs} make %{?_smp_mflags} %install @@ -379,12 +404,20 @@ systemctl --system daemon-reload >/dev/null || true %endif %files +# Core utilities %{_sbindir}/* -%{_bindir}/* -%{_libexecdir}/%{name} +%{_bindir}/raidz_test +%{_bindir}/zgenhostid +# Optional Python 2/3 scripts +%{_bindir}/arc_summary +%{_bindir}/arcstat +%{_bindir}/dbufstat +# Man pages %{_mandir}/man1/* %{_mandir}/man5/* %{_mandir}/man8/* +# Configuration files and scripts +%{_libexecdir}/%{name} %{_udevdir}/vdev_id %{_udevdir}/zvol_id %{_udevdir}/rules.d/* @@ -426,8 +459,8 @@ systemctl --system daemon-reload >/dev/null || true %doc contrib/dracut/README.dracut.markdown %{_dracutdir}/modules.d/* -%if 0%{?_pyzfs} -%files -n pyzfs +%if %{with pyzfs} +%files -n python%{__python_pkg_version}-pyzfs %doc contrib/pyzfs/README %doc contrib/pyzfs/LICENSE %defattr(-,root,root,-) diff --git a/tests/test-runner/bin/Makefile.am b/tests/test-runner/bin/Makefile.am index e843e4e09e..30c564e555 100644 --- a/tests/test-runner/bin/Makefile.am +++ b/tests/test-runner/bin/Makefile.am @@ -2,3 +2,14 @@ pkgdatadir = $(datadir)/@PACKAGE@/test-runner/bin dist_pkgdata_SCRIPTS = \ test-runner.py \ zts-report.py +# +# These scripts are compatibile with both Python 2.6 and 3.4. As such the +# python 3 shebang can be replaced at install time when targeting a python +# 2 system. This allows us to maintain a single version of the source. +# +if USING_PYTHON_2 +install-data-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(pkgdatadir)/test-runner.py \ + $(DESTDIR)$(pkgdatadir)/zts-report.py +endif diff --git a/tests/test-runner/bin/test-runner.py b/tests/test-runner/bin/test-runner.py index 7ef8a87ed3..2e26fa2664 100755 --- a/tests/test-runner/bin/test-runner.py +++ b/tests/test-runner/bin/test-runner.py @@ -15,6 +15,8 @@ # Copyright (c) 2012, 2015 by Delphix. All rights reserved. # Copyright (c) 2017 Datto Inc. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# # some python 2.7 system don't have a configparser shim try: diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py index 9502956017..4e51bc94ea 100755 --- a/tests/test-runner/bin/zts-report.py +++ b/tests/test-runner/bin/zts-report.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # This file and its contents are supplied under the terms of the @@ -15,6 +15,8 @@ # Copyright (c) 2017 by Delphix. All rights reserved. # Copyright (c) 2018 by Lawrence Livermore National Security, LLC. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import os import re diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 5efcb61022..8ced03e930 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -146,10 +146,10 @@ export ZFS_FILES='zdb zpool ztest raidz_test - arc_summary.py - arc_summary3.py - arcstat.py - dbufstat.py + arc_summary + arc_summary3 + arcstat + dbufstat zed zgenhostid zstreamdump' diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh index 5ceff962d4..7ec9eaf4c5 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh @@ -37,7 +37,7 @@ # 2. Store output from dbufs kstat # 3. Store output from dbufstats kstat # 4. Compare stats presented in dbufstats with stat generated using -# dbufstat.py and the dbufs kstat output +# dbufstat and the dbufs kstat output # DBUFSTATS_FILE=$(mktemp $TEST_BASE_DIR/dbufstats.out.XXXXXX) @@ -56,7 +56,7 @@ function testdbufstat # stat_name dbufstat_filter [[ -n "$2" ]] && filter="-F $2" from_dbufstat=$(grep -w "$name" "$DBUFSTATS_FILE" | awk '{ print $3 }') - from_dbufs=$(dbufstat.py -bxn -i "$DBUFS_FILE" "$filter" | wc -l) + from_dbufs=$(dbufstat -bxn -i "$DBUFS_FILE" "$filter" | wc -l) within_tolerance $from_dbufstat $from_dbufs 9 \ || log_fail "Stat $name exceeded tolerance" diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh index e256bfabe3..dc30b66065 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh @@ -62,18 +62,18 @@ objid=$(stat --format="%i" "$TESTDIR/file") log_note "Object ID for $TESTDIR/file is $objid" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mru" "mru count" verify_eq "0" "$mfu" "mfu count" log_must eval "cat $TESTDIR/file > /dev/null" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mfu" "mfu count" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh index 22dceaaf45..ff090baeee 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh @@ -45,12 +45,12 @@ fi set -A args "" "-a" "-d" "-p 1" "-g" "-s arc" "-r" -log_assert "arc_summary3.py generates output and doesn't return an error code" +log_assert "arc_summary3 generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary3.py ${args[i]} > /dev/null" + log_must eval "arc_summary3 ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arc_summary3.py generates output and doesn't return an error code" +log_pass "arc_summary3 generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh index 6653b9c1ad..8736b18d84 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-a" "-d" "-p 1" -log_assert "arc_summary.py generates output and doesn't return an error code" +log_assert "arc_summary generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary.py ${args[i]} > /dev/null" + log_must eval "arc_summary ${args[i]} > /dev/null" ((i = i + 1)) done -log_must eval "arc_summary.py | head > /dev/null" -log_must eval "arc_summary.py | head -1 > /dev/null" +log_must eval "arc_summary | head > /dev/null" +log_must eval "arc_summary | head -1 > /dev/null" -log_pass "arc_summary.py generates output and doesn't return an error code" +log_pass "arc_summary generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh index e63552feb4..477b7ca08f 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh @@ -29,10 +29,10 @@ typeset args=("-x" "-r" "-5" "-p 7" "--err" "-@") -log_assert "arc_summary.py generates an error code with invalid options" +log_assert "arc_summary generates an error code with invalid options" for arg in "${args[@]}"; do - log_mustnot eval "arc_summary.py $arg > /dev/null" + log_mustnot eval "arc_summary $arg > /dev/null" done -log_pass "arc_summary.py generates an error code with invalid options" +log_pass "arc_summary generates an error code with invalid options" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh index c8a89f8c4b..ab574731fe 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh @@ -30,12 +30,12 @@ set -A args "" "-s \",\"" "-x" "-v" \ "-f time,hit%,dh%,ph%,mh%" -log_assert "arcstat.py generates output and doesn't return an error code" +log_assert "arcstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arcstat.py ${args[i]} > /dev/null" + log_must eval "arcstat ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arcstat.py generates output and doesn't return an error code" +log_pass "arcstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh index 1c267d6af0..95f0598c61 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" "-n" -log_assert "dbufstat.py generates output and doesn't return an error code" +log_assert "dbufstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "dbufstat.py ${args[i]} > /dev/null" + log_must eval "dbufstat ${args[i]} > /dev/null" ((i = i + 1)) done -# A simple test of dbufstat.py filter functionality -log_must eval "dbufstat.py -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" +# A simple test of dbufstat filter functionality +log_must eval "dbufstat -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" -log_pass "dbufstat.py generates output and doesn't return an error code" +log_pass "dbufstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/pyzfs/.gitignore b/tests/zfs-tests/tests/functional/pyzfs/.gitignore new file mode 100644 index 0000000000..bcbe0573e3 --- /dev/null +++ b/tests/zfs-tests/tests/functional/pyzfs/.gitignore @@ -0,0 +1 @@ +pyzfs_unittest.ksh diff --git a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am index 61cb3d074d..0a27adecca 100644 --- a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am +++ b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am @@ -1,4 +1,18 @@ -pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs - -dist_pkgdata_SCRIPTS = \ +pkgpyzfsdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs +pkgpyzfs_SCRIPTS = \ pyzfs_unittest.ksh + +EXTRA_DIST = \ + pyzfs_unittest.ksh.in + +# +# The pyzfs module is built either for Python 2 or Python 3. In order +# to properly test it the unit tests must be updated to the matching vesion. +# +$(pkgpyzfs_SCRIPTS):%:%.in + -$(SED) -e 's,@PYTHON\@,$(PYTHON),g' \ + $< >'$@' + -chmod 775 $@ + +distclean-local:: + -$(RM) $(pkgpyzfs_SCRIPTS) diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in similarity index 95% rename from tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh rename to tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in index fb4b603617..4ca610e5f1 100755 --- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh +++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in @@ -28,7 +28,7 @@ verify_runnable "global" # Verify that the required dependencies for testing are installed. -python -c "import cffi" 2>/dev/null +@PYTHON@ -c "import cffi" 2>/dev/null if [ $? -eq 1 ]; then log_unsupported "python-cffi not found by Python" fi @@ -37,7 +37,7 @@ fi # only if pyzfs was not installed due to missing, build-time, dependencies; if # we cannot load "libzfs_core" due to other reasons, for instance an API/ABI # mismatch, we want to report it. -python -c ' +@PYTHON@ -c ' import pkgutil, sys sys.exit(pkgutil.find_loader("libzfs_core") is None)' if [ $? -eq 1 ]; then @@ -47,7 +47,7 @@ fi log_assert "Verify the nvlist and libzfs_core Python unittest run successfully" # NOTE: don't use log_must() here because it makes output unreadable -python -m unittest --verbose \ +@PYTHON@ -m unittest --verbose \ libzfs_core.test.test_nvlist.TestNVList \ libzfs_core.test.test_libzfs_core.ZFSTest if [ $? -ne 0 ]; then