pyzfs: python3 support (build system)

Almost all of the Python code in the respository has been updated
to be compatibile with Python 2.6, Python 3.4, or newer.  The only
exceptions are arc_summery3.py which requires Python 3, and pyzfs
which requires at least Python 2.7.  This allows us to maintain a
single version of the code and support most default versions of
python.  This change does the following:

* Sets the default shebang for all Python scripts to python3.  If
  only Python 2 is available, then at install time scripts which
  are compatible with Python 2 will have their shebangs replaced
  with /usr/bin/python.  This is done for compatibility until
  Python 2 goes end of life.  Since only the installed versions
  are changed this means Python 3 must be installed on the system
  for test-runner when testing in-tree.

* Added --with-python=<2|3|3.4,etc> configure option which sets
  the PYTHON environment variable to target a specific python
  version.  By default the newest installed version of Python
  will be used or the preferred distribution version when
  creating pacakges.

* Fixed --enable-pyzfs configure checks so they are run when
  --enable-pyzfs=check and --enable-pyzfs=yes.

* Enabled pyzfs for Python 3.4 and newer, which is now supported.

* Renamed pyzfs package to python<VERSION>-pyzfs and updated to
  install in the appropriate site location.  For example, when
  building with --with-python=3.4 a python34-pyzfs will be
  created which installs in /usr/lib/python3.4/site-packages/.

* Renamed the following python scripts according to the Fedora
  guidance for packaging utilities in /bin

  - dbufstat.py     -> dbufstat
  - arcstat.py      -> arcstat
  - arc_summary.py  -> arc_summary
  - arc_summary3.py -> arc_summary3

* Updated python-cffi package name.  On CentOS 6, CentOS 7, and
  Amazon Linux it's called python-cffi, not python2-cffi.  For
  Python3 it's called python3-cffi or python3x-cffi.

* Install one version of arc_summary.  Depending on the version
  of Python available install either arc_summary2 or arc_summary3
  as arc_summary.  The user output is only slightly different.

Reviewed-by: John Ramsden <johnramsden@riseup.net>
Reviewed-by: Neal Gompa <ngompa@datto.com>
Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #8096

tests/test-runner/bin/Makefile.am

@@ -2,3 +2,14 @@ pkgdatadir = $(datadir)/@PACKAGE@/test-runner/bin
 dist_pkgdata_SCRIPTS = \
 	test-runner.py \
 	zts-report.py
+#
+# These scripts are compatibile with both Python 2.6 and 3.4.  As such the
+# python 3 shebang can be replaced at install time when targeting a python
+# 2 system.  This allows us to maintain a single version of the source.
+#
+if USING_PYTHON_2
+install-data-hook:
+	sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \
+	    $(DESTDIR)$(pkgdatadir)/test-runner.py \
+	    $(DESTDIR)$(pkgdatadir)/zts-report.py
+endif

tests/test-runner/bin/test-runner.py

@@ -15,6 +15,8 @@
 # Copyright (c) 2012, 2015 by Delphix. All rights reserved.
 # Copyright (c) 2017 Datto Inc.
 #
+# This script must remain compatible with Python 2.6+ and Python 3.4+.
+#
 
 # some python 2.7 system don't have a configparser shim
 try:

tests/test-runner/bin/zts-report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 #
 # This file and its contents are supplied under the terms of the
@@ -15,6 +15,8 @@
 # Copyright (c) 2017 by Delphix. All rights reserved.
 # Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
 #
+# This script must remain compatible with Python 2.6+ and Python 3.4+.
+#
 
 import os
 import re

tests/zfs-tests/include/commands.cfg

@@ -146,10 +146,10 @@ export ZFS_FILES='zdb
     zpool
     ztest
     raidz_test
-    arc_summary.py
-    arc_summary3.py
-    arcstat.py
-    dbufstat.py
+    arc_summary
+    arc_summary3
+    arcstat
+    dbufstat
     zed
     zgenhostid
     zstreamdump'

tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh

@@ -37,7 +37,7 @@
 # 2. Store output from dbufs kstat
 # 3. Store output from dbufstats kstat
 # 4. Compare stats presented in dbufstats with stat generated using
-#    dbufstat.py and the dbufs kstat output
+#    dbufstat and the dbufs kstat output
 #
 
 DBUFSTATS_FILE=$(mktemp $TEST_BASE_DIR/dbufstats.out.XXXXXX)
@@ -56,7 +56,7 @@ function testdbufstat # stat_name dbufstat_filter
         [[ -n "$2" ]] && filter="-F $2"
 
 	from_dbufstat=$(grep -w "$name" "$DBUFSTATS_FILE" | awk '{ print $3 }')
-	from_dbufs=$(dbufstat.py -bxn -i "$DBUFS_FILE" "$filter" | wc -l)
+	from_dbufs=$(dbufstat -bxn -i "$DBUFS_FILE" "$filter" | wc -l)
 
 	within_tolerance $from_dbufstat $from_dbufs 9 \
 	    || log_fail "Stat $name exceeded tolerance"

tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh

@@ -62,18 +62,18 @@ objid=$(stat --format="%i" "$TESTDIR/file")
 log_note "Object ID for $TESTDIR/file is $objid"
 
 log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE"
-dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
-mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
-mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
+dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
+mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
+mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
 log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu"
 verify_ne "0" "$mru" "mru count"
 verify_eq "0" "$mfu" "mfu count"
 
 log_must eval "cat $TESTDIR/file > /dev/null"
 log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE"
-dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
-mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
-mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
+dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
+mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
+mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
 log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu"
 verify_ne "0" "$mfu" "mfu count"
 

tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh

@@ -45,12 +45,12 @@ fi
 
 
 set -A args  "" "-a" "-d" "-p 1" "-g" "-s arc" "-r"
-log_assert "arc_summary3.py generates output and doesn't return an error code"
+log_assert "arc_summary3 generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arc_summary3.py ${args[i]} > /dev/null"
+        log_must eval "arc_summary3 ${args[i]} > /dev/null"
         ((i = i + 1))
 done
 
-log_pass "arc_summary3.py generates output and doesn't return an error code"
+log_pass "arc_summary3 generates output and doesn't return an error code"

tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh

@@ -29,15 +29,15 @@
 
 set -A args  "" "-a" "-d" "-p 1"
 
-log_assert "arc_summary.py generates output and doesn't return an error code"
+log_assert "arc_summary generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arc_summary.py ${args[i]} > /dev/null"
+        log_must eval "arc_summary ${args[i]} > /dev/null"
         ((i = i + 1))
 done
 
-log_must eval "arc_summary.py | head > /dev/null"
-log_must eval "arc_summary.py | head -1 > /dev/null"
+log_must eval "arc_summary | head > /dev/null"
+log_must eval "arc_summary | head -1 > /dev/null"
 
-log_pass "arc_summary.py generates output and doesn't return an error code"
+log_pass "arc_summary generates output and doesn't return an error code"

tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh

@@ -29,10 +29,10 @@
 
 typeset args=("-x" "-r" "-5" "-p 7" "--err" "-@")
 
-log_assert "arc_summary.py generates an error code with invalid options"
+log_assert "arc_summary generates an error code with invalid options"
 
 for arg in "${args[@]}"; do
-        log_mustnot eval "arc_summary.py $arg > /dev/null"
+        log_mustnot eval "arc_summary $arg > /dev/null"
 done
 
-log_pass "arc_summary.py generates an error code with invalid options"
+log_pass "arc_summary generates an error code with invalid options"

tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh

@@ -30,12 +30,12 @@
 set -A args  "" "-s \",\"" "-x" "-v" \
     "-f time,hit%,dh%,ph%,mh%"
 
-log_assert "arcstat.py generates output and doesn't return an error code"
+log_assert "arcstat generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arcstat.py ${args[i]} > /dev/null"
+        log_must eval "arcstat ${args[i]} > /dev/null"
         ((i = i + 1))
 done
-log_pass "arcstat.py generates output and doesn't return an error code"
+log_pass "arcstat generates output and doesn't return an error code"
 

tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh

@@ -29,15 +29,15 @@
 
 set -A args  "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" "-n"
 
-log_assert "dbufstat.py generates output and doesn't return an error code"
+log_assert "dbufstat generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "dbufstat.py ${args[i]} > /dev/null"
+        log_must eval "dbufstat ${args[i]} > /dev/null"
         ((i = i + 1))
 done
 
-# A simple test of dbufstat.py filter functionality
-log_must eval "dbufstat.py -F object=10,dbc=1,pool=$TESTPOOL > /dev/null"
+# A simple test of dbufstat filter functionality
+log_must eval "dbufstat -F object=10,dbc=1,pool=$TESTPOOL > /dev/null"
 
-log_pass "dbufstat.py generates output and doesn't return an error code"
+log_pass "dbufstat generates output and doesn't return an error code"

tests/zfs-tests/tests/functional/pyzfs/.gitignore

@@ -0,0 +1 @@
+pyzfs_unittest.ksh

tests/zfs-tests/tests/functional/pyzfs/Makefile.am

@@ -1,4 +1,18 @@
-pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs
-
-dist_pkgdata_SCRIPTS = \
+pkgpyzfsdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs
+pkgpyzfs_SCRIPTS = \
 	pyzfs_unittest.ksh
+
+EXTRA_DIST = \
+	pyzfs_unittest.ksh.in
+
+#
+# The pyzfs module is built either for Python 2 or Python 3.  In order
+# to properly test it the unit tests must be updated to the matching vesion.
+#
+$(pkgpyzfs_SCRIPTS):%:%.in
+	-$(SED) -e 's,@PYTHON\@,$(PYTHON),g' \
+		$< >'$@'
+	-chmod 775 $@
+
+distclean-local::
+	-$(RM) $(pkgpyzfs_SCRIPTS)

tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in

@@ -28,7 +28,7 @@
 verify_runnable "global"
 
 # Verify that the required dependencies for testing are installed.
-python -c "import cffi" 2>/dev/null
+@PYTHON@ -c "import cffi" 2>/dev/null
 if [ $? -eq 1 ]; then
 	log_unsupported "python-cffi not found by Python"
 fi
@@ -37,7 +37,7 @@ fi
 # only if pyzfs was not installed due to missing, build-time, dependencies; if
 # we cannot load "libzfs_core" due to other reasons, for instance an API/ABI
 # mismatch, we want to report it.
-python -c '
+@PYTHON@ -c '
 import pkgutil, sys
 sys.exit(pkgutil.find_loader("libzfs_core") is None)'
 if [ $? -eq 1 ]; then
@@ -47,7 +47,7 @@ fi
 log_assert "Verify the nvlist and libzfs_core Python unittest run successfully"
 
 # NOTE: don't use log_must() here because it makes output unreadable
-python -m unittest --verbose \
+@PYTHON@ -m unittest --verbose \
 	libzfs_core.test.test_nvlist.TestNVList \
 	libzfs_core.test.test_libzfs_core.ZFSTest
 if [ $? -ne 0 ]; then