backport fix for AMX register breakage

vmexit's can cause the AMX registers to "misbehave" which can break
ZFS, even though ZFS doesn't use AMX at all.

This causes crashes and processes hanging forever in uninterruptible
sleep (the infamous D state) on Intel Xeon 4th gen HW, possible other
HW too, but we only got reports on Sapphire Rapids models.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
Tested-by: Aaron Lauterer <a.lauterer@proxmox.com>

.gitignore

@@ -0,0 +1,7 @@
+/*.build
+/*.buildinfo
+/*.changes
+/*.deb
+/*.dsc
+/*.tar*
+/zfs-utils-*.*/

Makefile

@@ -1,9 +1,11 @@
 include /usr/share/dpkg/default.mk
 # source form https://github.com/zfsonlinux/
 
-ZFSDIR=zfs-linux_$(DEB_VERSION_UPSTREAM)
-ZFSSRC=upstream
-ORIG_SRC_TAR=$(ZFSDIR).orig.tar.gz
+PACKAGE = zfs-linux
+
+SRCDIR = upstream
+BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
+ORIG_SRC_TAR = $(PACKAGE)_$(DEB_VERSION_UPSTREAM).orig.tar.gz
 
 ZFS_DEB1= libnvpair3linux_$(DEB_VERSION)_amd64.deb
 
@@ -30,10 +32,14 @@ DEBS= $(ZFS_DEB1) $(ZFS_DEB2) $(ZFS_DBG_DEBS)
 ZFS_DSC = zfs-linux_$(DEB_VERSION).dsc
 
 all: deb
-.PHONY: deb
+
+.PHONY: deb dsc
 deb: $(DEBS)
-.PHONY: dsc
-dsc: $(ZFS_DSC)
+
+dsc:
+	rm -rf *.dsc $(BUILDDIR)
+	$(MAKE) $(ZFS_DSC)
+	lintian $(ZFS_DSC)
 
 # called from pve-kernel's Makefile to get patched sources
 .PHONY: kernel
@@ -47,38 +53,41 @@ dinstall: $(DEBS)
 
 .PHONY: submodule
 submodule:
-	test -f "$(ZFSSRC)/README.md" || git submodule update --init
-$(ZFSSRC)/README.md: submodule
+	test -f "$(SRCDIR)/README.md" || git submodule update --init
+
+$(SRCDIR)/README.md: submodule
 
 .PHONY: zfs
 zfs: $(DEBS)
 $(ZFS_DEB2) $(ZFS_DBG_DEBS): $(ZFS_DEB1)
-$(ZFS_DEB1): $(ZFSDIR)
-	cd $(ZFSDIR); dpkg-buildpackage -b -uc -us
+$(ZFS_DEB1): $(BUILDDIR)
+	cd $(BUILDDIR); dpkg-buildpackage -b -uc -us
 	lintian $(DEBS)
 
-$(ORIG_SRC_TAR): $(ZFSDIR)
-	tar czf $(ORIG_SRC_TAR) --exclude="$(ZFSDIR)/debian" $(ZFSDIR)
+$(ORIG_SRC_TAR): $(BUILDDIR)
+	tar czf $(ORIG_SRC_TAR) --exclude="$(BUILDDIR)/debian" $(BUILDDIR)
 
-$(ZFS_DSC): $(ZFSDIR) $(ORIG_SRC_TAR)
-	tar czf zfs-linux_$(ZFSVER).orig.tar.gz $(ZFSDIR)
-	cd $(ZFSDIR); dpkg-buildpackage -S -uc -us -d
-	lintian $@
+$(ZFS_DSC): $(BUILDDIR) $(ORIG_SRC_TAR)
+	cd $(BUILDDIR); dpkg-buildpackage -S -uc -us -d
 
-$(ZFSDIR): $(ZFSSRC)/README.md $(ZFSSRC) debian
-	rm -rf $(ZFSDIR) $(ZFSDIR).tmp
-	cp -a $(ZFSSRC) $(ZFSDIR).tmp
-	cp -a debian $(ZFSDIR).tmp/debian
-	mv $(ZFSDIR).tmp $(ZFSDIR)
+sbuild: $(ZFS_DSC)
+	sbuild $(ZFS_DSC)
 
+$(BUILDDIR): $(SRCDIR)/README.md $(SRCDIR) debian
+	rm -rf $@ $@.tmp
+	cp -a $(SRCDIR) $@.tmp
+	cp -a debian $@.tmp/debian
+	mv $@.tmp $@
 
 .PHONY: clean
 clean: 	
-	rm -rf *~ *.deb *.changes *.buildinfo *.build *.dsc *.orig.tar.* *.debian.tar.* $(ZFSDIR)
+	rm -rf $(PACKAGE)-[0-9]*/
+	rm -f *~ *.deb *.changes *.buildinfo *.build *.dsc *.orig.tar.* *.debian.tar.*
 
 .PHONY: distclean
 distclean: clean
 
 .PHONY: upload
+upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION)
 upload: $(DEBS)
-	tar -cf - $(DEBS) | ssh repoman@repo.proxmox.com -- upload --product pve,pmg,pbs --dist bullseye --arch amd64
+	tar -cf - $(DEBS) | ssh repoman@repo.proxmox.com -- upload --product pve,pmg,pbs --dist $(UPLOAD_DIST) --arch $(DEB_HOST_ARCH)

debian/changelog

@@ -1,3 +1,23 @@
+zfs-linux (2.1.13-pve1) bookworm; urgency=medium
+
+  * update ZFS to 2.1.13
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 28 Sep 2023 12:22:28 +0200
+
+zfs-linux (2.1.12-pve1) bookworm; urgency=medium
+
+  * update ZFS to 2.1.12
+
+  * zfs trim: avoid exit-failure if last pool isn't nvme-only
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 13 Jun 2023 15:25:16 +0200
+
+zfs-linux (2.1.11-pve2) bookworm; urgency=medium
+
+  * re-build for Debian 12 Bookworm based releases
+
+ -- Proxmox Support Team <support@proxmox.com>  Sat, 20 May 2023 19:32:04 +0200
+
 zfs-linux (2.1.11-pve1) bullseye; urgency=medium
 
   * update ZFS to 2.1.11

debian/control

@@ -203,7 +203,7 @@ Package: zfsutils-linux
 Section: contrib/admin
 Architecture: linux-any
 Depends: python3, ${misc:Depends}, ${python3:Depends}, ${shlibs:Depends}
-Recommends: lsb-base, zfs-zed
+Recommends: zfs-zed
 Suggests: nfs-kernel-server,
           samba-common-bin (>= 3.0.23),
           zfs-initramfs
@@ -285,17 +285,3 @@ Description: Solaris Porting Layer user-space utilities for Linux (dummy)
  to Linux primitives.
  .
  This is a transitional dummy package. It can safely be removed.
-
-Package: zfs-dbg
-Section: contrib/metapackages
-Architecture: all
-Suggests: libnvpair3linux-dbgsym,
-         libpam-zfs-dbgsym,
-         libuutil3linux-dbgsym,
-         libzfs4linux-dbgsym,
-         libzfsbootenv1linux-dbgsym,
-         libzpool5linux-dbgsym,
-         zfs-test-dbgsym,
-         zfsutils-linux-dbgsym,
-         zfs-zed-dbgsym,
-Description: Transitional package. It can be safely removed.

debian/copyright

@@ -132,9 +132,9 @@ Copyright:
  2017, Intel Corporation.
 License: CDDL-1.0
 
-Files: cmd/zstreamdump/*
+Files: cmd/zstream/*
 Copyright:
-  2013, Delphix.
+  2013, 2015 Delphix.
   2012, Martin Matuska <martin@matuska.org>
   2010, Sun Microsystems, Inc.
 License: CDDL-1.0
@@ -201,7 +201,6 @@ Copyright:
  2011-2013, Darik Horn <dajhorn@vanadac.com>
  2018-2019, Mo Zhou <cdluminate@gmail.com>
  2018-2020, Mo Zhou <lumin@debian.org>
- 2015-2021 Proxmox Server Solutions GmbH <support@proxmox.com>
 License: GPL-2+
 
 Files: debian/po/*
@@ -399,12 +398,7 @@ Copyright: 2009, Oracle and/or its affiliates.
   2009, Michael Gebetsroither <michael.geb@gmx.at>
 License: CDDL-1.0
 
-Files: man/man5/zfs-events.5
-       man/man5/zfs-module-parameters.5
-Copyright: 2013, Turbo Fredriksson <turbo@bayour.com>
-License: CDDL-1.0
-
-Files: man/man5/zpool-features.5
+Files: man/man7/zpool-features.7
 Copyright:
   2013, Delphix
   2013, Saso Kiselkov
@@ -436,7 +430,7 @@ Copyright: 2011-2014, Delphix.
   2007, 2009, 2010, Sun Microsystems, Inc.
 License: CDDL-1.0
 
-Files: module/lua
+Files: module/lua/*
 Copyright: 1994-2015 Lua.org, PUC-Rio.
 License: Expat
 
@@ -483,7 +477,7 @@ Copyright: 2013, Saso Kiselkov.
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
 
-Files: module/zcommon/zfs_uio.c
+Files: module/os/linux/zfs/zfs_uio.c
 Copyright: 2007, 2009, 2010, Sun Microsystems, Inc.
   1983-1989, AT&T
   1982, 1986, 1988, The Regents of the University of California
@@ -697,13 +691,14 @@ Copyright: 2011, 2014, Nexenta Systems, Inc.
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
 
-Files: module/zfs/vdev_disk.c
+Files: module/os/linux/zfs/vdev_disk.c
 Copyright: 2012, 2014, Delphix.
   2008-2010, Lawrence Livermore National Security, LLC
 License: CDDL-1.0
 
-Files: module/zfs/zfs_ctldir.c
-Copyright: 2013, Delphix.
+Files: module/os/freebsd/zfs/zfs_ctldir.c
+  module/os/linux/zfs/zfs_ctldir.c
+Copyright: 2013, 2015 Delphix.
   2011, Lawrence Livermore National Security, LLC
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
@@ -726,7 +721,8 @@ Copyright: 2013, Delphix.
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
 
-Files: module/zfs/zfs_vfsops.c
+Files: module/os/freebsd/zfs/zfs_vfsops.c
+  module/os/linux/zfs/zfs_vfsops.c
   module/zfs/zil.c
 Copyright: 2011-2014, Delphix.
   2010, Robert Milkowski
@@ -741,8 +737,9 @@ Copyright: 2015, Chunwei Chen.
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
 
-Files: module/zfs/zfs_znode.c
-Copyright: 2013, Delphix.
+Files: module/os/freebsd/zfs/zfs_znode.c
+  module/os/linux/zfs/zfs_znode.c
+Copyright: 2013, 2015 Delphix.
   2007, Jeremy Teo
   2005, 2010, Oracle and/or its affiliates.
 License: CDDL-1.0
@@ -753,20 +750,20 @@ Copyright: 2013, Saso Kiselkov.
   2009, Sun Microsystems, Inc.
 License: CDDL-1.0
 
-Files: module/zfs/zpl_ctldir.c
-  module/zfs/zpl_super.c
-  module/zfs/zpl_xattr.c
+Files: module/os/linux/zfs/zpl_ctldir.c
+  module/os/linux/zfs/zpl_super.c
+  module/os/linux/zfs/zpl_xattr.c
   module/zfs/zvol.c
 Copyright: 2008-2011, Lawrence Livermore National Security, LLC
 License: CDDL-1.0
 
-Files: module/zfs/zpl_export.c
+Files: module/os/linux/zfs/zpl_export.c
 Copyright: 2012, Cyril Plisko.
   2011, Gunnar Beutner
 License: CDDL-1.0
 
-Files: module/zfs/zpl_file.c
-  module/zfs/zpl_inode.c
+Files: module/os/linux/zfs/zpl_file.c
+  module/os/linux/zfs/zpl_inode.c
 Copyright: 2015, Chunwei Chen.
   2011, Lawrence Livermore National Security, LLC
 License: CDDL-1.0

debian/patches/0005-Enable-zed-emails.patch

@@ -13,7 +13,7 @@ Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  1 file changed, 1 insertion(+), 1 deletion(-)
 
 diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc
-index 227b26c26..240d0dbfa 100644
+index 1dfd43454..0180dd827 100644
 --- a/cmd/zed/zed.d/zed.rc
 +++ b/cmd/zed/zed.d/zed.rc
 @@ -42,7 +42,7 @@ ZED_EMAIL_ADDR="root"

debian/patches/0006-dont-symlink-zed-scripts.patch

@@ -17,10 +17,10 @@ Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
  1 file changed, 1 insertion(+), 1 deletion(-)
 
 diff --git a/cmd/zed/zed.d/Makefile.am b/cmd/zed/zed.d/Makefile.am
-index 2c8173b3e..ad39292e4 100644
+index 1905a9207..6dc06252a 100644
 --- a/cmd/zed/zed.d/Makefile.am
 +++ b/cmd/zed/zed.d/Makefile.am
-@@ -49,7 +49,7 @@ install-data-hook:
+@@ -51,7 +51,7 @@ install-data-hook:
  	for f in $(zedconfdefaults); do \
  	  test -f "$(DESTDIR)$(zedconfdir)/$${f}" -o \
  	       -L "$(DESTDIR)$(zedconfdir)/$${f}" || \

debian/patches/0011-Avoid-save-restoring-AMX-registers-to-avoid-a-SPR-er.patch

@@ -0,0 +1,87 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Rich Ercolani <214141+rincebrain@users.noreply.github.com>
+Date: Sat, 26 Aug 2023 14:25:46 -0400
+Subject: [PATCH] Avoid save/restoring AMX registers to avoid a SPR erratum
+
+Intel SPR erratum SPR4 says that if you trip into a vmexit while
+doing FPU save/restore, your AMX register state might misbehave...
+and by misbehave, I mean save all zeroes incorrectly, leading to
+explosions if you restore it.
+
+Since we're not using AMX for anything, the simple way to avoid
+this is to just not save/restore those when we do anything, since
+we're killing preemption of any sort across our save/restores.
+
+If we ever decide to use AMX, it's not clear that we have any
+way to mitigate this, on Linux...but I am not an expert.
+
+Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
+Signed-off-by: Rich Ercolani <rincebrain@gmail.com>
+Closes #14989
+Closes #15168
+(cherry picked from commit 277f2e587b085d1eb8aa48b4ac0768a9ef5745ab)
+Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
+---
+ include/os/linux/kernel/linux/simd_x86.h | 19 ++++++++++++++-----
+ 1 file changed, 14 insertions(+), 5 deletions(-)
+
+diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h
+index 660f0d42d..455167ac8 100644
+--- a/include/os/linux/kernel/linux/simd_x86.h
++++ b/include/os/linux/kernel/linux/simd_x86.h
+@@ -157,6 +157,15 @@
+ #endif
+ #endif
+ 
++#ifndef XFEATURE_MASK_XTILE
++/*
++ * For kernels where this doesn't exist yet, we still don't want to break
++ * by save/restoring this broken nonsense.
++ * See issue #14989 or Intel errata SPR4 for why
++ */
++#define	XFEATURE_MASK_XTILE	0x60000
++#endif
++
+ #include <linux/mm.h>
+ #include <linux/slab.h>
+ 
+@@ -319,18 +328,18 @@ kfpu_begin(void)
+ 	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+ #if defined(HAVE_XSAVES)
+ 	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+-		kfpu_do_xsave("xsaves", &state->xsave, ~0);
++		kfpu_do_xsave("xsaves", &state->xsave, ~XFEATURE_MASK_XTILE);
+ 		return;
+ 	}
+ #endif
+ #if defined(HAVE_XSAVEOPT)
+ 	if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
+-		kfpu_do_xsave("xsaveopt", &state->xsave, ~0);
++		kfpu_do_xsave("xsaveopt", &state->xsave, ~XFEATURE_MASK_XTILE);
+ 		return;
+ 	}
+ #endif
+ 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+-		kfpu_do_xsave("xsave", &state->xsave, ~0);
++		kfpu_do_xsave("xsave", &state->xsave, ~XFEATURE_MASK_XTILE);
+ 	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+ 		kfpu_save_fxsr(&state->fxsave);
+ 	} else {
+@@ -415,12 +424,12 @@ kfpu_end(void)
+ 	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+ #if defined(HAVE_XSAVES)
+ 	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+-		kfpu_do_xrstor("xrstors", &state->xsave, ~0);
++		kfpu_do_xrstor("xrstors", &state->xsave, ~XFEATURE_MASK_XTILE);
+ 		goto out;
+ 	}
+ #endif
+ 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+-		kfpu_do_xrstor("xrstor", &state->xsave, ~0);
++		kfpu_do_xrstor("xrstor", &state->xsave, ~XFEATURE_MASK_XTILE);
+ 	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+ 		kfpu_restore_fxsr(&state->fxsave);
+ 	} else {
+-- 
+2.39.2
+

debian/patches/series

@@ -8,3 +8,4 @@
 0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
 0009-arcstat-Fix-integer-division-with-python3.patch
 0010-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
+0011-Avoid-save-restoring-AMX-registers-to-avoid-a-SPR-er.patch

debian/rules

@@ -10,7 +10,7 @@ SPHINX_BUILD = $(shell dpkg -L python3-sphinx | grep -m 1 "/sphinx-build$$")
 export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
 %:
-	dh $@ --with autoreconf,python3,sphinxdoc --parallel
+	dh $@ --with autoreconf,python3,sphinxdoc
 
 adapt_meta_file:
 	@# Embed the downstream version in the module.
@@ -91,9 +91,6 @@ override_dh_python3:
 override_dh_makeshlibs:
 	dh_makeshlibs -a -V
 
-override_dh_strip:
-	dh_strip --dbgsym-migration='zfs-dbg (<< 2.0.4~)'
-
 override_dh_auto_clean:
 	find . -name .gitignore -delete
 	rm -rf zfs-$(DEB_VERSION_UPSTREAM)

debian/tree/zfsutils-linux/usr/lib/zfs-linux/trim

@@ -60,7 +60,7 @@ do
 	case "${ret}" in
 		disable);;
 		enable)	trim_if_not_already_trimming "${pool}" ;;
-		-|auto)	pool_is_nvme_only "${pool}" && trim_if_not_already_trimming "${pool}" ;;
+		-|auto)	if pool_is_nvme_only "${pool}"; then trim_if_not_already_trimming "${pool}"; fi ;;
 		*)	cat > /dev/stderr <<EOF
 $0: [WARNING] illegal value "${ret}" for property "${PROPERTY_NAME}" of ZFS dataset "${pool}".
 $0: Acceptable choices for this property are: auto, enable, disable. The default is auto.