2018-01-15 14:26:15 +03:00
|
|
|
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
2018-01-06 17:13:39 +03:00
|
|
|
From: Peter Zijlstra <peterz@infradead.org>
|
|
|
|
|
Date: Mon, 4 Dec 2017 15:08:00 +0100
|
2018-01-15 14:26:15 +03:00
|
|
|
Subject: [PATCH] x86/mm: Optimize RESTORE_CR3
|
2018-01-06 17:13:39 +03:00
|
|
|
MIME-Version: 1.0
|
|
|
|
|
Content-Type: text/plain; charset=UTF-8
|
|
|
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
|
|
|
|
|
|
CVE-2017-5754
|
|
|
|
|
|
|
|
|
|
Most NMI/paranoid exceptions will not in fact change pagetables and would
|
|
|
|
|
thus not require TLB flushing, however RESTORE_CR3 uses flushing CR3
|
|
|
|
|
writes.
|
|
|
|
|
|
|
|
|
|
Restores to kernel PCIDs can be NOFLUSH, because we explicitly flush the
|
|
|
|
|
kernel mappings and now that we track which user PCIDs need flushing we can
|
|
|
|
|
avoid those too when possible.
|
|
|
|
|
|
|
|
|
|
This does mean RESTORE_CR3 needs an additional scratch_reg, luckily both
|
|
|
|
|
sites have plenty available.
|
|
|
|
|
|
|
|
|
|
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
|
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
|
Cc: Andy Lutomirski <luto@kernel.org>
|
|
|
|
|
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
|
|
|
|
|
Cc: Borislav Petkov <bp@alien8.de>
|
|
|
|
|
Cc: Brian Gerst <brgerst@gmail.com>
|
|
|
|
|
Cc: Dave Hansen <dave.hansen@linux.intel.com>
|
|
|
|
|
Cc: David Laight <David.Laight@aculab.com>
|
|
|
|
|
Cc: Denys Vlasenko <dvlasenk@redhat.com>
|
|
|
|
|
Cc: Eduardo Valentin <eduval@amazon.com>
|
|
|
|
|
Cc: Greg KH <gregkh@linuxfoundation.org>
|
|
|
|
|
Cc: H. Peter Anvin <hpa@zytor.com>
|
|
|
|
|
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
|
|
|
|
|
Cc: Juergen Gross <jgross@suse.com>
|
|
|
|
|
Cc: Linus Torvalds <torvalds@linux-foundation.org>
|
|
|
|
|
Cc: Peter Zijlstra <peterz@infradead.org>
|
|
|
|
|
Cc: Will Deacon <will.deacon@arm.com>
|
|
|
|
|
Cc: aliguori@amazon.com
|
|
|
|
|
Cc: daniel.gruss@iaik.tugraz.at
|
|
|
|
|
Cc: hughd@google.com
|
|
|
|
|
Cc: keescook@google.com
|
|
|
|
|
Signed-off-by: Ingo Molnar <mingo@kernel.org>
|
|
|
|
|
(cherry picked from commit 21e94459110252d41b45c0c8ba50fd72a664d50c)
|
|
|
|
|
Signed-off-by: Andy Whitcroft <apw@canonical.com>
|
|
|
|
|
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
|
|
|
|
|
(cherry picked from commit 6ebe6e2896841282357d43c09394b0ca47c41e4a)
|
|
|
|
|
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
|
|
|
|
|
---
|
|
|
|
|
arch/x86/entry/calling.h | 30 ++++++++++++++++++++++++++++--
|
|
|
|
|
arch/x86/entry/entry_64.S | 4 ++--
|
|
|
|
|
2 files changed, 30 insertions(+), 4 deletions(-)
|
|
|
|
|
|
|
|
|
|
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
|
|
|
|
|
index ce5fb309926d..015e0a84bb99 100644
|
|
|
|
|
--- a/arch/x86/entry/calling.h
|
|
|
|
|
+++ b/arch/x86/entry/calling.h
|
|
|
|
|
@@ -280,8 +280,34 @@ For 32-bit we have the following conventions - kernel is built with
|
|
|
|
|
.Ldone_\@:
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
-.macro RESTORE_CR3 save_reg:req
|
|
|
|
|
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
|
|
|
|
|
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
|
|
|
|
|
+
|
|
|
|
|
+ ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
|
|
|
|
|
+
|
|
|
|
|
+ /*
|
|
|
|
|
+ * KERNEL pages can always resume with NOFLUSH as we do
|
|
|
|
|
+ * explicit flushes.
|
|
|
|
|
+ */
|
|
|
|
|
+ bt $X86_CR3_PTI_SWITCH_BIT, \save_reg
|
|
|
|
|
+ jnc .Lnoflush_\@
|
|
|
|
|
+
|
|
|
|
|
+ /*
|
|
|
|
|
+ * Check if there's a pending flush for the user ASID we're
|
|
|
|
|
+ * about to set.
|
|
|
|
|
+ */
|
|
|
|
|
+ movq \save_reg, \scratch_reg
|
|
|
|
|
+ andq $(0x7FF), \scratch_reg
|
|
|
|
|
+ bt \scratch_reg, THIS_CPU_user_pcid_flush_mask
|
|
|
|
|
+ jnc .Lnoflush_\@
|
|
|
|
|
+
|
|
|
|
|
+ btr \scratch_reg, THIS_CPU_user_pcid_flush_mask
|
|
|
|
|
+ jmp .Lwrcr3_\@
|
|
|
|
|
+
|
|
|
|
|
+.Lnoflush_\@:
|
|
|
|
|
+ SET_NOFLUSH_BIT \save_reg
|
|
|
|
|
+
|
|
|
|
|
+.Lwrcr3_\@:
|
|
|
|
|
/*
|
|
|
|
|
* The CR3 write could be avoided when not changing its value,
|
|
|
|
|
* but would require a CR3 read *and* a scratch register.
|
|
|
|
|
@@ -300,7 +326,7 @@ For 32-bit we have the following conventions - kernel is built with
|
|
|
|
|
.endm
|
|
|
|
|
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
|
|
|
|
|
.endm
|
|
|
|
|
-.macro RESTORE_CR3 save_reg:req
|
|
|
|
|
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
|
|
|
|
|
index fb43f14ed299..b48f2c78a9bf 100644
|
|
|
|
|
--- a/arch/x86/entry/entry_64.S
|
|
|
|
|
+++ b/arch/x86/entry/entry_64.S
|
|
|
|
|
@@ -1300,7 +1300,7 @@ ENTRY(paranoid_exit)
|
|
|
|
|
testl %ebx, %ebx /* swapgs needed? */
|
|
|
|
|
jnz .Lparanoid_exit_no_swapgs
|
|
|
|
|
TRACE_IRQS_IRETQ
|
|
|
|
|
- RESTORE_CR3 save_reg=%r14
|
|
|
|
|
+ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14
|
|
|
|
|
SWAPGS_UNSAFE_STACK
|
|
|
|
|
jmp .Lparanoid_exit_restore
|
|
|
|
|
.Lparanoid_exit_no_swapgs:
|
|
|
|
|
@@ -1742,7 +1742,7 @@ end_repeat_nmi:
|
|
|
|
|
movq $-1, %rsi
|
|
|
|
|
call do_nmi
|
|
|
|
|
|
|
|
|
|
- RESTORE_CR3 save_reg=%r14
|
|
|
|
|
+ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
|
|
|
|
|
|
|
|
|
|
testl %ebx, %ebx /* swapgs needed? */
|
|
|
|
|
jnz nmi_restore
|
|
|
|
|
--
|
|
|
|
|
2.14.2
|
|
|
|
|
|
