282 lines
8.5 KiB
Diff
282 lines
8.5 KiB
Diff
From 68338a3b7267b4fc346630b2d82a3599b5fbf54e Mon Sep 17 00:00:00 2001
|
|
From: Hugh Dickins <hughd@google.com>
|
|
Date: Mon, 4 Dec 2017 15:07:50 +0100
|
|
Subject: [PATCH 203/231] x86/events/intel/ds: Map debug buffers in
|
|
cpu_entry_area
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
CVE-2017-5754
|
|
|
|
The BTS and PEBS buffers both have their virtual addresses programmed into
|
|
the hardware. This means that any access to them is performed via the page
|
|
tables. The times that the hardware accesses these are entirely dependent
|
|
on how the performance monitoring hardware events are set up. In other
|
|
words, there is no way for the kernel to tell when the hardware might
|
|
access these buffers.
|
|
|
|
To avoid perf crashes, place 'debug_store' allocate pages and map them into
|
|
the cpu_entry_area.
|
|
|
|
The PEBS fixup buffer does not need this treatment.
|
|
|
|
[ tglx: Got rid of the kaiser_add_mapping() complication ]
|
|
|
|
Signed-off-by: Hugh Dickins <hughd@google.com>
|
|
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
Cc: Andy Lutomirski <luto@kernel.org>
|
|
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
|
|
Cc: Borislav Petkov <bp@alien8.de>
|
|
Cc: Brian Gerst <brgerst@gmail.com>
|
|
Cc: David Laight <David.Laight@aculab.com>
|
|
Cc: Denys Vlasenko <dvlasenk@redhat.com>
|
|
Cc: Eduardo Valentin <eduval@amazon.com>
|
|
Cc: Greg KH <gregkh@linuxfoundation.org>
|
|
Cc: H. Peter Anvin <hpa@zytor.com>
|
|
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
|
|
Cc: Juergen Gross <jgross@suse.com>
|
|
Cc: Linus Torvalds <torvalds@linux-foundation.org>
|
|
Cc: Peter Zijlstra <peterz@infradead.org>
|
|
Cc: Will Deacon <will.deacon@arm.com>
|
|
Cc: aliguori@amazon.com
|
|
Cc: daniel.gruss@iaik.tugraz.at
|
|
Cc: keescook@google.com
|
|
Signed-off-by: Ingo Molnar <mingo@kernel.org>
|
|
(cherry picked from commit c1961a4631daef4aeabee8e368b1b13e8f173c91)
|
|
Signed-off-by: Andy Whitcroft <apw@canonical.com>
|
|
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
|
|
(cherry picked from commit 569dedbb62e16e3268f006dcf745b8d27690ef91)
|
|
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
|
|
---
|
|
arch/x86/events/perf_event.h | 2 +
|
|
arch/x86/events/intel/ds.c | 125 +++++++++++++++++++++++++++----------------
|
|
2 files changed, 82 insertions(+), 45 deletions(-)
|
|
|
|
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
|
|
index 308bc14f58af..eb0876475f18 100644
|
|
--- a/arch/x86/events/perf_event.h
|
|
+++ b/arch/x86/events/perf_event.h
|
|
@@ -199,6 +199,8 @@ struct cpu_hw_events {
|
|
* Intel DebugStore bits
|
|
*/
|
|
struct debug_store *ds;
|
|
+ void *ds_pebs_vaddr;
|
|
+ void *ds_bts_vaddr;
|
|
u64 pebs_enabled;
|
|
int n_pebs;
|
|
int n_large_pebs;
|
|
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
|
|
index 21a4ed789ec0..85df1f12c49e 100644
|
|
--- a/arch/x86/events/intel/ds.c
|
|
+++ b/arch/x86/events/intel/ds.c
|
|
@@ -2,6 +2,7 @@
|
|
#include <linux/types.h>
|
|
#include <linux/slab.h>
|
|
|
|
+#include <asm/cpu_entry_area.h>
|
|
#include <asm/perf_event.h>
|
|
#include <asm/insn.h>
|
|
|
|
@@ -279,17 +280,52 @@ void fini_debug_store_on_cpu(int cpu)
|
|
|
|
static DEFINE_PER_CPU(void *, insn_buffer);
|
|
|
|
-static int alloc_pebs_buffer(int cpu)
|
|
+static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)
|
|
{
|
|
- struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
|
+ phys_addr_t pa;
|
|
+ size_t msz = 0;
|
|
+
|
|
+ pa = virt_to_phys(addr);
|
|
+ for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)
|
|
+ cea_set_pte(cea, pa, prot);
|
|
+}
|
|
+
|
|
+static void ds_clear_cea(void *cea, size_t size)
|
|
+{
|
|
+ size_t msz = 0;
|
|
+
|
|
+ for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)
|
|
+ cea_set_pte(cea, 0, PAGE_NONE);
|
|
+}
|
|
+
|
|
+static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
|
|
+{
|
|
+ unsigned int order = get_order(size);
|
|
int node = cpu_to_node(cpu);
|
|
- int max;
|
|
- void *buffer, *ibuffer;
|
|
+ struct page *page;
|
|
+
|
|
+ page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
|
|
+ return page ? page_address(page) : NULL;
|
|
+}
|
|
+
|
|
+static void dsfree_pages(const void *buffer, size_t size)
|
|
+{
|
|
+ if (buffer)
|
|
+ free_pages((unsigned long)buffer, get_order(size));
|
|
+}
|
|
+
|
|
+static int alloc_pebs_buffer(int cpu)
|
|
+{
|
|
+ struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
|
|
+ struct debug_store *ds = hwev->ds;
|
|
+ size_t bsiz = x86_pmu.pebs_buffer_size;
|
|
+ int max, node = cpu_to_node(cpu);
|
|
+ void *buffer, *ibuffer, *cea;
|
|
|
|
if (!x86_pmu.pebs)
|
|
return 0;
|
|
|
|
- buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node);
|
|
+ buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
|
|
if (unlikely(!buffer))
|
|
return -ENOMEM;
|
|
|
|
@@ -300,25 +336,27 @@ static int alloc_pebs_buffer(int cpu)
|
|
if (x86_pmu.intel_cap.pebs_format < 2) {
|
|
ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
|
|
if (!ibuffer) {
|
|
- kfree(buffer);
|
|
+ dsfree_pages(buffer, bsiz);
|
|
return -ENOMEM;
|
|
}
|
|
per_cpu(insn_buffer, cpu) = ibuffer;
|
|
}
|
|
-
|
|
- max = x86_pmu.pebs_buffer_size / x86_pmu.pebs_record_size;
|
|
-
|
|
- ds->pebs_buffer_base = (u64)(unsigned long)buffer;
|
|
+ hwev->ds_pebs_vaddr = buffer;
|
|
+ /* Update the cpu entry area mapping */
|
|
+ cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
|
|
+ ds->pebs_buffer_base = (unsigned long) cea;
|
|
+ ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL);
|
|
ds->pebs_index = ds->pebs_buffer_base;
|
|
- ds->pebs_absolute_maximum = ds->pebs_buffer_base +
|
|
- max * x86_pmu.pebs_record_size;
|
|
-
|
|
+ max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size);
|
|
+ ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;
|
|
return 0;
|
|
}
|
|
|
|
static void release_pebs_buffer(int cpu)
|
|
{
|
|
- struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
|
+ struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
|
|
+ struct debug_store *ds = hwev->ds;
|
|
+ void *cea;
|
|
|
|
if (!ds || !x86_pmu.pebs)
|
|
return;
|
|
@@ -326,73 +364,70 @@ static void release_pebs_buffer(int cpu)
|
|
kfree(per_cpu(insn_buffer, cpu));
|
|
per_cpu(insn_buffer, cpu) = NULL;
|
|
|
|
- kfree((void *)(unsigned long)ds->pebs_buffer_base);
|
|
+ /* Clear the fixmap */
|
|
+ cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
|
|
+ ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
|
|
ds->pebs_buffer_base = 0;
|
|
+ dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);
|
|
+ hwev->ds_pebs_vaddr = NULL;
|
|
}
|
|
|
|
static int alloc_bts_buffer(int cpu)
|
|
{
|
|
- struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
|
- int node = cpu_to_node(cpu);
|
|
- int max, thresh;
|
|
- void *buffer;
|
|
+ struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
|
|
+ struct debug_store *ds = hwev->ds;
|
|
+ void *buffer, *cea;
|
|
+ int max;
|
|
|
|
if (!x86_pmu.bts)
|
|
return 0;
|
|
|
|
- buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node);
|
|
+ buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu);
|
|
if (unlikely(!buffer)) {
|
|
WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
|
|
return -ENOMEM;
|
|
}
|
|
-
|
|
- max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
|
|
- thresh = max / 16;
|
|
-
|
|
- ds->bts_buffer_base = (u64)(unsigned long)buffer;
|
|
+ hwev->ds_bts_vaddr = buffer;
|
|
+ /* Update the fixmap */
|
|
+ cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
|
|
+ ds->bts_buffer_base = (unsigned long) cea;
|
|
+ ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL);
|
|
ds->bts_index = ds->bts_buffer_base;
|
|
- ds->bts_absolute_maximum = ds->bts_buffer_base +
|
|
- max * BTS_RECORD_SIZE;
|
|
- ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
|
|
- thresh * BTS_RECORD_SIZE;
|
|
-
|
|
+ max = BTS_RECORD_SIZE * (BTS_BUFFER_SIZE / BTS_RECORD_SIZE);
|
|
+ ds->bts_absolute_maximum = ds->bts_buffer_base + max;
|
|
+ ds->bts_interrupt_threshold = ds->bts_absolute_maximum - (max / 16);
|
|
return 0;
|
|
}
|
|
|
|
static void release_bts_buffer(int cpu)
|
|
{
|
|
- struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
|
+ struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
|
|
+ struct debug_store *ds = hwev->ds;
|
|
+ void *cea;
|
|
|
|
if (!ds || !x86_pmu.bts)
|
|
return;
|
|
|
|
- kfree((void *)(unsigned long)ds->bts_buffer_base);
|
|
+ /* Clear the fixmap */
|
|
+ cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
|
|
+ ds_clear_cea(cea, BTS_BUFFER_SIZE);
|
|
ds->bts_buffer_base = 0;
|
|
+ dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);
|
|
+ hwev->ds_bts_vaddr = NULL;
|
|
}
|
|
|
|
static int alloc_ds_buffer(int cpu)
|
|
{
|
|
- int node = cpu_to_node(cpu);
|
|
- struct debug_store *ds;
|
|
-
|
|
- ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node);
|
|
- if (unlikely(!ds))
|
|
- return -ENOMEM;
|
|
+ struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;
|
|
|
|
+ memset(ds, 0, sizeof(*ds));
|
|
per_cpu(cpu_hw_events, cpu).ds = ds;
|
|
-
|
|
return 0;
|
|
}
|
|
|
|
static void release_ds_buffer(int cpu)
|
|
{
|
|
- struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
|
-
|
|
- if (!ds)
|
|
- return;
|
|
-
|
|
per_cpu(cpu_hw_events, cpu).ds = NULL;
|
|
- kfree(ds);
|
|
}
|
|
|
|
void release_ds_buffers(void)
|
|
--
|
|
2.14.2
|
|
|