lkml.org 
[lkml]   [2019]   [Apr]   [10]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Date
From
Subject[PATCH RT] bpf: Guard bpf_prog_active with a locallock
The bpf_prog_active counter is used to avoid recursion on the same CPU.
On RT we can't keep it with the preempt-disable part because the syscall
may need to acquire locks or allocate memory.

Use a locallock() to avoid recursion on the same CPU.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/bpf.h | 2 ++
kernel/bpf/hashtab.c | 4 ++--
kernel/bpf/syscall.c | 13 +++++++------
kernel/events/core.c | 3 ++-
kernel/trace/bpf_trace.c | 5 ++---
5 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index e734f163bd0b9..667f45de65be8 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -16,6 +16,7 @@
#include <linux/rbtree_latch.h>
#include <linux/numa.h>
#include <linux/wait.h>
+#include <linux/locallock.h>

struct bpf_verifier_env;
struct perf_event;
@@ -467,6 +468,7 @@ _out: \

#ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active);
+DECLARE_LOCAL_IRQ_LOCK(bpf_prog_active_lock);

extern const struct file_operations bpf_map_fops;
extern const struct file_operations bpf_prog_fops;
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index b4f903a5ef36e..15120d2d8b659 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -668,11 +668,11 @@ static void htab_elem_free_rcu(struct rcu_head *head)
* we're calling kfree, otherwise deadlock is possible if kprobes
* are placed somewhere inside of slub
*/
- preempt_disable();
+ local_lock(bpf_prog_active_lock);
__this_cpu_inc(bpf_prog_active);
htab_elem_free(htab, l);
__this_cpu_dec(bpf_prog_active);
- preempt_enable();
+ local_unlock(bpf_prog_active_lock);
}

static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 84470d1480aa4..73f2edbe3b28c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -42,6 +42,7 @@
#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)

DEFINE_PER_CPU(int, bpf_prog_active);
+DEFINE_LOCAL_IRQ_LOCK(bpf_prog_active_lock);
static DEFINE_IDR(prog_idr);
static DEFINE_SPINLOCK(prog_idr_lock);
static DEFINE_IDR(map_idr);
@@ -716,7 +717,7 @@ static int map_lookup_elem(union bpf_attr *attr)
goto done;
}

- preempt_disable();
+ local_lock(bpf_prog_active_lock);
this_cpu_inc(bpf_prog_active);
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
@@ -750,7 +751,7 @@ static int map_lookup_elem(union bpf_attr *attr)
rcu_read_unlock();
}
this_cpu_dec(bpf_prog_active);
- preempt_enable();
+ local_unlock(bpf_prog_active_lock);

done:
if (err)
@@ -845,7 +846,7 @@ static int map_update_elem(union bpf_attr *attr)
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
* inside bpf map update or delete otherwise deadlocks are possible
*/
- preempt_disable();
+ local_lock(bpf_prog_active_lock);
__this_cpu_inc(bpf_prog_active);
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
@@ -878,7 +879,7 @@ static int map_update_elem(union bpf_attr *attr)
rcu_read_unlock();
}
__this_cpu_dec(bpf_prog_active);
- preempt_enable();
+ local_unlock(bpf_prog_active_lock);
maybe_wait_bpf_programs(map);
out:
free_value:
@@ -925,13 +926,13 @@ static int map_delete_elem(union bpf_attr *attr)
goto out;
}

- preempt_disable();
+ local_lock(bpf_prog_active_lock);
__this_cpu_inc(bpf_prog_active);
rcu_read_lock();
err = map->ops->map_delete_elem(map, key);
rcu_read_unlock();
__this_cpu_dec(bpf_prog_active);
- preempt_enable();
+ local_unlock(bpf_prog_active_lock);
maybe_wait_bpf_programs(map);
out:
kfree(key);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index b3155a155a645..6facb80af7c0e 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -8546,7 +8546,7 @@ static void bpf_overflow_handler(struct perf_event *event,
int ret = 0;

ctx.regs = perf_arch_bpf_user_pt_regs(regs);
- preempt_disable();
+ local_lock(bpf_prog_active_lock);
if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
goto out;
rcu_read_lock();
@@ -8555,6 +8555,7 @@ static void bpf_overflow_handler(struct perf_event *event,
out:
__this_cpu_dec(bpf_prog_active);
preempt_enable();
+ local_unlock(bpf_prog_active_lock);
if (!ret)
return;

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index f1a86a0d881dd..bb92cf31481b4 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -78,8 +78,7 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
if (in_nmi()) /* not supported yet */
return 1;

- preempt_disable();
-
+ local_lock(bpf_prog_active_lock);
if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
/*
* since some bpf program is already running on this cpu,
@@ -110,7 +109,7 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)

out:
__this_cpu_dec(bpf_prog_active);
- preempt_enable();
+ local_unlock(bpf_prog_active_lock);

return ret;
}
--
2.20.1
\
 
 \ /
  Last update: 2019-04-10 16:43    [W:0.872 / U:0.008 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site