diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 200995c5210e..9d455fd543b9 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -41,6 +41,7 @@ struct perf_guest_info_callbacks { #include #include #include +#include #include #include #include @@ -1269,8 +1270,13 @@ static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) * pending stores/changes to the callback pointers are visible before a * non-NULL perf_guest_cbs is visible to readers, and to prevent a * module from unloading callbacks while readers are active. + * + * BPF locked rcu using rcu_read_lock_trace() in + * bpf_prog_test_run_syscall() */ - return rcu_dereference(perf_guest_cbs); + return rcu_dereference_check(perf_guest_cbs, + rcu_read_lock_trace_held() || + rcu_read_lock_held()); } extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c index 58cbe357fb2b..e47402c97ae0 100644 --- a/kernel/events/callchain.c +++ b/kernel/events/callchain.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "internal.h" @@ -32,7 +33,7 @@ static inline size_t perf_callchain_entry__sizeof(void) static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); static atomic_t nr_callchain_events; static DEFINE_MUTEX(callchain_mutex); -static struct callchain_cpus_entries *callchain_cpus_entries; +static struct callchain_cpus_entries __rcu *callchain_cpus_entries; __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, @@ -157,8 +158,13 @@ struct perf_callchain_entry *get_callchain_entry(int *rctx) *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion)); if (*rctx == -1) return NULL; - - entries = rcu_dereference(callchain_cpus_entries); + /* + * BPF locked rcu using rcu_read_lock_trace() in + * bpf_prog_test_run_syscall() + */ + entries = rcu_dereference_check(callchain_cpus_entries, + rcu_read_lock_trace_held() || + rcu_read_lock_held()); if (!entries) { put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx); return NULL;