Following up on the earlier discussion,
Post by Arnd BergmannPost by Paul E. McKenneyArnd, would it be reasonable to extend your RCU-sparse changes to have
four different pointer namespaces, one for each flavor of RCU? (RCU,
RCU-bh, RCU-sched, and SRCU)? Always a fan of making the computer do
the auditing where reasonable. ;-)
Yes, I guess that would be possible. I'd still leave out the rculist
from any annotations for now, as this would get even more complex then.
One consequence will be the need for new rcu_assign_pointer{,_bh,_sched}
macros that check the address space of the first argument, otherwise
you'd be able to stick anything in there, including non-__rcu pointers.
I've tested this out now, see the patch below. I needed to add a number
of interfaces, but it still seems ok. Doing it for all the rculist
functions most likely would be less so.
This is currently the head of my rcu-annotate branch of playground.git.
Paul, before I split it up and merge this with the per-subsystem patches,
can you tell me if this is what you had in mind?
Post by Arnd BergmannPost by Paul E. McKenneyThis could potentially catch the mismatched call_rcu()s, at least if the
rcu_head could be labeled.
...
#define rcu_exchange_call(ptr, new, member, func) \
({ \
typeof(new) old = rcu_exchange((ptr),(new)); \
if (old) \
call_rcu(&(old)->member, (func)); \
old; \
})
Unfortunately, this did not work out at all. Almost every user follows
a slightly different pattern for call_rcu, so I did not find a way
to match the call_rcu calls with the pointers. In particular, the functions
calling call_rcu() sometimes no longer have access to the 'old' data,
e.g. in case of synchronize_rcu.
My current take is that static annotations won't help us here.
Arnd
---
rcu: split up __rcu annotations
This adds separate name spaces for the four distinct types of RCU
that we use in the kernel, namely __rcu, __rcu_bh, __rcu_sched and
__srcu.
Signed-off-by: Arnd Bergmann <***@arndb.de>
---
arch/x86/kvm/mmu.c | 6 ++--
arch/x86/kvm/vmx.c | 2 +-
drivers/net/macvlan.c | 8 ++--
include/linux/compiler.h | 6 ++++
include/linux/kvm_host.h | 4 +-
include/linux/netdevice.h | 2 +-
include/linux/rcupdate.h | 68 +++++++++++++++++++++++++++++++++----------
include/linux/srcu.h | 5 ++-
include/net/dst.h | 4 +-
include/net/llc.h | 3 +-
include/net/sock.h | 2 +-
include/trace/events/kvm.h | 4 +-
kernel/cgroup.c | 10 +++---
kernel/perf_event.c | 8 ++--
kernel/sched.c | 6 ++--
kernel/sched_fair.c | 2 +-
lib/radix-tree.c | 8 ++--
net/core/filter.c | 4 +-
net/core/sock.c | 6 ++--
net/decnet/dn_route.c | 2 +-
net/ipv4/route.c | 60 +++++++++++++++++++-------------------
net/ipv4/tcp.c | 4 +-
net/llc/llc_core.c | 6 ++--
net/llc/llc_input.c | 2 +-
virt/kvm/iommu.c | 4 +-
virt/kvm/kvm_main.c | 56 +++++++++++++++++++-----------------
26 files changed, 171 insertions(+), 121 deletions(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 741373e..45877ca 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -793,7 +793,7 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
int retval = 0;
struct kvm_memslots *slots;
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < slots->nmemslots; i++) {
struct kvm_memory_slot *memslot = &slots->memslots[i];
@@ -3007,7 +3007,7 @@ unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
unsigned int nr_pages = 0;
struct kvm_memslots *slots;
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < slots->nmemslots; i++)
nr_pages += slots->memslots[i].npages;
@@ -3282,7 +3282,7 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
int i, j, k, idx;
idx = srcu_read_lock(&kvm->srcu);
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
struct kvm_memory_slot *m = &slots->memslots[i];
struct kvm_rmap_desc *d;
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 0aec1f3..d0c82ed 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1513,7 +1513,7 @@ static gva_t rmode_tss_base(struct kvm *kvm)
struct kvm_memslots *slots;
gfn_t base_gfn;
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
base_gfn = slots->memslots[0].base_gfn +
slots->memslots[0].npages - 3;
return base_gfn << PAGE_SHIFT;
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 95e1bcc..b958d5a 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -531,15 +531,15 @@ static int macvlan_port_create(struct net_device *dev)
INIT_LIST_HEAD(&port->vlans);
for (i = 0; i < MACVLAN_HASH_SIZE; i++)
INIT_HLIST_HEAD(&port->vlan_hash[i]);
- rcu_assign_pointer(dev->macvlan_port, port);
+ rcu_assign_pointer_bh(dev->macvlan_port, port);
return 0;
}
static void macvlan_port_destroy(struct net_device *dev)
{
- struct macvlan_port *port = rcu_dereference_const(dev->macvlan_port);
+ struct macvlan_port *port = rcu_dereference_bh_const(dev->macvlan_port);
- rcu_assign_pointer(dev->macvlan_port, NULL);
+ rcu_assign_pointer_bh(dev->macvlan_port, NULL);
synchronize_rcu();
kfree(port);
}
@@ -624,7 +624,7 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
if (err < 0)
return err;
}
- port = rcu_dereference(lowerdev->macvlan_port);
+ port = rcu_dereference_bh(lowerdev->macvlan_port);
vlan->lowerdev = lowerdev;
vlan->dev = dev;
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 0ab21c2..d5756d4 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -17,6 +17,9 @@
# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
# define __percpu __attribute__((noderef, address_space(3)))
# define __rcu __attribute__((noderef, address_space(4)))
+# define __rcu_bh __attribute__((noderef, address_space(5)))
+# define __rcu_sched __attribute__((noderef, address_space(6)))
+# define __srcu __attribute__((noderef, address_space(7)))
extern void __chk_user_ptr(const volatile void __user *);
extern void __chk_io_ptr(const volatile void __iomem *);
#else
@@ -36,6 +39,9 @@ extern void __chk_io_ptr(const volatile void __iomem *);
# define __cond_lock(x,c) (c)
# define __percpu
# define __rcu
+# define __rcu_bh
+# define __rcu_sched
+# define __srcu
#endif
#ifdef __KERNEL__
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9eb0f9c..bad1787 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -164,7 +164,7 @@ struct kvm {
raw_spinlock_t requests_lock;
struct mutex slots_lock;
struct mm_struct *mm; /* userspace tied to this vm */
- struct kvm_memslots __rcu *memslots;
+ struct kvm_memslots __srcu *memslots;
struct srcu_struct srcu;
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
u32 bsp_vcpu_id;
@@ -174,7 +174,7 @@ struct kvm {
atomic_t online_vcpus;
struct list_head vm_list;
struct mutex lock;
- struct kvm_io_bus __rcu *buses[KVM_NR_BUSES];
+ struct kvm_io_bus __srcu *buses[KVM_NR_BUSES];
#ifdef CONFIG_HAVE_KVM_EVENTFD
struct {
spinlock_t lock;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index fd7e8de..1b72188 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -949,7 +949,7 @@ struct net_device {
/* bridge stuff */
void __rcu *br_port;
/* macvlan */
- struct macvlan_port __rcu *macvlan_port;
+ struct macvlan_port __rcu_bh *macvlan_port;
/* GARP */
struct garp_port __rcu *garp_port;
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 03702cc..b4c6f39 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -183,19 +183,33 @@ static inline int rcu_read_lock_sched_held(void)
* read-side critical section. It is also possible to check for
* locks being held, for example, by using lockdep_is_held().
*/
-#define rcu_dereference_check(p, c) \
+#define __rcu_dereference_check(p, c, space) \
({ \
if (debug_locks && !(c)) \
lockdep_rcu_dereference(__FILE__, __LINE__); \
- rcu_dereference_raw(p); \
+ __rcu_dereference_raw(p, space); \
})
+
#else /* #ifdef CONFIG_PROVE_RCU */
-#define rcu_dereference_check(p, c) rcu_dereference_raw(p)
+#define __rcu_dereference_check(p, c, space) \
+ __rcu_dereference_raw(p, space)
#endif /* #else #ifdef CONFIG_PROVE_RCU */
+#define rcu_dereference_check(p, c) \
+ __rcu_dereference_check(p, c, __rcu)
+
+#define rcu_dereference_bh_check(p, c) \
+ __rcu_dereference_check(p, rcu_read_lock_bh_held() || (c), __rcu_bh)
+
+#define rcu_dereference_sched_check(p, c) \
+ __rcu_dereference_check(p, rcu_read_lock_sched_held() || (c), __rcu_sched)
+
+#define srcu_dereference_check(p, c) \
+ __rcu_dereference_check(p, srcu_read_lock_held() || (c), __srcu)
+
/**
* rcu_read_lock - mark the beginning of an RCU read-side critical section.
*
@@ -341,13 +355,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* exactly which pointers are protected by RCU and checks that
* the pointer is annotated as __rcu.
*/
-#define rcu_dereference_raw(p) ({ \
+#define __rcu_dereference_raw(p, space) ({ \
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
- (void) (((typeof (*p) __rcu *)p) == p); \
+ (void) (((typeof (*p) space *)p) == p); \
smp_read_barrier_depends(); \
((typeof(*p) __force __kernel *)(_________p1)); \
})
+#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __rcu)
+
/**
* rcu_dereference_const - fetch an __rcu pointer outside of a
* read-side critical section.
@@ -360,18 +376,22 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* or in an RCU call.
*/
-#define rcu_dereference_const(p) ({ \
- (void) (((typeof (*p) __rcu *)p) == p); \
+#define __rcu_dereference_const(p, space) ({ \
+ (void) (((typeof (*p) space *)p) == p); \
((typeof(*p) __force __kernel *)(p)); \
})
+#define rcu_dereference_const(p) __rcu_dereference_const(p, __rcu)
+#define rcu_dereference_bh_const(p) __rcu_dereference_const(p, __rcu_bh)
+#define rcu_dereference_sched_const(p) __rcu_dereference_const(p, __rcu_sched)
+
/**
* rcu_dereference - fetch an RCU-protected pointer, checking for RCU
*
* Makes rcu_dereference_check() do the dirty work.
*/
#define rcu_dereference(p) \
- rcu_dereference_check(p, rcu_read_lock_held())
+ __rcu_dereference_check(p, rcu_read_lock_held(), __rcu)
/**
* rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh
@@ -379,7 +399,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* Makes rcu_dereference_check() do the dirty work.
*/
#define rcu_dereference_bh(p) \
- rcu_dereference_check(p, rcu_read_lock_bh_held())
+ __rcu_dereference_check(p, rcu_read_lock_bh_held(), __rcu_bh)
/**
* rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched
@@ -387,7 +407,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* Makes rcu_dereference_check() do the dirty work.
*/
#define rcu_dereference_sched(p) \
- rcu_dereference_check(p, rcu_read_lock_sched_held())
+ __rcu_dereference_check(p, rcu_read_lock_sched_held(), __rcu_sched)
/**
* rcu_assign_pointer - assign (publicize) a pointer to a newly
@@ -402,12 +422,12 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* code.
*/
-#define rcu_assign_pointer(p, v) \
+#define __rcu_assign_pointer(p, v, space) \
({ \
if (!__builtin_constant_p(v) || \
((v) != NULL)) \
smp_wmb(); \
- (p) = (typeof(*v) __force __rcu *)(v); \
+ (p) = (typeof(*v) __force space *)(v); \
})
/**
@@ -415,10 +435,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* without barriers.
* Using this is almost always a bug.
*/
-#define __rcu_assign_pointer(p, v) \
- ({ \
- (p) = (typeof(*v) __force __rcu *)(v); \
- })
+#define rcu_assign_pointer(p, v) \
+ __rcu_assign_pointer(p, v, __rcu)
+
+#define rcu_assign_pointer_bh(p, v) \
+ __rcu_assign_pointer(p, v, __rcu_bh)
+
+#define rcu_assign_pointer_sched(p, v) \
+ __rcu_assign_pointer(p, v, __rcu_sched)
+
+#define srcu_assign_pointer(p, v) \
+ __rcu_assign_pointer(p, v, __srcu)
/**
* RCU_INIT_POINTER - initialize an RCU protected member
@@ -427,6 +454,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
#define RCU_INIT_POINTER(p, v) \
p = (typeof(*v) __force __rcu *)(v)
+#define RCU_INIT_POINTER_BH(p, v) \
+ p = (typeof(*v) __force __rcu_bh *)(v)
+
+#define RCU_INIT_POINTER_SCHED(p, v) \
+ p = (typeof(*v) __force __rcu_sched *)(v)
+
+#define SRCU_INIT_POINTER(p, v) \
+ p = (typeof(*v) __force __srcu *)(v)
+
/* Infrastructure to implement the synchronize_() primitives. */
struct rcu_synchronize {
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 4d5ecb2..feaf661 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -111,7 +111,10 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp)
* Makes rcu_dereference_check() do the dirty work.
*/
#define srcu_dereference(p, sp) \
- rcu_dereference_check(p, srcu_read_lock_held(sp))
+ __rcu_dereference_check(p, srcu_read_lock_held(sp), __srcu)
+
+#define srcu_dereference_const(p) \
+ __rcu_dereference_const(p, __srcu)
/**
* srcu_read_lock - register a new reader for an SRCU-protected structure.
diff --git a/include/net/dst.h b/include/net/dst.h
index 5f839aa..bbeaba2 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -94,9 +94,9 @@ struct dst_entry {
unsigned long lastuse;
union {
struct dst_entry *next;
- struct rtable __rcu *rt_next;
+ struct rtable __rcu_bh *rt_next;
struct rt6_info *rt6_next;
- struct dn_route *dn_next;
+ struct dn_route __rcu_bh *dn_next;
};
};
diff --git a/include/net/llc.h b/include/net/llc.h
index 8299cb2..5700082 100644
--- a/include/net/llc.h
+++ b/include/net/llc.h
@@ -59,7 +59,8 @@ struct llc_sap {
int (* rcv_func)(struct sk_buff *skb,
struct net_device *dev,
struct packet_type *pt,
- struct net_device *orig_dev) __rcu;
+ struct net_device *orig_dev)
+ __rcu_bh;
struct llc_addr laddr;
struct list_head node;
spinlock_t sk_lock;
diff --git a/include/net/sock.h b/include/net/sock.h
index e07cd78..66d5e09 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -290,7 +290,7 @@ struct sock {
struct ucred sk_peercred;
long sk_rcvtimeo;
long sk_sndtimeo;
- struct sk_filter __rcu *sk_filter;
+ struct sk_filter __rcu_bh *sk_filter;
void *sk_protinfo;
struct timer_list sk_timer;
ktime_t sk_stamp;
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index fc45694..db3e502 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -1392,7 +1392,7 @@ static int cgroup_get_sb(struct file_system_type *fs_type,
root_count++;
sb->s_root->d_fsdata = root_cgrp;
- __rcu_assign_pointer(root->top_cgroup.dentry, sb->s_root);
+ rcu_assign_pointer(root->top_cgroup.dentry, sb->s_root);
/* Link the top cgroup in this hierarchy into all
* the css_set objects */
@@ -3243,7 +3243,7 @@ int __init cgroup_init_early(void)
css_set_count = 1;
init_cgroup_root(&rootnode);
root_count = 1;
- __rcu_assign_pointer(init_task.cgroups, &init_css_set);
+ rcu_assign_pointer(init_task.cgroups, &init_css_set);
init_css_set_link.cg = &init_css_set;
init_css_set_link.cgrp = dummytop;
@@ -3551,7 +3551,7 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
/* Reassign the task to the init_css_set. */
task_lock(tsk);
cg = rcu_dereference_const(tsk->cgroups);
- __rcu_assign_pointer(tsk->cgroups, &init_css_set);
+ rcu_assign_pointer(tsk->cgroups, &init_css_set);
task_unlock(tsk);
if (cg)
put_css_set_taskexit(cg);
@@ -3959,8 +3959,8 @@ static int __init cgroup_subsys_init_idr(struct cgroup_subsys *ss)
return PTR_ERR(newid);
newid->stack[0] = newid->id;
- __rcu_assign_pointer(newid->css, rootcss);
- __rcu_assign_pointer(rootcss->id, newid);
+ rcu_assign_pointer(newid->css, rootcss);
+ rcu_assign_pointer(rootcss->id, newid);
return 0;
}
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index ac8bcbd..e1b65b2 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -1223,8 +1223,8 @@ void perf_event_task_sched_out(struct task_struct *task,
* XXX do we need a memory barrier of sorts
* wrt to rcu_dereference() of perf_event_ctxp
*/
- __rcu_assign_pointer(task->perf_event_ctxp, next_ctx);
- __rcu_assign_pointer(next->perf_event_ctxp, ctx);
+ rcu_assign_pointer(task->perf_event_ctxp, next_ctx);
+ rcu_assign_pointer(next->perf_event_ctxp, ctx);
ctx->task = next;
next_ctx->task = task;
do_switch = 0;
@@ -5376,10 +5376,10 @@ int perf_event_init_task(struct task_struct *child)
*/
cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
if (cloned_ctx) {
- __rcu_assign_pointer(child_ctx->parent_ctx, cloned_ctx);
+ rcu_assign_pointer(child_ctx->parent_ctx, cloned_ctx);
child_ctx->parent_gen = parent_ctx->parent_gen;
} else {
- __rcu_assign_pointer(child_ctx->parent_ctx, parent_ctx);
+ rcu_assign_pointer(child_ctx->parent_ctx, parent_ctx);
child_ctx->parent_gen = parent_ctx->generation;
}
get_ctx(rcu_dereference_const(child_ctx->parent_ctx));
diff --git a/kernel/sched.c b/kernel/sched.c
index 05fd61e..83744d6 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -528,7 +528,7 @@ struct rq {
#ifdef CONFIG_SMP
struct root_domain *rd;
- struct sched_domain __rcu *sd;
+ struct sched_domain __rcu_sched *sd;
unsigned char idle_at_tick;
/* For active balancing */
@@ -603,7 +603,7 @@ static inline int cpu_of(struct rq *rq)
}
#define rcu_dereference_check_sched_domain(p) \
- rcu_dereference_check((p), \
+ rcu_dereference_sched_check((p), \
rcu_read_lock_sched_held() || \
lockdep_is_held(&sched_domains_mutex))
@@ -6323,7 +6323,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
sched_domain_debug(sd, cpu);
rq_attach_root(rq, rd);
- rcu_assign_pointer(rq->sd, sd);
+ rcu_assign_pointer_sched(rq->sd, sd);
}
/* cpus with isolated domains */
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 3e1fd96..5a5ea2c 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -3476,7 +3476,7 @@ static void run_rebalance_domains(struct softirq_action *h)
static inline int on_null_domain(int cpu)
{
- return !rcu_dereference(cpu_rq(cpu)->sd);
+ return !rcu_dereference_sched(cpu_rq(cpu)->sd);
}
/*
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index f6ae74c..4c6f149 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -264,7 +264,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
return -ENOMEM;
/* Increase the height. */
- __rcu_assign_pointer(node->slots[0],
+ rcu_assign_pointer(node->slots[0],
radix_tree_indirect_to_ptr(rcu_dereference_const(root->rnode)));
/* Propagate the aggregated tag info into the new root */
@@ -1090,7 +1090,7 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
newptr = rcu_dereference_const(to_free->slots[0]);
if (root->height > 1)
newptr = radix_tree_ptr_to_indirect(newptr);
- __rcu_assign_pointer(root->rnode, newptr);
+ rcu_assign_pointer(root->rnode, newptr);
root->height--;
radix_tree_node_free(to_free);
}
@@ -1125,7 +1125,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
slot = rcu_dereference_const(root->rnode);
if (height == 0) {
root_tag_clear_all(root);
- __rcu_assign_pointer(root->rnode, NULL);
+ rcu_assign_pointer(root->rnode, NULL);
goto out;
}
slot = radix_tree_indirect_to_ptr(slot);
@@ -1183,7 +1183,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
}
root_tag_clear_all(root);
root->height = 0;
- __rcu_assign_pointer(root->rnode, NULL);
+ rcu_assign_pointer(root->rnode, NULL);
if (to_free)
radix_tree_node_free(to_free);
diff --git a/net/core/filter.c b/net/core/filter.c
index d38ef7f..b88675b 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -522,7 +522,7 @@ int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
rcu_read_lock_bh();
old_fp = rcu_dereference_bh(sk->sk_filter);
- rcu_assign_pointer(sk->sk_filter, fp);
+ rcu_assign_pointer_bh(sk->sk_filter, fp);
rcu_read_unlock_bh();
if (old_fp)
@@ -539,7 +539,7 @@ int sk_detach_filter(struct sock *sk)
rcu_read_lock_bh();
filter = rcu_dereference_bh(sk->sk_filter);
if (filter) {
- rcu_assign_pointer(sk->sk_filter, NULL);
+ rcu_assign_pointer_bh(sk->sk_filter, NULL);
sk_filter_delayed_uncharge(sk, filter);
ret = 0;
}
diff --git a/net/core/sock.c b/net/core/sock.c
index 74242e2..8549387 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1073,11 +1073,11 @@ static void __sk_free(struct sock *sk)
if (sk->sk_destruct)
sk->sk_destruct(sk);
- filter = rcu_dereference_check(sk->sk_filter,
+ filter = rcu_dereference_bh_check(sk->sk_filter,
atomic_read(&sk->sk_wmem_alloc) == 0);
if (filter) {
sk_filter_uncharge(sk, filter);
- rcu_assign_pointer(sk->sk_filter, NULL);
+ rcu_assign_pointer_bh(sk->sk_filter, NULL);
}
sock_disable_timestamp(sk, SOCK_TIMESTAMP);
@@ -1167,7 +1167,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
sock_reset_flag(newsk, SOCK_DONE);
skb_queue_head_init(&newsk->sk_error_queue);
- filter = rcu_dereference_const(newsk->sk_filter);
+ filter = rcu_dereference_bh_const(newsk->sk_filter);
if (filter != NULL)
sk_filter_charge(newsk, filter);
diff --git a/net/decnet/dn_route.c b/net/decnet/dn_route.c
index a7bf03c..22ec1d1 100644
--- a/net/decnet/dn_route.c
+++ b/net/decnet/dn_route.c
@@ -92,7 +92,7 @@
struct dn_rt_hash_bucket
{
- struct dn_route *chain;
+ struct dn_route __rcu_bh *chain;
spinlock_t lock;
};
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 37bf0d9..99cef80 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -200,7 +200,7 @@ const __u8 ip_tos2prio[16] = {
*/
struct rt_hash_bucket {
- struct rtable __rcu *chain;
+ struct rtable __rcu_bh *chain;
};
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
@@ -731,26 +731,26 @@ static void rt_do_flush(int process_context)
spin_lock_bh(rt_hash_lock_addr(i));
#ifdef CONFIG_NET_NS
{
- struct rtable __rcu ** prev;
+ struct rtable __rcu_bh ** prev;
struct rtable * p;
- rth = rcu_dereference_const(rt_hash_table[i].chain);
+ rth = rcu_dereference_bh(rt_hash_table[i].chain);
/* defer releasing the head of the list after spin_unlock */
- for (tail = rth; tail; tail = rcu_dereference_const(tail->u.dst.rt_next))
+ for (tail = rth; tail; tail = rcu_dereference_bh(tail->u.dst.rt_next))
if (!rt_is_expired(tail))
break;
if (rth != tail)
- __rcu_assign_pointer(rt_hash_table[i].chain, tail);
+ rcu_assign_pointer_bh(rt_hash_table[i].chain, tail);
/* call rt_free on entries after the tail requiring flush */
prev = &rt_hash_table[i].chain;
- for (p = rcu_dereference_const(*prev); p; p = next) {
- next = rcu_dereference_const(p->u.dst.rt_next);
+ for (p = rcu_dereference_bh(*prev); p; p = next) {
+ next = rcu_dereference_bh(p->u.dst.rt_next);
if (!rt_is_expired(p)) {
prev = &p->u.dst.rt_next;
} else {
- __rcu_assign_pointer(*prev, next);
+ rcu_assign_pointer_bh(*prev, next);
rt_free(p);
}
}
@@ -763,7 +763,7 @@ static void rt_do_flush(int process_context)
spin_unlock_bh(rt_hash_lock_addr(i));
for (; rth != tail; rth = next) {
- next = rcu_dereference_const(rth->u.dst.rt_next);
+ next = rcu_dereference_bh(rth->u.dst.rt_next);
rt_free(rth);
}
}
@@ -785,7 +785,7 @@ static void rt_check_expire(void)
static unsigned int rover;
unsigned int i = rover, goal;
struct rtable *rth, *aux;
- struct rtable __rcu **rthp;
+ struct rtable __rcu_bh **rthp;
unsigned long samples = 0;
unsigned long sum = 0, sum2 = 0;
unsigned long delta;
@@ -815,8 +815,8 @@ static void rt_check_expire(void)
continue;
length = 0;
spin_lock_bh(rt_hash_lock_addr(i));
- while ((rth = rcu_dereference_const(*rthp)) != NULL) {
- prefetch(rcu_dereference_const(rth->u.dst.rt_next));
+ while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
+ prefetch(rcu_dereference_bh(rth->u.dst.rt_next));
if (rt_is_expired(rth)) {
*rthp = rth->u.dst.rt_next;
rt_free(rth);
@@ -836,14 +836,14 @@ nofree:
* attributes don't unfairly skew
* the length computation
*/
- for (aux = rcu_dereference_const(rt_hash_table[i].chain);;) {
+ for (aux = rcu_dereference_bh(rt_hash_table[i].chain);;) {
if (aux == rth) {
length += ONE;
break;
}
if (compare_hash_inputs(&aux->fl, &rth->fl))
break;
- aux = rcu_dereference_const(aux->u.dst.rt_next);
+ aux = rcu_dereference_bh(aux->u.dst.rt_next);
}
continue;
}
@@ -959,7 +959,7 @@ static int rt_garbage_collect(struct dst_ops *ops)
static int rover;
static int equilibrium;
struct rtable *rth;
- struct rtable __rcu **rthp;
+ struct rtable __rcu_bh **rthp;
unsigned long now = jiffies;
int goal;
@@ -1012,7 +1012,7 @@ static int rt_garbage_collect(struct dst_ops *ops)
k = (k + 1) & rt_hash_mask;
rthp = &rt_hash_table[k].chain;
spin_lock_bh(rt_hash_lock_addr(k));
- while ((rth = rcu_dereference_const(*rthp)) != NULL) {
+ while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
if (!rt_is_expired(rth) &&
!rt_may_expire(rth, tmo, expire)) {
tmo >>= 1;
@@ -1079,10 +1079,10 @@ static int rt_intern_hash(unsigned hash, struct rtable *rt,
struct rtable **rp, struct sk_buff *skb)
{
struct rtable *rth;
- struct rtable __rcu **rthp;
+ struct rtable __rcu_bh **rthp;
unsigned long now;
struct rtable *cand;
- struct rtable __rcu **candp;
+ struct rtable __rcu_bh **candp;
u32 min_score;
int chain_length;
int attempts = !in_softirq();
@@ -1129,7 +1129,7 @@ restart:
rthp = &rt_hash_table[hash].chain;
spin_lock_bh(rt_hash_lock_addr(hash));
- while ((rth = rcu_dereference_const(*rthp)) != NULL) {
+ while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
if (rt_is_expired(rth)) {
*rthp = rth->u.dst.rt_next;
rt_free(rth);
@@ -1143,13 +1143,13 @@ restart:
* must be visible to another weakly ordered CPU before
* the insertion at the start of the hash chain.
*/
- rcu_assign_pointer(rth->u.dst.rt_next,
+ rcu_assign_pointer_bh(rth->u.dst.rt_next,
rt_hash_table[hash].chain);
/*
* Since lookup is lockfree, the update writes
* must be ordered for consistency on SMP.
*/
- rcu_assign_pointer(rt_hash_table[hash].chain, rth);
+ rcu_assign_pointer_bh(rt_hash_table[hash].chain, rth);
dst_use(&rth->u.dst, now);
spin_unlock_bh(rt_hash_lock_addr(hash));
@@ -1252,7 +1252,7 @@ restart:
* previous writes to rt are comitted to memory
* before making rt visible to other CPUS.
*/
- rcu_assign_pointer(rt_hash_table[hash].chain, rt);
+ rcu_assign_pointer_bh(rt_hash_table[hash].chain, rt);
spin_unlock_bh(rt_hash_lock_addr(hash));
@@ -1325,13 +1325,13 @@ void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
static void rt_del(unsigned hash, struct rtable *rt)
{
- struct rtable __rcu **rthp;
+ struct rtable __rcu_bh **rthp;
struct rtable *aux;
rthp = &rt_hash_table[hash].chain;
spin_lock_bh(rt_hash_lock_addr(hash));
ip_rt_put(rt);
- while ((aux = rcu_dereference_const(*rthp)) != NULL) {
+ while ((aux = rcu_dereference_bh(*rthp)) != NULL) {
if (aux == rt || rt_is_expired(aux)) {
*rthp = aux->u.dst.rt_next;
rt_free(aux);
@@ -1348,7 +1348,7 @@ void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
int i, k;
struct in_device *in_dev = in_dev_get(dev);
struct rtable *rth;
- struct rtable __rcu **rthp;
+ struct rtable __rcu_bh **rthp;
__be32 skeys[2] = { saddr, 0 };
int ikeys[2] = { dev->ifindex, 0 };
struct netevent_redirect netevent;
@@ -1384,7 +1384,7 @@ void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
rthp=&rt_hash_table[hash].chain;
rcu_read_lock();
- while ((rth = rcu_dereference(*rthp)) != NULL) {
+ while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
struct rtable *rt;
if (rth->fl.fl4_dst != daddr ||
@@ -1646,8 +1646,8 @@ unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
rt_genid(net));
rcu_read_lock();
- for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
- rth = rcu_dereference(rth->u.dst.rt_next)) {
+ for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
+ rth = rcu_dereference_bh(rth->u.dst.rt_next)) {
unsigned short mtu = new_mtu;
if (rth->fl.fl4_dst != daddr ||
@@ -2287,8 +2287,8 @@ int ip_route_input(struct sk_buff *skb, __be32 daddr, __be32 saddr,
hash = rt_hash(daddr, saddr, iif, rt_genid(net));
rcu_read_lock();
- for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
- rth = rcu_dereference(rth->u.dst.rt_next)) {
+ for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
+ rth = rcu_dereference_bh(rth->u.dst.rt_next)) {
if (((rth->fl.fl4_dst ^ daddr) |
(rth->fl.fl4_src ^ saddr) |
(rth->fl.iif ^ iif) |
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index d8ce05b..003d54f 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -3252,9 +3252,9 @@ void __init tcp_init(void)
memset(&tcp_secret_two.secrets[0], 0, sizeof(tcp_secret_two.secrets));
tcp_secret_one.expires = jiffy; /* past due */
tcp_secret_two.expires = jiffy; /* past due */
- __rcu_assign_pointer(tcp_secret_generating, &tcp_secret_one);
+ rcu_assign_pointer(tcp_secret_generating, &tcp_secret_one);
tcp_secret_primary = &tcp_secret_one;
- __rcu_assign_pointer(tcp_secret_retiring, &tcp_secret_two);
+ rcu_assign_pointer(tcp_secret_retiring, &tcp_secret_two);
tcp_secret_secondary = &tcp_secret_two;
}
diff --git a/net/llc/llc_core.c b/net/llc/llc_core.c
index ed7f424..8696677 100644
--- a/net/llc/llc_core.c
+++ b/net/llc/llc_core.c
@@ -50,7 +50,7 @@ static struct llc_sap *__llc_sap_find(unsigned char sap_value)
{
struct llc_sap* sap;
- list_for_each_entry(sap, &llc_sap_list, node)
+ list_for_each_entry_rcu(sap, &llc_sap_list, node)
if (sap->laddr.lsap == sap_value)
goto out;
sap = NULL;
@@ -103,7 +103,7 @@ struct llc_sap *llc_sap_open(unsigned char lsap,
if (!sap)
goto out;
sap->laddr.lsap = lsap;
- rcu_assign_pointer(sap->rcv_func, func);
+ rcu_assign_pointer_bh(sap->rcv_func, func);
list_add_tail_rcu(&sap->node, &llc_sap_list);
out:
spin_unlock_bh(&llc_sap_list_lock);
@@ -127,7 +127,7 @@ void llc_sap_close(struct llc_sap *sap)
list_del_rcu(&sap->node);
spin_unlock_bh(&llc_sap_list_lock);
- synchronize_rcu();
+ synchronize_rcu_bh();
kfree(sap);
}
diff --git a/net/llc/llc_input.c b/net/llc/llc_input.c
index 57ad974..b775530 100644
--- a/net/llc/llc_input.c
+++ b/net/llc/llc_input.c
@@ -179,7 +179,7 @@ int llc_rcv(struct sk_buff *skb, struct net_device *dev,
* First the upper layer protocols that don't need the full
* LLC functionality
*/
- rcv = rcu_dereference(sap->rcv_func);
+ rcv = rcu_dereference_bh(sap->rcv_func);
if (rcv) {
struct sk_buff *cskb = skb_clone(skb, GFP_ATOMIC);
if (cskb)
diff --git a/virt/kvm/iommu.c b/virt/kvm/iommu.c
index 80fd3ad..2ba7048 100644
--- a/virt/kvm/iommu.c
+++ b/virt/kvm/iommu.c
@@ -78,7 +78,7 @@ static int kvm_iommu_map_memslots(struct kvm *kvm)
int i, r = 0;
struct kvm_memslots *slots;
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < slots->nmemslots; i++) {
r = kvm_iommu_map_pages(kvm, &slots->memslots[i]);
@@ -217,7 +217,7 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm)
int i;
struct kvm_memslots *slots;
- slots = rcu_dereference(kvm->memslots);
+ slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < slots->nmemslots; i++) {
kvm_iommu_put_pages(kvm, slots->memslots[i].base_gfn,
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 548f925..ae28c71 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -372,6 +372,8 @@ static struct kvm *kvm_create_vm(void)
{
int r = 0, i;
struct kvm *kvm = kvm_arch_create_vm();
+ struct kvm_io_bus *buses[KVM_NR_BUSES];
+ struct kvm_memslots *memslots;
if (IS_ERR(kvm))
goto out;
@@ -386,14 +388,15 @@ static struct kvm *kvm_create_vm(void)
#endif
r = -ENOMEM;
- kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+ memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+ srcu_assign_pointer(kvm->memslots, memslots);
if (!kvm->memslots)
goto out_err;
if (init_srcu_struct(&kvm->srcu))
goto out_err;
for (i = 0; i < KVM_NR_BUSES; i++) {
- kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
- GFP_KERNEL);
+ buses[i] = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
+ srcu_assign_pointer(kvm->buses[i], buses[i]);
if (!kvm->buses[i]) {
cleanup_srcu_struct(&kvm->srcu);
goto out_err;
@@ -428,8 +431,8 @@ out_err:
hardware_disable_all();
out_err_nodisable:
for (i = 0; i < KVM_NR_BUSES; i++)
- kfree(kvm->buses[i]);
- kfree(kvm->memslots);
+ kfree(buses[i]);
+ kfree(memslots);
kfree(kvm);
return ERR_PTR(r);
}
@@ -464,12 +467,12 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
void kvm_free_physmem(struct kvm *kvm)
{
int i;
- struct kvm_memslots *slots = kvm->memslots;
+ struct kvm_memslots *slots = srcu_dereference_const(kvm->memslots);
for (i = 0; i < slots->nmemslots; ++i)
kvm_free_physmem_slot(&slots->memslots[i], NULL);
- kfree(kvm->memslots);
+ kfree(slots);
}
static void kvm_destroy_vm(struct kvm *kvm)
@@ -483,7 +486,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
spin_unlock(&kvm_lock);
kvm_free_irq_routing(kvm);
for (i = 0; i < KVM_NR_BUSES; i++)
- kvm_io_bus_destroy(kvm->buses[i]);
+ kvm_io_bus_destroy(srcu_dereference_const(kvm->buses[i]));
kvm_coalesced_mmio_free(kvm);
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
@@ -552,7 +555,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
goto out;
- memslot = &kvm->memslots->memslots[mem->slot];
+ old_memslots = srcu_dereference(kvm->memslots, &kvm->srcu);
+ memslot = &old_memslots->memslots[mem->slot];
base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
npages = mem->memory_size >> PAGE_SHIFT;
@@ -573,7 +577,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
/* Check for overlaps */
r = -EEXIST;
for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
- struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
+ struct kvm_memory_slot *s = &old_memslots->memslots[i];
if (s == memslot || !s->npages)
continue;
@@ -669,13 +673,13 @@ skip_lpage:
slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
if (!slots)
goto out_free;
- memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
+ old_memslots = srcu_dereference_const(kvm->memslots);
+ memcpy(slots, old_memslots, sizeof(struct kvm_memslots));
if (mem->slot >= slots->nmemslots)
slots->nmemslots = mem->slot + 1;
slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
- old_memslots = kvm->memslots;
- rcu_assign_pointer(kvm->memslots, slots);
+ srcu_assign_pointer(kvm->memslots, slots);
synchronize_srcu_expedited(&kvm->srcu);
/* From this point no new shadow pages pointing to a deleted
* memslot will be created.
@@ -705,7 +709,8 @@ skip_lpage:
slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
if (!slots)
goto out_free;
- memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
+ old_memslots = srcu_dereference_const(kvm->memslots);
+ memcpy(slots, old_memslots, sizeof(struct kvm_memslots));
if (mem->slot >= slots->nmemslots)
slots->nmemslots = mem->slot + 1;
@@ -718,8 +723,7 @@ skip_lpage:
}
slots->memslots[mem->slot] = new;
- old_memslots = kvm->memslots;
- rcu_assign_pointer(kvm->memslots, slots);
+ srcu_assign_pointer(kvm->memslots, slots);
synchronize_srcu_expedited(&kvm->srcu);
kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
@@ -775,7 +779,7 @@ int kvm_get_dirty_log(struct kvm *kvm,
if (log->slot >= KVM_MEMORY_SLOTS)
goto out;
- memslot = &kvm->memslots->memslots[log->slot];
+ memslot = &srcu_dereference(kvm->memslots, &kvm->srcu)->memslots[log->slot];
r = -ENOENT;
if (!memslot->dirty_bitmap)
goto out;
@@ -829,7 +833,7 @@ EXPORT_SYMBOL_GPL(kvm_is_error_hva);
struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
{
int i;
- struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+ struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
for (i = 0; i < slots->nmemslots; ++i) {
struct kvm_memory_slot *memslot = &slots->memslots[i];
@@ -851,7 +855,7 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
int i;
- struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+ struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
gfn = unalias_gfn_instantiation(kvm, gfn);
for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
@@ -895,7 +899,7 @@ out:
int memslot_id(struct kvm *kvm, gfn_t gfn)
{
int i;
- struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+ struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
struct kvm_memory_slot *memslot = NULL;
gfn = unalias_gfn(kvm, gfn);
@@ -1984,7 +1988,7 @@ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
int len, const void *val)
{
int i;
- struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
+ struct kvm_io_bus *bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
for (i = 0; i < bus->dev_count; i++)
if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
return 0;
@@ -1996,7 +2000,7 @@ int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
int len, void *val)
{
int i;
- struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
+ struct kvm_io_bus *bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
for (i = 0; i < bus->dev_count; i++)
if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
@@ -2010,7 +2014,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
{
struct kvm_io_bus *new_bus, *bus;
- bus = kvm->buses[bus_idx];
+ bus = srcu_dereference_const(kvm->buses[bus_idx]);
if (bus->dev_count > NR_IOBUS_DEVS-1)
return -ENOSPC;
@@ -2019,7 +2023,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
return -ENOMEM;
memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
new_bus->devs[new_bus->dev_count++] = dev;
- rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+ srcu_assign_pointer(kvm->buses[bus_idx], new_bus);
synchronize_srcu_expedited(&kvm->srcu);
kfree(bus);
@@ -2037,7 +2041,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
if (!new_bus)
return -ENOMEM;
- bus = kvm->buses[bus_idx];
+ bus = srcu_dereference_const(kvm->buses[bus_idx]);
memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
r = -ENOENT;
@@ -2053,7 +2057,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
return r;
}
- rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+ srcu_assign_pointer(kvm->buses[bus_idx], new_bus);
synchronize_srcu_expedited(&kvm->srcu);
kfree(bus);
return r;
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to ***@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html