mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'locking-urgent-2026-11-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Ingo Molnar: - Fix lockdep message output (Zhuangzhuang Wang) - Fix OSQ memory ordering bugs, one of which can cause unlink queue corruption (Peter Zijlstra) * tag 'locking-urgent-2026-11-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/osq: Fix unlink queue corruption locking/osq: Switch osq_wait_next()/atomic_cmpxchg() ordering locking/lockdep: Fix inverted "on"/"last" CPU label in lockdep_print_held_locks()
This commit is contained in:
@@ -811,7 +811,7 @@ static void lockdep_print_held_locks(struct task_struct *p)
|
||||
*/
|
||||
printk("locks held by %s/%d: %d, %s CPU#%d%s\n",
|
||||
p->comm, task_pid_nr(p), depth,
|
||||
task_is_running(p) ? "last" : "on", task_cpu(p),
|
||||
task_is_running(p) ? "on" : "last", task_cpu(p),
|
||||
depth > 0 ? ":" : "");
|
||||
|
||||
for (i = 0; i < depth; i++) {
|
||||
|
||||
+46
-16
@@ -59,7 +59,7 @@ osq_wait_next(struct optimistic_spin_queue *lock,
|
||||
|
||||
for (;;) {
|
||||
if (atomic_read(&lock->tail) == curr &&
|
||||
atomic_cmpxchg_acquire(&lock->tail, curr, old_cpu) == curr) {
|
||||
atomic_cmpxchg_release(&lock->tail, curr, old_cpu) == curr) {
|
||||
/*
|
||||
* We were the last queued, we moved @lock back. @prev
|
||||
* will now observe @lock and will complete its
|
||||
@@ -90,6 +90,49 @@ osq_wait_next(struct optimistic_spin_queue *lock,
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Set prev->next; this is the counterpart of osq_wait_next() in that
|
||||
* by setting ->next the wait is terminated and progress is resumed.
|
||||
*/
|
||||
static inline void osq_link_next(struct optimistic_spin_node *prev,
|
||||
struct optimistic_spin_node *next)
|
||||
{
|
||||
/*
|
||||
* Suppose:
|
||||
*
|
||||
* tail
|
||||
* |
|
||||
* V
|
||||
* CPU1 -> CPU2 -> CPU3:
|
||||
* n: 2 n: 3 n: nil
|
||||
* p: nil p: 1 p: 2
|
||||
*
|
||||
* And this is CPU2 doing the self-unqueue concurrent against CPU1s
|
||||
* unlock()/unqueue(). Since 'prev->next == NULL' and CPU1 will be
|
||||
* stuck in osq_wait_next() until the below store of 'prev->next'.
|
||||
*
|
||||
* CPU2 CPU1
|
||||
*
|
||||
* next->prev = prev; osq_wait_next()
|
||||
* WMB MB
|
||||
* prev->next = next; next->prev = prev
|
||||
*
|
||||
* Without the WMB it would be possible to have conflicting stores
|
||||
* like:
|
||||
*
|
||||
* CPU2 CPU1
|
||||
*
|
||||
* prev->next = next // CPU1.n = 3
|
||||
* next = osq_wait_next() // = 3
|
||||
* next->prev = prev // CPU3.p = nil
|
||||
* next->prev = prev // CPU3.p = 1
|
||||
*
|
||||
* Which would result in list corruption.
|
||||
*/
|
||||
smp_wmb();
|
||||
WRITE_ONCE(prev->next, next);
|
||||
}
|
||||
|
||||
bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
{
|
||||
struct optimistic_spin_node *node = this_cpu_ptr(&osq_node);
|
||||
@@ -113,20 +156,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
|
||||
prev = decode_cpu(old);
|
||||
node->prev = prev;
|
||||
|
||||
/*
|
||||
* osq_lock() unqueue
|
||||
*
|
||||
* node->prev = prev osq_wait_next()
|
||||
* WMB MB
|
||||
* prev->next = node next->prev = prev // unqueue-C
|
||||
*
|
||||
* Here 'node->prev' and 'next->prev' are the same variable and we need
|
||||
* to ensure these stores happen in-order to avoid corrupting the list.
|
||||
*/
|
||||
smp_wmb();
|
||||
|
||||
WRITE_ONCE(prev->next, node);
|
||||
osq_link_next(prev, node);
|
||||
|
||||
/*
|
||||
* Normally @prev is untouchable after the above store; because at that
|
||||
@@ -202,7 +232,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
*/
|
||||
|
||||
WRITE_ONCE(next->prev, prev);
|
||||
WRITE_ONCE(prev->next, next);
|
||||
osq_link_next(prev, next);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user