locking/osq: Fix unlink queue corruption

Very similar to commit 50972fe78f ("locking/osq_lock: Fix osq_lock
queue corruption"), make sure to order the prev->next store against
the next->prev store.

Fixes: 50972fe78f ("locking/osq_lock: Fix osq_lock queue corruption")
Suggested-by: David Laight <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Signed-off-by: Ingo Molnar <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Waiman Long <[email protected]>
Cc: [email protected]
This commit is contained in:
Peter Zijlstra
2026-11-06 11:16:14 +01:00
committed by Ingo Molnar
parent 1b9e01fcc4
commit 408b129aac
+45 -15
View File
@@ -90,6 +90,49 @@ osq_wait_next(struct optimistic_spin_queue *lock,
}
}
/*
* Set prev->next; this is the counterpart of osq_wait_next() in that
* by setting ->next the wait is terminated and progress is resumed.
*/
static inline void osq_link_next(struct optimistic_spin_node *prev,
struct optimistic_spin_node *next)
{
/*
* Suppose:
*
* tail
* |
* V
* CPU1 -> CPU2 -> CPU3:
* n: 2 n: 3 n: nil
* p: nil p: 1 p: 2
*
* And this is CPU2 doing the self-unqueue concurrent against CPU1s
* unlock()/unqueue(). Since 'prev->next == NULL' and CPU1 will be
* stuck in osq_wait_next() until the below store of 'prev->next'.
*
* CPU2 CPU1
*
* next->prev = prev; osq_wait_next()
* WMB MB
* prev->next = next; next->prev = prev
*
* Without the WMB it would be possible to have conflicting stores
* like:
*
* CPU2 CPU1
*
* prev->next = next // CPU1.n = 3
* next = osq_wait_next() // = 3
* next->prev = prev // CPU3.p = nil
* next->prev = prev // CPU3.p = 1
*
* Which would result in list corruption.
*/
smp_wmb();
WRITE_ONCE(prev->next, next);
}
bool osq_lock(struct optimistic_spin_queue *lock)
{
struct optimistic_spin_node *node = this_cpu_ptr(&osq_node);
@@ -113,20 +156,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
prev = decode_cpu(old);
node->prev = prev;
/*
* osq_lock() unqueue
*
* node->prev = prev osq_wait_next()
* WMB MB
* prev->next = node next->prev = prev // unqueue-C
*
* Here 'node->prev' and 'next->prev' are the same variable and we need
* to ensure these stores happen in-order to avoid corrupting the list.
*/
smp_wmb();
WRITE_ONCE(prev->next, node);
osq_link_next(prev, node);
/*
* Normally @prev is untouchable after the above store; because at that
@@ -202,7 +232,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
*/
WRITE_ONCE(next->prev, prev);
WRITE_ONCE(prev->next, next);
osq_link_next(prev, next);
return false;
}