mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
locking/osq: Fix unlink queue corruption
Very similar to commit50972fe78f("locking/osq_lock: Fix osq_lock queue corruption"), make sure to order the prev->next store against the next->prev store. Fixes:50972fe78f("locking/osq_lock: Fix osq_lock queue corruption") Suggested-by: David Laight <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Signed-off-by: Ingo Molnar <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Waiman Long <[email protected]> Cc: [email protected]
This commit is contained in:
committed by
Ingo Molnar
parent
1b9e01fcc4
commit
408b129aac
+45
-15
@@ -90,6 +90,49 @@ osq_wait_next(struct optimistic_spin_queue *lock,
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Set prev->next; this is the counterpart of osq_wait_next() in that
|
||||
* by setting ->next the wait is terminated and progress is resumed.
|
||||
*/
|
||||
static inline void osq_link_next(struct optimistic_spin_node *prev,
|
||||
struct optimistic_spin_node *next)
|
||||
{
|
||||
/*
|
||||
* Suppose:
|
||||
*
|
||||
* tail
|
||||
* |
|
||||
* V
|
||||
* CPU1 -> CPU2 -> CPU3:
|
||||
* n: 2 n: 3 n: nil
|
||||
* p: nil p: 1 p: 2
|
||||
*
|
||||
* And this is CPU2 doing the self-unqueue concurrent against CPU1s
|
||||
* unlock()/unqueue(). Since 'prev->next == NULL' and CPU1 will be
|
||||
* stuck in osq_wait_next() until the below store of 'prev->next'.
|
||||
*
|
||||
* CPU2 CPU1
|
||||
*
|
||||
* next->prev = prev; osq_wait_next()
|
||||
* WMB MB
|
||||
* prev->next = next; next->prev = prev
|
||||
*
|
||||
* Without the WMB it would be possible to have conflicting stores
|
||||
* like:
|
||||
*
|
||||
* CPU2 CPU1
|
||||
*
|
||||
* prev->next = next // CPU1.n = 3
|
||||
* next = osq_wait_next() // = 3
|
||||
* next->prev = prev // CPU3.p = nil
|
||||
* next->prev = prev // CPU3.p = 1
|
||||
*
|
||||
* Which would result in list corruption.
|
||||
*/
|
||||
smp_wmb();
|
||||
WRITE_ONCE(prev->next, next);
|
||||
}
|
||||
|
||||
bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
{
|
||||
struct optimistic_spin_node *node = this_cpu_ptr(&osq_node);
|
||||
@@ -113,20 +156,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
|
||||
prev = decode_cpu(old);
|
||||
node->prev = prev;
|
||||
|
||||
/*
|
||||
* osq_lock() unqueue
|
||||
*
|
||||
* node->prev = prev osq_wait_next()
|
||||
* WMB MB
|
||||
* prev->next = node next->prev = prev // unqueue-C
|
||||
*
|
||||
* Here 'node->prev' and 'next->prev' are the same variable and we need
|
||||
* to ensure these stores happen in-order to avoid corrupting the list.
|
||||
*/
|
||||
smp_wmb();
|
||||
|
||||
WRITE_ONCE(prev->next, node);
|
||||
osq_link_next(prev, node);
|
||||
|
||||
/*
|
||||
* Normally @prev is untouchable after the above store; because at that
|
||||
@@ -202,7 +232,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
|
||||
*/
|
||||
|
||||
WRITE_ONCE(next->prev, prev);
|
||||
WRITE_ONCE(prev->next, next);
|
||||
osq_link_next(prev, next);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user