1028 lines
31 KiB
C++
1028 lines
31 KiB
C++
/**
|
|
* PROJECT: ExectOS
|
|
* COPYRIGHT: See COPYING.md in the top level directory
|
|
* FILE: xtoskrnl/ke/pushlock.cc
|
|
* DESCRIPTION: XT Kernel Push Lock support
|
|
* DEVELOPERS: Aiken Harris <harraiken91@gmail.com>
|
|
*/
|
|
|
|
#include <xtos.hh>
|
|
|
|
|
|
/**
|
|
* Attempts to acquire a push lock for exclusive access via the fast-path.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be acquired.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
/* Attempt to swap a NULL pointer with the EXCLUSIVE flag */
|
|
if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)KPUSHLOCK_LOCK))
|
|
{
|
|
/* Lock contention detected, enter the slow-path */
|
|
AcquireWaitExclusivePushLock(PushLock);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempts to acquire a push lock for shared access via the fast-path.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be acquired.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::AcquireSharedPushLock(PKPUSH_LOCK PushLock)
|
|
{
|
|
/* Attempt to set the reader flag */
|
|
if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED)))
|
|
{
|
|
/* Lock contention or existing readers detected, Enter the slow-path */
|
|
AcquireWaitSharedPushLock(PushLock);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enqueues the current thread's wait block into the lock's wait list, conditionally assumes responsibility
|
|
* for list optimization and transitions the thread into a wait state until exclusive ownership is acquired.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be acquired.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK OldState, NewState, TempState;
|
|
KPUSH_LOCK_WAIT_BLOCK WaitBlock;
|
|
PVOID ActualState;
|
|
BOOLEAN Optimize;
|
|
|
|
/* Initialize a synchronization event */
|
|
KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE);
|
|
|
|
/* Capture the snapshot of the push lock pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter contention loop */
|
|
while(TRUE)
|
|
{
|
|
/* Check if the lock is free */
|
|
if(!(OldState.Value & KPUSHLOCK_LOCK))
|
|
{
|
|
/* Assert the LOCK bit */
|
|
NewState.Value = OldState.Value | KPUSHLOCK_LOCK;
|
|
|
|
/* Commit the acquisition */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Lock acquired, return */
|
|
return;
|
|
}
|
|
|
|
/* The lock state changed, retry */
|
|
OldState.Ptr = ActualState;
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
/* Prepare the wait block for enqueuing */
|
|
WaitBlock.Flags = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING;
|
|
WaitBlock.Previous = NULLPTR;
|
|
|
|
/* Assume no list optimization is required initially */
|
|
Optimize = FALSE;
|
|
|
|
/* Check if other threads are already queued */
|
|
if(OldState.Value & KPUSHLOCK_WAITING)
|
|
{
|
|
/* Hook into the head of the existing LIFO list */
|
|
WaitBlock.Last = NULLPTR;
|
|
WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
|
|
WaitBlock.ShareCount = 0;
|
|
|
|
/* Construct the new pointer containing wait block and preserved flags */
|
|
NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) |
|
|
KPUSHLOCK_LOCK |
|
|
KPUSHLOCK_WAKING |
|
|
KPUSHLOCK_WAITING |
|
|
(ULONG_PTR)&WaitBlock;
|
|
|
|
/* Check if no one is currently organizing the list */
|
|
if(!(OldState.Value & KPUSHLOCK_WAKING))
|
|
{
|
|
/* Take responsibility for optimizing the wait list */
|
|
Optimize = TRUE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* Mark this block as the tail of the new wait list and preserve the shared reader count */
|
|
WaitBlock.Last = &WaitBlock;
|
|
WaitBlock.ShareCount = (ULONG)OldState.Shared;
|
|
|
|
/* Check if multiple readers currently hold the lock */
|
|
if(OldState.Shared > 1)
|
|
{
|
|
/* Inject the wait block and preserve the multiple shared state */
|
|
NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
|
|
}
|
|
else
|
|
{
|
|
/* Clear the share count for a single owner */
|
|
WaitBlock.ShareCount = 0;
|
|
|
|
/* Inject the wait block for a single owner scenario */
|
|
NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
|
|
}
|
|
}
|
|
|
|
/* insert wait block into the PushLock */
|
|
TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(TempState.Ptr != OldState.Ptr)
|
|
{
|
|
/* Lost the queue race, retry */
|
|
OldState.Ptr = TempState.Ptr;
|
|
continue;
|
|
}
|
|
|
|
/* Check if this thread took the optimization role */
|
|
if(Optimize)
|
|
{
|
|
/* Convert the LIFO wait list into a FIFO list */
|
|
OptimizePushLockList(PushLock, NewState);
|
|
}
|
|
|
|
/* Block the thread */
|
|
SpinPushLock(&WaitBlock);
|
|
|
|
/* Reset state and retry */
|
|
WaitBlock.ShareCount = 0;
|
|
OldState.Ptr = PushLock->Ptr;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enqueues the current thread into the wait list if the lock is held exclusively or if a writer is already waiting.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be acquired.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::AcquireWaitSharedPushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK OldState, NewState, TempState;
|
|
KPUSH_LOCK_WAIT_BLOCK WaitBlock;
|
|
PVOID ActualState;
|
|
BOOLEAN Optimize;
|
|
|
|
/* Initialize a synchronization event */
|
|
KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE);
|
|
|
|
/* Capture the snapshot of the push lock pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter contention loop */
|
|
while(TRUE)
|
|
{
|
|
/* Check if the lock is free */
|
|
if(!(OldState.Value & (KPUSHLOCK_LOCK | KPUSHLOCK_WAITING)))
|
|
{
|
|
/* Increment the shared reader count */
|
|
NewState.Value = OldState.Value + KPUSHLOCK_INCREMENT_SHARED;
|
|
|
|
/* Commit the shared acquisition */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Lock acquired, return */
|
|
return;
|
|
}
|
|
|
|
/* The lock state changed, retry */
|
|
OldState.Ptr = ActualState;
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
/* Prepare the wait block for enqueuing as a reader */
|
|
WaitBlock.Flags = KPUSHLOCK_WAITING;
|
|
WaitBlock.Previous = NULLPTR;
|
|
|
|
/* Assume no list optimization is required */
|
|
Optimize = FALSE;
|
|
|
|
/* Check if other threads are already queued */
|
|
if(OldState.Value & KPUSHLOCK_WAITING)
|
|
{
|
|
/* Hook into the head of the existing LIFO wait list */
|
|
WaitBlock.Last = NULLPTR;
|
|
WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
|
|
WaitBlock.ShareCount = 0;
|
|
|
|
/* Construct the new pointer containing wait block and preserved flags */
|
|
NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) |
|
|
KPUSHLOCK_LOCK |
|
|
KPUSHLOCK_WAITING |
|
|
(ULONG_PTR)&WaitBlock;
|
|
|
|
/* Check if no one is currently organizing the list */
|
|
if(!(OldState.Value & KPUSHLOCK_WAKING))
|
|
{
|
|
/* Take responsibility for optimizing the wait list */
|
|
Optimize = TRUE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* Mark this block as the tail of the new wait list and preserve the shared reader count */
|
|
WaitBlock.Last = &WaitBlock;
|
|
WaitBlock.ShareCount = (ULONG)OldState.Shared;
|
|
|
|
/* Check if multiple readers currently hold the lock */
|
|
if(OldState.Shared > 1)
|
|
{
|
|
/* Inject the wait block and preserve the multiple shared state */
|
|
NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
|
|
}
|
|
else
|
|
{
|
|
/* Inject the wait block for a single owner */
|
|
NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
|
|
}
|
|
}
|
|
|
|
/* Insert the wait block into the PushLock */
|
|
TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(TempState.Ptr != OldState.Ptr)
|
|
{
|
|
/* Lost the queue race, retry */
|
|
OldState.Ptr = TempState.Ptr;
|
|
continue;
|
|
}
|
|
|
|
/* Check if this thread took the optimization role */
|
|
if(Optimize)
|
|
{
|
|
/* Convert the LIFO wait list into a FIFO list */
|
|
OptimizePushLockList(PushLock, NewState);
|
|
}
|
|
|
|
/* Block the thread */
|
|
SpinPushLock(&WaitBlock);
|
|
|
|
/* Exit loop */
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempts to convert a push lock from shared to exclusive access.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock structure.
|
|
*
|
|
* @return This routine returns TRUE if the lock was successfully converted to exclusive mode, or FALSE otherwise.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
BOOLEAN
|
|
KE::PushLock::ConvertSharedPushLockToExclusive(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
PVOID OldValue;
|
|
|
|
/* Swap the push lock state */
|
|
OldValue = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr,
|
|
(PVOID)KPUSHLOCK_INCREMENT_SHARED,
|
|
(PVOID)KPUSHLOCK_LOCK);
|
|
|
|
/* Return conversion result */
|
|
return (OldValue == (PVOID)KPUSHLOCK_INCREMENT_SHARED);
|
|
}
|
|
|
|
/**
|
|
* Initializes a push lock.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be initialized.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::InitializePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
/* Initialize the push lock state */
|
|
PushLock->Value = 0;
|
|
}
|
|
|
|
/**
|
|
* Optimizes the push lock wait list by converting it from a singly-linked LIFO list into a doubly-linked FIFO list.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock whose list is being optimized.
|
|
*
|
|
* @param OldValue
|
|
* Supplies the snapshot of the push lock state containing the pointer to the list head.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTINLINE
|
|
VOID
|
|
KE::PushLock::OptimizePushLockList(IN PKPUSH_LOCK PushLock,
|
|
IN KPUSH_LOCK OldValue)
|
|
{
|
|
PKPUSH_LOCK_WAIT_BLOCK CurrentBlock, NextBlock;
|
|
|
|
/* Extract the pointer to the head of the wait block list */
|
|
CurrentBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~KPUSHLOCK_PTR_BITS);
|
|
|
|
/* Ensure the list is not empty */
|
|
if(CurrentBlock)
|
|
{
|
|
/* Traverse the list */
|
|
while((NextBlock = CurrentBlock->Next))
|
|
{
|
|
/* Link the next block back to the current block */
|
|
NextBlock->Previous = CurrentBlock;
|
|
CurrentBlock = NextBlock;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Releases a push lock that was acquired for exclusive access.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK OldState;
|
|
|
|
/* Subtract the LOCK bit */
|
|
OldState.Value = RTL::Atomic::ExchangeAdd64((PLONG_PTR)&PushLock->Value, -(SSIZE_T)KPUSHLOCK_LOCK);
|
|
|
|
/* Check if waiters are queued and no one is currently waking them */
|
|
if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
|
|
{
|
|
/* Initiate the wake sequence */
|
|
TryWakePushLock(PushLock);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempts to release the lock immediately via the fast-path.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleasePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK NewState, OldState;
|
|
|
|
/* Capture the snapshot of the push lock pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Check if the push lock is held by multiple readers */
|
|
if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
|
|
{
|
|
/* Decrement the shared reader count */
|
|
NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
|
|
}
|
|
else
|
|
{
|
|
/* Clear the push lock state */
|
|
NewState.Value = 0;
|
|
}
|
|
|
|
/* Check if there are waiting threads or if the swap fails due to state changes */
|
|
if((OldState.Value & KPUSHLOCK_WAITING) ||
|
|
(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr) != OldState.Ptr))
|
|
{
|
|
/* Contention or waiters present, execute the slow-path release */
|
|
ReleaseWaitPushLock(PushLock);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Releases a push lock that was acquired for shared access.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTINLINE
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
/* Attempt to clear the lock */
|
|
if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED), NULLPTR)
|
|
!= (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED))
|
|
{
|
|
/* Contention or multiple readers detected, call slow-path */
|
|
ReleaseWaitSharedPushLock(PushLock);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Drops the lock bit and transitions into the waking phase.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK NewState, OldState;
|
|
PVOID ActualState;
|
|
|
|
/* Capture the snapshot of the push lock pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter release loop */
|
|
while(TRUE)
|
|
{
|
|
/* Strip the lock bit from the current state */
|
|
NewState.Value = OldState.Value & ~KPUSHLOCK_LOCK;
|
|
|
|
/* Check if waiters are queued and no thread is waking them */
|
|
if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
|
|
{
|
|
/* Add the waking flag to the state */
|
|
NewState.Value |= KPUSHLOCK_WAKING;
|
|
}
|
|
|
|
/* Commit the modified state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Verify if acquired the waking role */
|
|
if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
|
|
{
|
|
/* Execute the wake sequence */
|
|
WakePushLockWaiters(PushLock, NewState);
|
|
}
|
|
return;
|
|
}
|
|
|
|
/* State changed, update snapshot and retry */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Handles decrements under contention and delegates to the wake mechanism if threads are queued
|
|
* and the lock becomes fully available.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
PKPUSH_LOCK_WAIT_BLOCK WaitBlock;
|
|
KPUSH_LOCK NewState, OldState;
|
|
PVOID ActualState;
|
|
|
|
/* Capture the snapshot of the push lock pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter release loop */
|
|
while(TRUE)
|
|
{
|
|
/* Check if there are no waiters queued */
|
|
if(!(OldState.Value & KPUSHLOCK_WAITING))
|
|
{
|
|
/* Check if the lock is held by multiple readers */
|
|
if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
|
|
{
|
|
/* Decrement the shared reader count in the new state */
|
|
NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
|
|
}
|
|
else
|
|
{
|
|
/* Clear the new state */
|
|
NewState.Value = 0;
|
|
}
|
|
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Lock released, return */
|
|
return;
|
|
}
|
|
|
|
/* State changed, retry */
|
|
OldState.Ptr = ActualState;
|
|
continue;
|
|
}
|
|
|
|
/* Check if multiple readers hold the lock while waiters are queued */
|
|
if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
|
|
{
|
|
/* Extract the wait block list head */
|
|
WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
|
|
|
|
/* Traverse to find the last wait block */
|
|
while(WaitBlock->Next)
|
|
{
|
|
/* Check if the last block pointer is already cached */
|
|
if(WaitBlock->Last)
|
|
{
|
|
/* Jump directly to the cached last block */
|
|
WaitBlock = WaitBlock->Last;
|
|
break;
|
|
}
|
|
|
|
/* Advance to the next wait block */
|
|
WaitBlock = WaitBlock->Next;
|
|
}
|
|
|
|
/* Check if the last block is a reader block */
|
|
if(WaitBlock->ShareCount > 0)
|
|
{
|
|
/* Decrement the block's reader share count */
|
|
if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0)
|
|
{
|
|
/* There are still active readers, return */
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Strip the lock and multiple shared flags from the current state */
|
|
NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED);
|
|
|
|
/* Check if another thread is already handling the wake process */
|
|
if(OldState.Value & KPUSHLOCK_WAKING)
|
|
{
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Lock released, return */
|
|
return;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* Add the waking flag to the new state */
|
|
NewState.Value |= KPUSHLOCK_WAKING;
|
|
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Assumed the waking role, execute wake sequence and return */
|
|
WakePushLockWaiters(PushLock, NewState);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* State changed, update snapshot and retry */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Handles decrements under contention and delegates to the wake mechanism if needed.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be released.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
PKPUSH_LOCK_WAIT_BLOCK WaitBlock;
|
|
KPUSH_LOCK NewState, OldState;
|
|
PVOID ActualState;
|
|
|
|
/* Capture the snapshot of the pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter release loop */
|
|
while(TRUE)
|
|
{
|
|
/* Check if there are no waiters currently queued */
|
|
if(!(OldState.Value & KPUSHLOCK_WAITING))
|
|
{
|
|
/* Check if the lock is held by multiple readers */
|
|
if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
|
|
{
|
|
/* Decrement the shared reader count in the new state */
|
|
NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
|
|
}
|
|
else
|
|
{
|
|
/* Clear the new state completely */
|
|
NewState.Value = 0;
|
|
}
|
|
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr) return;
|
|
|
|
/* State changed, retry */
|
|
OldState.Ptr = ActualState;
|
|
continue;
|
|
}
|
|
|
|
/* Check if multiple readers hold the lock while waiters are queued */
|
|
if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
|
|
{
|
|
/* Extract the wait block list head */
|
|
WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
|
|
|
|
/* Traverse to find the last wait block */
|
|
while(WaitBlock->Next)
|
|
{
|
|
/* Check if the last block pointer is already cached */
|
|
if(WaitBlock->Last)
|
|
{
|
|
/* Jump to the cached last block */
|
|
WaitBlock = WaitBlock->Last;
|
|
break;
|
|
}
|
|
|
|
/* Advance to the next wait block */
|
|
WaitBlock = WaitBlock->Next;
|
|
}
|
|
|
|
/* Check if the last block is a reader block */
|
|
if(WaitBlock->ShareCount > 0)
|
|
{
|
|
/* Decrement the block's reader share count */
|
|
if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) return;
|
|
}
|
|
}
|
|
|
|
/* Strip the lock and multiple shared flags from the current state */
|
|
NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED);
|
|
|
|
/* Check if another thread is already handling the wake process */
|
|
if(OldState.Value & KPUSHLOCK_WAKING)
|
|
{
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr) return;
|
|
}
|
|
else
|
|
{
|
|
/* Add the waking flag to the new state */
|
|
NewState.Value |= KPUSHLOCK_WAKING;
|
|
|
|
/* Commit the new state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Execute wake sequence */
|
|
WakePushLockWaiters(PushLock, NewState);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* State changed, update snapshot and retry */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Handles the blocking phase for a thread waiting on a push lock.
|
|
*
|
|
* @param WaitBlock
|
|
* Supplies a pointer to the thread's local wait block allocated on the stack.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTINLINE
|
|
VOID
|
|
KE::PushLock::SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock)
|
|
{
|
|
ULONG SpinCount;
|
|
|
|
/* Initialize the spin count */
|
|
SpinCount = KPUSH_LOCK_SPIN_COUNT;
|
|
|
|
/* Enter adaptive Spin loop */
|
|
while(SpinCount--)
|
|
{
|
|
/* Check if the unlocker cleared our wait flag */
|
|
if(!(WaitBlock->Flags & KPUSHLOCK_WAITING))
|
|
{
|
|
/* Wait flag cleared, return */
|
|
return;
|
|
}
|
|
|
|
/* Yield processor */
|
|
AR::CpuFunctions::YieldProcessor();
|
|
}
|
|
|
|
/* Test and clear the WAIT bit */
|
|
if(RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1))
|
|
{
|
|
/* Delegate to the Dispatcher */
|
|
KE::Dispatcher::WaitForSingleObject(&WaitBlock->WakeEvent, WrPushLock, KernelMode, FALSE, NULLPTR);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempts to acquire a push lock for exclusive access without entering a wait state.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to be acquired.
|
|
*
|
|
* @return This routine returns TRUE if the exclusive lock was successfully acquired, or FALSE otherwise.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
BOOLEAN
|
|
KE::PushLock::TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock)
|
|
{
|
|
/* Acquire the lock */
|
|
if(RTL::Atomic::BitTestAndSet((PLONG)&PushLock->Value, KPUSHLOCK_INDEX))
|
|
{
|
|
/* Lock not acquired, return FALSE */
|
|
return FALSE;
|
|
}
|
|
|
|
/* Lock acquired, return TRUE */
|
|
return TRUE;
|
|
}
|
|
|
|
/**
|
|
* Initiates the wake sequence for queued waiters if the lock is free and no other thread is already performing
|
|
* the wake operation.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to evaluate.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::TryWakePushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK NewState, OldState;
|
|
PVOID ActualState;
|
|
|
|
/* Capture the snapshot of the pointer */
|
|
OldState.Ptr = PushLock->Ptr;
|
|
|
|
/* Enter evaluation loop */
|
|
while(TRUE)
|
|
{
|
|
/* Check if already waking, locked, or no one is waiting */
|
|
if((OldState.Value & KPUSHLOCK_WAKING) ||
|
|
(OldState.Value & KPUSHLOCK_LOCK) ||
|
|
!(OldState.Value & KPUSHLOCK_WAITING))
|
|
{
|
|
/* Abort */
|
|
return;
|
|
}
|
|
|
|
/* Add the waking flag to the state */
|
|
NewState.Value = OldState.Value | KPUSHLOCK_WAKING;
|
|
|
|
/* Commit the waking state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Assumed the waking role, execute wake sequence */
|
|
WakePushLockWaiters(PushLock, NewState);
|
|
return;
|
|
}
|
|
|
|
/* State changed, update snapshot and retry */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Wakes threads waiting on a PushLock.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock being processed.
|
|
*
|
|
* @param OldState
|
|
* Supplies the current snapshot of the push lock state.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState)
|
|
{
|
|
PKPUSH_LOCK_WAIT_BLOCK FirstWaitBlock, PreviousWaitBlock, WaitBlock;
|
|
KPUSH_LOCK NewState;
|
|
PVOID ActualState;
|
|
|
|
/* Loop while the push lock is held by another thread */
|
|
while(OldState.Value & KPUSHLOCK_LOCK)
|
|
{
|
|
/* Strip the waking flag */
|
|
NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING;
|
|
|
|
/* Commit the modified state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Lock released, return */
|
|
return;
|
|
}
|
|
|
|
/* Update the snapshot pointer */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
|
|
/* Extract the first wait block pointer */
|
|
FirstWaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
|
|
WaitBlock = FirstWaitBlock;
|
|
|
|
/* Loop through the wait block list */
|
|
while(WaitBlock->Next)
|
|
{
|
|
/* Check if the last block pointer is cached */
|
|
if(WaitBlock->Last)
|
|
{
|
|
/* Jump to the cached last block */
|
|
WaitBlock = WaitBlock->Last;
|
|
break;
|
|
}
|
|
|
|
/* Store the current block as the previous block */
|
|
PreviousWaitBlock = WaitBlock;
|
|
|
|
/* Advance to the next block and set the back-pointer to the previous block */
|
|
WaitBlock = WaitBlock->Next;
|
|
WaitBlock->Previous = PreviousWaitBlock;
|
|
}
|
|
|
|
/* Point WaitBlock to the oldest waiter */
|
|
PreviousWaitBlock = WaitBlock->Previous;
|
|
|
|
/* Check if the target is a reader block */
|
|
if(!(WaitBlock->Flags & KPUSHLOCK_LOCK) || (PreviousWaitBlock == NULLPTR))
|
|
{
|
|
/* Enter clearing loop */
|
|
while(TRUE)
|
|
{
|
|
/* Clear the new state */
|
|
NewState.Value = 0;
|
|
|
|
/* Commit the cleared state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Cleared the lock state, break the loop */
|
|
break;
|
|
}
|
|
|
|
/* Update the snapshot pointer */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* Update the cached last block pointer and unlink the target block */
|
|
FirstWaitBlock->Last = PreviousWaitBlock;
|
|
WaitBlock->Previous = NULLPTR;
|
|
|
|
/* Enter clearing loop */
|
|
while(TRUE)
|
|
{
|
|
/* Strip the waking flag from the current state */
|
|
NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING;
|
|
|
|
/* Commit the modified state */
|
|
ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
|
|
if(ActualState == OldState.Ptr)
|
|
{
|
|
/* Cleared the waking flag, break the loop */
|
|
break;
|
|
}
|
|
|
|
/* Update the snapshot pointer */
|
|
OldState.Ptr = ActualState;
|
|
}
|
|
}
|
|
|
|
/* Raise runlevel to DISPATCH level if there are multiple blocks */
|
|
KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL, WaitBlock->Previous != NULLPTR);
|
|
|
|
/* Traverse the reversed list */
|
|
while(WaitBlock)
|
|
{
|
|
/* Save previous wait block */
|
|
PreviousWaitBlock = WaitBlock->Previous;
|
|
|
|
/* Clear the WAIT flag */
|
|
if(!RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1))
|
|
{
|
|
/* Thread is fully asleep, Signal its event and boost priority */
|
|
KE::Event::SetEventBoostPriority(&WaitBlock->WakeEvent, NULLPTR);
|
|
}
|
|
|
|
/* Go to the next block */
|
|
WaitBlock = PreviousWaitBlock;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Synchronizes the current thread with the PushLock.
|
|
*
|
|
* @param PushLock
|
|
* Supplies a pointer to the push lock to wait on.
|
|
*
|
|
* @return This routine does not return any value.
|
|
*
|
|
* @since XT 1.0
|
|
*/
|
|
XTFASTCALL
|
|
VOID
|
|
KE::PushLock::WaitOnPushLock(IN PKPUSH_LOCK PushLock)
|
|
{
|
|
KPUSH_LOCK State;
|
|
|
|
/* Capture the snapshot of the pointer */
|
|
State.Ptr = PushLock->Ptr;
|
|
|
|
/* Verify if the lock is actively held by any thread */
|
|
if(State.Value & KPUSHLOCK_LOCK)
|
|
{
|
|
/* Force serialization by acquiring the lock */
|
|
AcquireExclusivePushLock(PushLock);
|
|
|
|
/* Immediately release the lock */
|
|
ReleaseExclusivePushLock(PushLock);
|
|
}
|
|
}
|