/** * PROJECT: ExectOS * COPYRIGHT: See COPYING.md in the top level directory * FILE: xtoskrnl/ke/pushlock.cc * DESCRIPTION: XT Kernel Push Lock support * DEVELOPERS: Aiken Harris */ #include /** * Attempts to acquire a push lock for exclusive access via the fast-path. * * @param PushLock * Supplies a pointer to the push lock to be acquired. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock) { /* Attempt to swap a NULL pointer with the EXCLUSIVE flag */ if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)KPUSHLOCK_LOCK)) { /* Lock contention detected, enter the slow-path */ AcquireWaitExclusivePushLock(PushLock); } } /** * Attempts to acquire a push lock for shared access via the fast-path. * * @param PushLock * Supplies a pointer to the push lock to be acquired. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::AcquireSharedPushLock(PKPUSH_LOCK PushLock) { /* Attempt to set the reader flag */ if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED))) { /* Lock contention or existing readers detected, Enter the slow-path */ AcquireWaitSharedPushLock(PushLock); } } /** * Enqueues the current thread's wait block into the lock's wait list, conditionally assumes responsibility * for list optimization and transitions the thread into a wait state until exclusive ownership is acquired. * * @param PushLock * Supplies a pointer to the push lock to be acquired. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK OldState, NewState, TempState; KPUSH_LOCK_WAIT_BLOCK WaitBlock; PVOID ActualState; BOOLEAN Optimize; /* Initialize a synchronization event */ KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE); /* Capture the snapshot of the push lock pointer */ OldState.Ptr = PushLock->Ptr; /* Enter contention loop */ while(TRUE) { /* Check if the lock is free */ if(!(OldState.Value & KPUSHLOCK_LOCK)) { /* Assert the LOCK bit */ NewState.Value = OldState.Value | KPUSHLOCK_LOCK; /* Commit the acquisition */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Lock acquired, return */ return; } /* The lock state changed, retry */ OldState.Ptr = ActualState; continue; } else { /* Prepare the wait block for enqueuing */ WaitBlock.Flags = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING; WaitBlock.Previous = NULLPTR; /* Assume no list optimization is required initially */ Optimize = FALSE; /* Check if other threads are already queued */ if(OldState.Value & KPUSHLOCK_WAITING) { /* Hook into the head of the existing LIFO list */ WaitBlock.Last = NULLPTR; WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); WaitBlock.ShareCount = 0; /* Construct the new pointer containing wait block and preserved flags */ NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) | KPUSHLOCK_LOCK | KPUSHLOCK_WAKING | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; /* Check if no one is currently organizing the list */ if(!(OldState.Value & KPUSHLOCK_WAKING)) { /* Take responsibility for optimizing the wait list */ Optimize = TRUE; } } else { /* Mark this block as the tail of the new wait list and preserve the shared reader count */ WaitBlock.Last = &WaitBlock; WaitBlock.ShareCount = (ULONG)OldState.Shared; /* Check if multiple readers currently hold the lock */ if(OldState.Shared > 1) { /* Inject the wait block and preserve the multiple shared state */ NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; } else { /* Clear the share count for a single owner */ WaitBlock.ShareCount = 0; /* Inject the wait block for a single owner scenario */ NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; } } /* insert wait block into the PushLock */ TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(TempState.Ptr != OldState.Ptr) { /* Lost the queue race, retry */ OldState.Ptr = TempState.Ptr; continue; } /* Check if this thread took the optimization role */ if(Optimize) { /* Convert the LIFO wait list into a FIFO list */ OptimizePushLockList(PushLock, NewState); } /* Block the thread */ SpinPushLock(&WaitBlock); /* Reset state and retry */ WaitBlock.ShareCount = 0; OldState.Ptr = PushLock->Ptr; } } } /** * Enqueues the current thread into the wait list if the lock is held exclusively or if a writer is already waiting. * * @param PushLock * Supplies a pointer to the push lock to be acquired. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::AcquireWaitSharedPushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK OldState, NewState, TempState; KPUSH_LOCK_WAIT_BLOCK WaitBlock; PVOID ActualState; BOOLEAN Optimize; /* Initialize a synchronization event */ KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE); /* Capture the snapshot of the push lock pointer */ OldState.Ptr = PushLock->Ptr; /* Enter contention loop */ while(TRUE) { /* Check if the lock is free */ if(!(OldState.Value & (KPUSHLOCK_LOCK | KPUSHLOCK_WAITING))) { /* Increment the shared reader count */ NewState.Value = OldState.Value + KPUSHLOCK_INCREMENT_SHARED; /* Commit the shared acquisition */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Lock acquired, return */ return; } /* The lock state changed, retry */ OldState.Ptr = ActualState; continue; } else { /* Prepare the wait block for enqueuing as a reader */ WaitBlock.Flags = KPUSHLOCK_WAITING; WaitBlock.Previous = NULLPTR; /* Assume no list optimization is required */ Optimize = FALSE; /* Check if other threads are already queued */ if(OldState.Value & KPUSHLOCK_WAITING) { /* Hook into the head of the existing LIFO wait list */ WaitBlock.Last = NULLPTR; WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); WaitBlock.ShareCount = 0; /* Construct the new pointer containing wait block and preserved flags */ NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; /* Check if no one is currently organizing the list */ if(!(OldState.Value & KPUSHLOCK_WAKING)) { /* Take responsibility for optimizing the wait list */ Optimize = TRUE; } } else { /* Mark this block as the tail of the new wait list and preserve the shared reader count */ WaitBlock.Last = &WaitBlock; WaitBlock.ShareCount = (ULONG)OldState.Shared; /* Check if multiple readers currently hold the lock */ if(OldState.Shared > 1) { /* Inject the wait block and preserve the multiple shared state */ NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; } else { /* Inject the wait block for a single owner */ NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; } } /* Insert the wait block into the PushLock */ TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(TempState.Ptr != OldState.Ptr) { /* Lost the queue race, retry */ OldState.Ptr = TempState.Ptr; continue; } /* Check if this thread took the optimization role */ if(Optimize) { /* Convert the LIFO wait list into a FIFO list */ OptimizePushLockList(PushLock, NewState); } /* Block the thread */ SpinPushLock(&WaitBlock); /* Exit loop */ return; } } } /** * Initializes a push lock. * * @param PushLock * Supplies a pointer to the push lock to be initialized. * * @return This routine does not return any value. * * @since XT 1.0 * */ XTFASTCALL VOID KE::PushLock::InitializePushLock(IN PKPUSH_LOCK PushLock) { /* Initialize the push lock state */ PushLock->Value = 0; } /** * Optimizes the push lock wait list by converting it from a singly-linked LIFO list into a doubly-linked FIFO list. * * @param PushLock * Supplies a pointer to the push lock whose list is being optimized. * * @param OldValue * Supplies the snapshot of the push lock state containing the pointer to the list head. * * @return This routine does not return any value. * * @since XT 1.0 */ XTINLINE VOID KE::PushLock::OptimizePushLockList(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldValue) { PKPUSH_LOCK_WAIT_BLOCK CurrentBlock, NextBlock; /* Extract the pointer to the head of the wait block list */ CurrentBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~KPUSHLOCK_PTR_BITS); /* Ensure the list is not empty */ if(CurrentBlock) { /* Traverse the list */ while((NextBlock = CurrentBlock->Next)) { /* Link the next block back to the current block */ NextBlock->Previous = CurrentBlock; CurrentBlock = NextBlock; } } } /** * Releases a push lock that was acquired for exclusive access. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK OldState; /* Subtract the LOCK bit */ OldState.Value = RTL::Atomic::ExchangeAdd64((PLONG_PTR)&PushLock->Value, -(SSIZE_T)KPUSHLOCK_LOCK); /* Check if waiters are queued and no one is currently waking them */ if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) { /* Initiate the wake sequence */ TryWakePushLock(PushLock); } } /** * Attempts to release the lock immediately via the fast-path. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::ReleasePushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK NewState, OldState; /* Capture the snapshot of the push lock pointer */ OldState.Ptr = PushLock->Ptr; /* Check if the push lock is held by multiple readers */ if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) { /* Decrement the shared reader count */ NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; } else { /* Clear the push lock state */ NewState.Value = 0; } /* Check if there are waiting threads or if the swap fails due to state changes */ if((OldState.Value & KPUSHLOCK_WAITING) || (RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr) != OldState.Ptr)) { /* Contention or waiters present, execute the slow-path release */ ReleaseWaitPushLock(PushLock); } } /** * Releases a push lock that was acquired for shared access. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTINLINE XTFASTCALL VOID KE::PushLock::ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock) { /* Attempt to clear the lock */ if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED), NULLPTR) != (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED)) { /* Contention or multiple readers detected, call slow-path */ ReleaseWaitSharedPushLock(PushLock); } } /** * Drops the lock bit and transitions into the waking phase. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK NewState, OldState; PVOID ActualState; /* Capture the snapshot of the push lock pointer */ OldState.Ptr = PushLock->Ptr; /* Enter release loop */ while(TRUE) { /* Strip the lock bit from the current state */ NewState.Value = OldState.Value & ~KPUSHLOCK_LOCK; /* Check if waiters are queued and no thread is waking them */ if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) { /* Add the waking flag to the state */ NewState.Value |= KPUSHLOCK_WAKING; } /* Commit the modified state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Verify if acquired the waking role */ if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) { /* Execute the wake sequence */ WakePushLockWaiters(PushLock, NewState); } return; } /* State changed, update snapshot and retry */ OldState.Ptr = ActualState; } } /** * Handles decrements under contention and delegates to the wake mechanism if threads are queued * and the lock becomes fully available. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock) { PKPUSH_LOCK_WAIT_BLOCK WaitBlock; KPUSH_LOCK NewState, OldState; PVOID ActualState; /* Capture the snapshot of the push lock pointer */ OldState.Ptr = PushLock->Ptr; /* Enter release loop */ while(TRUE) { /* Check if there are no waiters queued */ if(!(OldState.Value & KPUSHLOCK_WAITING)) { /* Check if the lock is held by multiple readers */ if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) { /* Decrement the shared reader count in the new state */ NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; } else { /* Clear the new state */ NewState.Value = 0; } /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Lock released, return */ return; } /* State changed, retry */ OldState.Ptr = ActualState; continue; } /* Check if multiple readers hold the lock while waiters are queued */ if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) { /* Extract the wait block list head */ WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); /* Traverse to find the last wait block */ while(WaitBlock->Next) { /* Check if the last block pointer is already cached */ if(WaitBlock->Last) { /* Jump directly to the cached last block */ WaitBlock = WaitBlock->Last; break; } /* Advance to the next wait block */ WaitBlock = WaitBlock->Next; } /* Check if the last block is a reader block */ if(WaitBlock->ShareCount > 0) { /* Decrement the block's reader share count */ if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) { /* There are still active readers, return */ return; } } } /* Strip the lock and multiple shared flags from the current state */ NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED); /* Check if another thread is already handling the wake process */ if(OldState.Value & KPUSHLOCK_WAKING) { /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Lock released, return */ return; } } else { /* Add the waking flag to the new state */ NewState.Value |= KPUSHLOCK_WAKING; /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Assumed the waking role, execute wake sequence and return */ WakePushLockWaiters(PushLock, NewState); return; } } /* State changed, update snapshot and retry */ OldState.Ptr = ActualState; } } /** * Handles decrements under contention and delegates to the wake mechanism if needed. * * @param PushLock * Supplies a pointer to the push lock to be released. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock) { PKPUSH_LOCK_WAIT_BLOCK WaitBlock; KPUSH_LOCK NewState, OldState; PVOID ActualState; /* Capture the snapshot of the pointer */ OldState.Ptr = PushLock->Ptr; /* Enter release loop */ while(TRUE) { /* Check if there are no waiters currently queued */ if(!(OldState.Value & KPUSHLOCK_WAITING)) { /* Check if the lock is held by multiple readers */ if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) { /* Decrement the shared reader count in the new state */ NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; } else { /* Clear the new state completely */ NewState.Value = 0; } /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) return; /* State changed, retry */ OldState.Ptr = ActualState; continue; } /* Check if multiple readers hold the lock while waiters are queued */ if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) { /* Extract the wait block list head */ WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); /* Traverse to find the last wait block */ while(WaitBlock->Next) { /* Check if the last block pointer is already cached */ if(WaitBlock->Last) { /* Jump to the cached last block */ WaitBlock = WaitBlock->Last; break; } /* Advance to the next wait block */ WaitBlock = WaitBlock->Next; } /* Check if the last block is a reader block */ if(WaitBlock->ShareCount > 0) { /* Decrement the block's reader share count */ if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) return; } } /* Strip the lock and multiple shared flags from the current state */ NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED); /* Check if another thread is already handling the wake process */ if(OldState.Value & KPUSHLOCK_WAKING) { /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) return; } else { /* Add the waking flag to the new state */ NewState.Value |= KPUSHLOCK_WAKING; /* Commit the new state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Execute wake sequence */ WakePushLockWaiters(PushLock, NewState); return; } } /* State changed, update snapshot and retry */ OldState.Ptr = ActualState; } } /** * Handles the blocking phase for a thread waiting on a push lock. * * @param WaitBlock * Supplies a pointer to the thread's local wait block allocated on the stack. * * @return This routine does not return any value. * * @since XT 1.0 */ XTINLINE VOID KE::PushLock::SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock) { ULONG SpinCount; /* Initialize the spin count */ SpinCount = KPUSH_LOCK_SPIN_COUNT; /* Enter adaptive Spin loop */ while(SpinCount--) { /* Check if the unlocker cleared our wait flag */ if(!(WaitBlock->Flags & KPUSHLOCK_WAITING)) { /* Wait flag cleared, return */ return; } /* Yield processor */ AR::CpuFunctions::YieldProcessor(); } /* Test and clear the WAIT bit */ if(RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1)) { /* Delegate to the Dispatcher */ KE::Dispatcher::WaitForSingleObject(&WaitBlock->WakeEvent, WrPushLock, KernelMode, FALSE, NULLPTR); } } /** * Attempts to acquire a push lock for exclusive access without entering a wait state. * * @param PushLock * Supplies a pointer to the push lock to be acquired. * * @return This routine returns TRUE if the exclusive lock was successfully acquired, or FALSE otherwise. * * @since XT 1.0 */ XTFASTCALL BOOLEAN KE::PushLock::TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock) { /* Acquire the lock */ if(RTL::Atomic::BitTestAndSet((PLONG)&PushLock->Value, KPUSHLOCK_INDEX)) { /* Lock not acquired, return FALSE */ return FALSE; } /* Lock acquired, return TRUE */ return TRUE; } /** * Initiates the wake sequence for queued waiters if the lock is free and no other thread is already performing * the wake operation. * * @param PushLock * Supplies a pointer to the push lock to evaluate. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::TryWakePushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK NewState, OldState; PVOID ActualState; /* Capture the snapshot of the pointer */ OldState.Ptr = PushLock->Ptr; /* Enter evaluation loop */ while(TRUE) { /* Check if already waking, locked, or no one is waiting */ if((OldState.Value & KPUSHLOCK_WAKING) || (OldState.Value & KPUSHLOCK_LOCK) || !(OldState.Value & KPUSHLOCK_WAITING)) { /* Abort */ return; } /* Add the waking flag to the state */ NewState.Value = OldState.Value | KPUSHLOCK_WAKING; /* Commit the waking state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Assumed the waking role, execute wake sequence */ WakePushLockWaiters(PushLock, NewState); return; } /* State changed, update snapshot and retry */ OldState.Ptr = ActualState; } } /** * Wakes threads waiting on a PushLock. * * @param PushLock * Supplies a pointer to the push lock being processed. * * @param OldState * Supplies the current snapshot of the push lock state. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState) { PKPUSH_LOCK_WAIT_BLOCK FirstWaitBlock, PreviousWaitBlock, WaitBlock; KPUSH_LOCK NewState; PVOID ActualState; /* Loop while the push lock is held by another thread */ while(OldState.Value & KPUSHLOCK_LOCK) { /* Strip the waking flag */ NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING; /* Commit the modified state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Lock released, return */ return; } /* Update the snapshot pointer */ OldState.Ptr = ActualState; } /* Extract the first wait block pointer */ FirstWaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); WaitBlock = FirstWaitBlock; /* Loop through the wait block list */ while(WaitBlock->Next) { /* Check if the last block pointer is cached */ if(WaitBlock->Last) { /* Jump to the cached last block */ WaitBlock = WaitBlock->Last; break; } /* Store the current block as the previous block */ PreviousWaitBlock = WaitBlock; /* Advance to the next block and set the back-pointer to the previous block */ WaitBlock = WaitBlock->Next; WaitBlock->Previous = PreviousWaitBlock; } /* Point WaitBlock to the oldest waiter */ PreviousWaitBlock = WaitBlock->Previous; /* Check if the target is a reader block */ if(!(WaitBlock->Flags & KPUSHLOCK_LOCK) || (PreviousWaitBlock == NULLPTR)) { /* Enter clearing loop */ while(TRUE) { /* Clear the new state */ NewState.Value = 0; /* Commit the cleared state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Cleared the lock state, break the loop */ break; } /* Update the snapshot pointer */ OldState.Ptr = ActualState; } } else { /* Update the cached last block pointer and unlink the target block */ FirstWaitBlock->Last = PreviousWaitBlock; WaitBlock->Previous = NULLPTR; /* Enter clearing loop */ while(TRUE) { /* Strip the waking flag from the current state */ NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING; /* Commit the modified state */ ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); if(ActualState == OldState.Ptr) { /* Cleared the waking flag, break the loop */ break; } /* Update the snapshot pointer */ OldState.Ptr = ActualState; } } /* Raise runlevel to DISPATCH level if there are multiple blocks */ KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL, WaitBlock->Previous != NULLPTR); /* Traverse the reversed list */ while(WaitBlock) { /* Save previous wait block */ PreviousWaitBlock = WaitBlock->Previous; /* Clear the WAIT flag */ if(!RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1)) { /* Thread is fully asleep, Signal its event and boost priority */ KE::Event::SetEventBoostPriority(&WaitBlock->WakeEvent, NULLPTR); } /* Go to the next block */ WaitBlock = PreviousWaitBlock; } } /** * Synchronizes the current thread with the PushLock. * * @param PushLock * Supplies a pointer to the push lock to wait on. * * @return This routine does not return any value. * * @since XT 1.0 */ XTFASTCALL VOID KE::PushLock::WaitOnPushLock(IN PKPUSH_LOCK PushLock) { KPUSH_LOCK State; /* Capture the snapshot of the pointer */ State.Ptr = PushLock->Ptr; /* Verify if the lock is actively held by any thread */ if(State.Value & KPUSHLOCK_LOCK) { /* Force serialization by acquiring the lock */ AcquireExclusivePushLock(PushLock); /* Immediately release the lock */ ReleaseExclusivePushLock(PushLock); } }