From 84d35e3c97fe6a7eac8ae396027acc0a871ec431 Mon Sep 17 00:00:00 2001 From: Aiken Harris Date: Mon, 29 Jun 2026 18:25:40 +0200 Subject: [PATCH] Implement kernel push lock infrastructure --- sdk/xtdk/ketypes.h | 14 +- xtoskrnl/CMakeLists.txt | 1 + xtoskrnl/includes/ke.hh | 1 + xtoskrnl/includes/ke/pushlock.hh | 42 ++ xtoskrnl/ke/pushlock.cc | 984 +++++++++++++++++++++++++++++++ 5 files changed, 1041 insertions(+), 1 deletion(-) create mode 100644 xtoskrnl/includes/ke/pushlock.hh create mode 100644 xtoskrnl/ke/pushlock.cc diff --git a/sdk/xtdk/ketypes.h b/sdk/xtdk/ketypes.h index a6343df..53d67d7 100644 --- a/sdk/xtdk/ketypes.h +++ b/sdk/xtdk/ketypes.h @@ -47,8 +47,20 @@ /* APC pending state length */ #define KAPC_STATE_LENGTH (FIELD_OFFSET(KAPC_STATE, UserApcPending) + sizeof(BOOLEAN)) -/* Push lock total bits */ +/* Indices used to access the PushLock data structure */ +#define KPUSHLOCK_INDEX ((ULONG_PTR)0x00) +#define KPUSHLOCK_LOCK ((ULONG_PTR)0x01) +#define KPUSHLOCK_WAITING ((ULONG_PTR)0x02) +#define KPUSHLOCK_WAKING ((ULONG_PTR)0x04) +#define KPUSHLOCK_MULTIPLE_SHARED ((ULONG_PTR)0x08) +#define KPUSHLOCK_INCREMENT_SHARED ((ULONG_PTR)0x10) + +/* Mask for the pointer bits */ +#define KPUSHLOCK_PTR_BITS ((ULONG_PTR)0x0F) + +/* PushLock related definitions */ #define KPUSH_LOCK_TOTAL_BITS (sizeof(ULONG_PTR) * 8) +#define KPUSH_LOCK_SPIN_COUNT 1024 /* Kernel service descriptor tables count */ #define KSERVICE_TABLES_COUNT 4 diff --git a/xtoskrnl/CMakeLists.txt b/xtoskrnl/CMakeLists.txt index e167f2d..3a1c242 100644 --- a/xtoskrnl/CMakeLists.txt +++ b/xtoskrnl/CMakeLists.txt @@ -58,6 +58,7 @@ list(APPEND XTOSKRNL_SOURCE ${XTOSKRNL_SOURCE_DIR}/ke/krnlinit.cc ${XTOSKRNL_SOURCE_DIR}/ke/kthread.cc ${XTOSKRNL_SOURCE_DIR}/ke/kubsan.cc + ${XTOSKRNL_SOURCE_DIR}/ke/pushlock.cc ${XTOSKRNL_SOURCE_DIR}/ke/runlevel.cc ${XTOSKRNL_SOURCE_DIR}/ke/schedule.cc ${XTOSKRNL_SOURCE_DIR}/ke/semphore.cc diff --git a/xtoskrnl/includes/ke.hh b/xtoskrnl/includes/ke.hh index 4032b0c..f64abbf 100644 --- a/xtoskrnl/includes/ke.hh +++ b/xtoskrnl/includes/ke.hh @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/xtoskrnl/includes/ke/pushlock.hh b/xtoskrnl/includes/ke/pushlock.hh new file mode 100644 index 0000000..a4aae28 --- /dev/null +++ b/xtoskrnl/includes/ke/pushlock.hh @@ -0,0 +1,42 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/ke/pushlock.hh + * DESCRIPTION: XT Kernel Push Lock support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_KE_PUSHLOCK_HH +#define __XTOSKRNL_KE_PUSHLOCK_HH + +#include + + +/* Kernel Library */ +namespace KE +{ + class PushLock + { + public: + STATIC XTFASTCALL VOID AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID AcquireSharedPushLock(PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID AcquireWaitSharedPushLock(IN OUT PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleasePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL BOOLEAN TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID TryWakePushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID WaitOnPushLock(IN PKPUSH_LOCK PushLock); + STATIC XTFASTCALL VOID WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState); + + private: + STATIC XTINLINE VOID OptimizePushLockList(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldValue); + STATIC XTINLINE XTFASTCALL VOID SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock); + }; +} + +#endif /* __XTOSKRNL_KE_PUSHLOCK_HH */ diff --git a/xtoskrnl/ke/pushlock.cc b/xtoskrnl/ke/pushlock.cc new file mode 100644 index 0000000..ea73a30 --- /dev/null +++ b/xtoskrnl/ke/pushlock.cc @@ -0,0 +1,984 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/ke/pushlock.cc + * DESCRIPTION: XT Kernel Push Lock support + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Attempts to acquire a push lock for exclusive access via the fast-path. + * + * @param PushLock + * Supplies a pointer to the push lock to be acquired. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock) +{ + /* Attempt to swap a NULL pointer with the EXCLUSIVE flag */ + if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)KPUSHLOCK_LOCK)) + { + /* Lock contention detected, enter the slow-path */ + AcquireWaitExclusivePushLock(PushLock); + } +} + +/** + * Attempts to acquire a push lock for shared access via the fast-path. + * + * @param PushLock + * Supplies a pointer to the push lock to be acquired. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::AcquireSharedPushLock(PKPUSH_LOCK PushLock) +{ + /* Attempt to set the reader flag */ + if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED))) + { + /* Lock contention or existing readers detected, Enter the slow-path */ + AcquireWaitSharedPushLock(PushLock); + } +} + +/** + * Enqueues the current thread's wait block into the lock's wait list, conditionally assumes responsibility + * for list optimization and transitions the thread into a wait state until exclusive ownership is acquired. + * + * @param PushLock + * Supplies a pointer to the push lock to be acquired. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK OldState, NewState, TempState; + KPUSH_LOCK_WAIT_BLOCK WaitBlock; + PVOID ActualState; + BOOLEAN Optimize; + + /* Initialize a synchronization event */ + KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE); + + /* Capture the snapshot of the push lock pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter contention loop */ + while(TRUE) + { + /* Check if the lock is free */ + if(!(OldState.Value & KPUSHLOCK_LOCK)) + { + /* Assert the LOCK bit */ + NewState.Value = OldState.Value | KPUSHLOCK_LOCK; + + /* Commit the acquisition */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Lock acquired, return */ + return; + } + + /* The lock state changed, retry */ + OldState.Ptr = ActualState; + continue; + } + else + { + /* Prepare the wait block for enqueuing */ + WaitBlock.Flags = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING; + WaitBlock.Previous = NULLPTR; + + /* Assume no list optimization is required initially */ + Optimize = FALSE; + + /* Check if other threads are already queued */ + if(OldState.Value & KPUSHLOCK_WAITING) + { + /* Hook into the head of the existing LIFO list */ + WaitBlock.Last = NULLPTR; + WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); + WaitBlock.ShareCount = 0; + + /* Construct the new pointer containing wait block and preserved flags */ + NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) | + KPUSHLOCK_LOCK | + KPUSHLOCK_WAKING | + KPUSHLOCK_WAITING | + (ULONG_PTR)&WaitBlock; + + /* Check if no one is currently organizing the list */ + if(!(OldState.Value & KPUSHLOCK_WAKING)) + { + /* Take responsibility for optimizing the wait list */ + Optimize = TRUE; + } + } + else + { + /* Mark this block as the tail of the new wait list and preserve the shared reader count */ + WaitBlock.Last = &WaitBlock; + WaitBlock.ShareCount = (ULONG)OldState.Shared; + + /* Check if multiple readers currently hold the lock */ + if(OldState.Shared > 1) + { + /* Inject the wait block and preserve the multiple shared state */ + NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; + } + else + { + /* Clear the share count for a single owner */ + WaitBlock.ShareCount = 0; + + /* Inject the wait block for a single owner scenario */ + NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; + } + } + + /* insert wait block into the PushLock */ + TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(TempState.Ptr != OldState.Ptr) + { + /* Lost the queue race, retry */ + OldState.Ptr = TempState.Ptr; + continue; + } + + /* Check if this thread took the optimization role */ + if(Optimize) + { + /* Convert the LIFO wait list into a FIFO list */ + OptimizePushLockList(PushLock, NewState); + } + + /* Block the thread */ + SpinPushLock(&WaitBlock); + + /* Reset state and retry */ + WaitBlock.ShareCount = 0; + OldState.Ptr = PushLock->Ptr; + } + } +} + +/** + * Enqueues the current thread into the wait list if the lock is held exclusively or if a writer is already waiting. + * + * @param PushLock + * Supplies a pointer to the push lock to be acquired. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::AcquireWaitSharedPushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK OldState, NewState, TempState; + KPUSH_LOCK_WAIT_BLOCK WaitBlock; + PVOID ActualState; + BOOLEAN Optimize; + + /* Initialize a synchronization event */ + KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE); + + /* Capture the snapshot of the push lock pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter contention loop */ + while(TRUE) + { + /* Check if the lock is free */ + if(!(OldState.Value & (KPUSHLOCK_LOCK | KPUSHLOCK_WAITING))) + { + /* Increment the shared reader count */ + NewState.Value = OldState.Value + KPUSHLOCK_INCREMENT_SHARED; + + /* Commit the shared acquisition */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Lock acquired, return */ + return; + } + + /* The lock state changed, retry */ + OldState.Ptr = ActualState; + continue; + } + else + { + /* Prepare the wait block for enqueuing as a reader */ + WaitBlock.Flags = KPUSHLOCK_WAITING; + WaitBlock.Previous = NULLPTR; + + /* Assume no list optimization is required */ + Optimize = FALSE; + + /* Check if other threads are already queued */ + if(OldState.Value & KPUSHLOCK_WAITING) + { + /* Hook into the head of the existing LIFO wait list */ + WaitBlock.Last = NULLPTR; + WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); + WaitBlock.ShareCount = 0; + + /* Construct the new pointer containing wait block and preserved flags */ + NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) | + KPUSHLOCK_LOCK | + KPUSHLOCK_WAITING | + (ULONG_PTR)&WaitBlock; + + /* Check if no one is currently organizing the list */ + if(!(OldState.Value & KPUSHLOCK_WAKING)) + { + /* Take responsibility for optimizing the wait list */ + Optimize = TRUE; + } + } + else + { + /* Mark this block as the tail of the new wait list and preserve the shared reader count */ + WaitBlock.Last = &WaitBlock; + WaitBlock.ShareCount = (ULONG)OldState.Shared; + + /* Check if multiple readers currently hold the lock */ + if(OldState.Shared > 1) + { + /* Inject the wait block and preserve the multiple shared state */ + NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; + } + else + { + /* Inject the wait block for a single owner */ + NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock; + } + } + + /* Insert the wait block into the PushLock */ + TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(TempState.Ptr != OldState.Ptr) + { + /* Lost the queue race, retry */ + OldState.Ptr = TempState.Ptr; + continue; + } + + /* Check if this thread took the optimization role */ + if(Optimize) + { + /* Convert the LIFO wait list into a FIFO list */ + OptimizePushLockList(PushLock, NewState); + } + + /* Block the thread */ + SpinPushLock(&WaitBlock); + + /* Exit loop */ + return; + } + } +} + +/** + * Optimizes the push lock wait list by converting it from a singly-linked LIFO list into a doubly-linked FIFO list. + * + * @param PushLock + * Supplies a pointer to the push lock whose list is being optimized. + * + * @param OldValue + * Supplies the snapshot of the push lock state containing the pointer to the list head. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTINLINE +VOID +KE::PushLock::OptimizePushLockList(IN PKPUSH_LOCK PushLock, + IN KPUSH_LOCK OldValue) +{ + PKPUSH_LOCK_WAIT_BLOCK CurrentBlock, NextBlock; + + /* Extract the pointer to the head of the wait block list */ + CurrentBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~KPUSHLOCK_PTR_BITS); + + /* Ensure the list is not empty */ + if(CurrentBlock) + { + /* Traverse the list */ + while((NextBlock = CurrentBlock->Next)) + { + /* Link the next block back to the current block */ + NextBlock->Previous = CurrentBlock; + CurrentBlock = NextBlock; + } + } +} + +/** + * Releases a push lock that was acquired for exclusive access. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK OldState; + + /* Subtract the LOCK bit */ + OldState.Value = RTL::Atomic::ExchangeAdd64((PLONG_PTR)&PushLock->Value, -(SSIZE_T)KPUSHLOCK_LOCK); + + /* Check if waiters are queued and no one is currently waking them */ + if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) + { + /* Initiate the wake sequence */ + TryWakePushLock(PushLock); + } +} + +/** + * Attempts to release the lock immediately via the fast-path. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::ReleasePushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK NewState, OldState; + + /* Capture the snapshot of the push lock pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Check if the push lock is held by multiple readers */ + if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) + { + /* Decrement the shared reader count */ + NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; + } + else + { + /* Clear the push lock state */ + NewState.Value = 0; + } + + /* Check if there are waiting threads or if the swap fails due to state changes */ + if((OldState.Value & KPUSHLOCK_WAITING) || + (RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr) != OldState.Ptr)) + { + /* Contention or waiters present, execute the slow-path release */ + ReleaseWaitPushLock(PushLock); + } +} + +/** + * Releases a push lock that was acquired for shared access. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTINLINE +XTFASTCALL +VOID +KE::PushLock::ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock) +{ + /* Attempt to clear the lock */ + if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED), NULLPTR) + != (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED)) + { + /* Contention or multiple readers detected, call slow-path */ + ReleaseWaitSharedPushLock(PushLock); + } +} + +/** + * Drops the lock bit and transitions into the waking phase. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK NewState, OldState; + PVOID ActualState; + + /* Capture the snapshot of the push lock pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter release loop */ + while(TRUE) + { + /* Strip the lock bit from the current state */ + NewState.Value = OldState.Value & ~KPUSHLOCK_LOCK; + + /* Check if waiters are queued and no thread is waking them */ + if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) + { + /* Add the waking flag to the state */ + NewState.Value |= KPUSHLOCK_WAKING; + } + + /* Commit the modified state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Verify if acquired the waking role */ + if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING)) + { + /* Execute the wake sequence */ + WakePushLockWaiters(PushLock, NewState); + } + return; + } + + /* State changed, update snapshot and retry */ + OldState.Ptr = ActualState; + } +} + +/** + * Handles decrements under contention and delegates to the wake mechanism if threads are queued + * and the lock becomes fully available. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock) +{ + PKPUSH_LOCK_WAIT_BLOCK WaitBlock; + KPUSH_LOCK NewState, OldState; + PVOID ActualState; + + /* Capture the snapshot of the push lock pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter release loop */ + while(TRUE) + { + /* Check if there are no waiters queued */ + if(!(OldState.Value & KPUSHLOCK_WAITING)) + { + /* Check if the lock is held by multiple readers */ + if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) + { + /* Decrement the shared reader count in the new state */ + NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; + } + else + { + /* Clear the new state */ + NewState.Value = 0; + } + + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Lock released, return */ + return; + } + + /* State changed, retry */ + OldState.Ptr = ActualState; + continue; + } + + /* Check if multiple readers hold the lock while waiters are queued */ + if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) + { + /* Extract the wait block list head */ + WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); + + /* Traverse to find the last wait block */ + while(WaitBlock->Next) + { + /* Check if the last block pointer is already cached */ + if(WaitBlock->Last) + { + /* Jump directly to the cached last block */ + WaitBlock = WaitBlock->Last; + break; + } + + /* Advance to the next wait block */ + WaitBlock = WaitBlock->Next; + } + + /* Check if the last block is a reader block */ + if(WaitBlock->ShareCount > 0) + { + /* Decrement the block's reader share count */ + if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) + { + /* There are still active readers, return */ + return; + } + } + } + + /* Strip the lock and multiple shared flags from the current state */ + NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED); + + /* Check if another thread is already handling the wake process */ + if(OldState.Value & KPUSHLOCK_WAKING) + { + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Lock released, return */ + return; + } + } + else + { + /* Add the waking flag to the new state */ + NewState.Value |= KPUSHLOCK_WAKING; + + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Assumed the waking role, execute wake sequence and return */ + WakePushLockWaiters(PushLock, NewState); + return; + } + } + + /* State changed, update snapshot and retry */ + OldState.Ptr = ActualState; + } +} + +/** + * Handles decrements under contention and delegates to the wake mechanism if needed. + * + * @param PushLock + * Supplies a pointer to the push lock to be released. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock) +{ + PKPUSH_LOCK_WAIT_BLOCK WaitBlock; + KPUSH_LOCK NewState, OldState; + PVOID ActualState; + + /* Capture the snapshot of the pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter release loop */ + while(TRUE) + { + /* Check if there are no waiters currently queued */ + if(!(OldState.Value & KPUSHLOCK_WAITING)) + { + /* Check if the lock is held by multiple readers */ + if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) + { + /* Decrement the shared reader count in the new state */ + NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED; + } + else + { + /* Clear the new state completely */ + NewState.Value = 0; + } + + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) return; + + /* State changed, retry */ + OldState.Ptr = ActualState; + continue; + } + + /* Check if multiple readers hold the lock while waiters are queued */ + if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) + { + /* Extract the wait block list head */ + WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); + + /* Traverse to find the last wait block */ + while(WaitBlock->Next) + { + /* Check if the last block pointer is already cached */ + if(WaitBlock->Last) + { + /* Jump to the cached last block */ + WaitBlock = WaitBlock->Last; + break; + } + + /* Advance to the next wait block */ + WaitBlock = WaitBlock->Next; + } + + /* Check if the last block is a reader block */ + if(WaitBlock->ShareCount > 0) + { + /* Decrement the block's reader share count */ + if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) return; + } + } + + /* Strip the lock and multiple shared flags from the current state */ + NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED); + + /* Check if another thread is already handling the wake process */ + if(OldState.Value & KPUSHLOCK_WAKING) + { + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) return; + } + else + { + /* Add the waking flag to the new state */ + NewState.Value |= KPUSHLOCK_WAKING; + + /* Commit the new state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Execute wake sequence */ + WakePushLockWaiters(PushLock, NewState); + return; + } + } + + /* State changed, update snapshot and retry */ + OldState.Ptr = ActualState; + } +} + +/** + * Handles the blocking phase for a thread waiting on a push lock. + * + * @param WaitBlock + * Supplies a pointer to the thread's local wait block allocated on the stack. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTINLINE +XTFASTCALL +VOID +KE::PushLock::SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock) +{ + ULONG SpinCount; + + /* Initialize the spin count */ + SpinCount = KPUSH_LOCK_SPIN_COUNT; + + /* Enter adaptive Spin loop */ + while(SpinCount--) + { + /* Check if the unlocker cleared our wait flag */ + if(!(WaitBlock->Flags & KPUSHLOCK_WAITING)) + { + /* Wait flag cleared, return */ + return; + } + + /* Yield processor */ + AR::CpuFunctions::YieldProcessor(); + } + + /* Test and clear the WAIT bit */ + if(RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1)) + { + /* Delegate to the Dispatcher */ + KE::Dispatcher::WaitForSingleObject(&WaitBlock->WakeEvent, WrPushLock, KernelMode, FALSE, NULLPTR); + } +} + +/** + * Attempts to acquire a push lock for exclusive access without entering a wait state. + * + * @param PushLock + * Supplies a pointer to the push lock to be acquired. + * + * @return This routine returns TRUE if the exclusive lock was successfully acquired, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTFASTCALL +BOOLEAN +KE::PushLock::TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock) +{ + /* Acquire the lock */ + if(RTL::Atomic::BitTestAndSet((PLONG)&PushLock->Value, KPUSHLOCK_INDEX)) + { + /* Lock not acquired, return FALSE */ + return FALSE; + } + + /* Lock acquired, return TRUE */ + return TRUE; +} + +/** + * Initiates the wake sequence for queued waiters if the lock is free and no other thread is already performing + * the wake operation. + * + * @param PushLock + * Supplies a pointer to the push lock to evaluate. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::TryWakePushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK NewState, OldState; + PVOID ActualState; + + /* Capture the snapshot of the pointer */ + OldState.Ptr = PushLock->Ptr; + + /* Enter evaluation loop */ + while(TRUE) + { + /* Check if already waking, locked, or no one is waiting */ + if((OldState.Value & KPUSHLOCK_WAKING) || + (OldState.Value & KPUSHLOCK_LOCK) || + !(OldState.Value & KPUSHLOCK_WAITING)) + { + /* Abort */ + return; + } + + /* Add the waking flag to the state */ + NewState.Value = OldState.Value | KPUSHLOCK_WAKING; + + /* Commit the waking state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Assumed the waking role, execute wake sequence */ + WakePushLockWaiters(PushLock, NewState); + return; + } + + /* State changed, update snapshot and retry */ + OldState.Ptr = ActualState; + } +} + +/** + * Wakes threads waiting on a PushLock. + * + * @param PushLock + * Supplies a pointer to the push lock being processed. + * + * @param OldState + * Supplies the current snapshot of the push lock state. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState) +{ + PKPUSH_LOCK_WAIT_BLOCK FirstWaitBlock, PreviousWaitBlock, WaitBlock; + KPUSH_LOCK NewState; + PVOID ActualState; + + /* Loop while the push lock is held by another thread */ + while(OldState.Value & KPUSHLOCK_LOCK) + { + /* Strip the waking flag */ + NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING; + + /* Commit the modified state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Lock released, return */ + return; + } + + /* Update the snapshot pointer */ + OldState.Ptr = ActualState; + } + + /* Extract the first wait block pointer */ + FirstWaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS); + WaitBlock = FirstWaitBlock; + + /* Loop through the wait block list */ + while(WaitBlock->Next) + { + /* Check if the last block pointer is cached */ + if(WaitBlock->Last) + { + /* Jump to the cached last block */ + WaitBlock = WaitBlock->Last; + break; + } + + /* Store the current block as the previous block */ + PreviousWaitBlock = WaitBlock; + + /* Advance to the next block and set the back-pointer to the previous block */ + WaitBlock = WaitBlock->Next; + WaitBlock->Previous = PreviousWaitBlock; + } + + /* Point WaitBlock to the oldest waiter */ + PreviousWaitBlock = WaitBlock->Previous; + + /* Check if the target is a reader block */ + if(!(WaitBlock->Flags & KPUSHLOCK_LOCK) || (PreviousWaitBlock == NULLPTR)) + { + /* Enter clearing loop */ + while(TRUE) + { + /* Clear the new state */ + NewState.Value = 0; + + /* Commit the cleared state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Cleared the lock state, break the loop */ + break; + } + + /* Update the snapshot pointer */ + OldState.Ptr = ActualState; + } + } + else + { + /* Update the cached last block pointer and unlink the target block */ + FirstWaitBlock->Last = PreviousWaitBlock; + WaitBlock->Previous = NULLPTR; + + /* Enter clearing loop */ + while(TRUE) + { + /* Strip the waking flag from the current state */ + NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING; + + /* Commit the modified state */ + ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr); + if(ActualState == OldState.Ptr) + { + /* Cleared the waking flag, break the loop */ + break; + } + + /* Update the snapshot pointer */ + OldState.Ptr = ActualState; + } + } + + /* Raise runlevel to DISPATCH level if there are multiple blocks */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL, WaitBlock->Previous != NULLPTR); + + /* Traverse the reversed list */ + while(WaitBlock) + { + /* Save previous wait block */ + PreviousWaitBlock = WaitBlock->Previous; + + /* Clear the WAIT flag */ + if(!RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1)) + { + /* Thread is fully asleep, Signal its event and boost priority */ + KE::Event::SetEventBoostPriority(&WaitBlock->WakeEvent, NULLPTR); + } + + /* Go to the next block */ + WaitBlock = PreviousWaitBlock; + } +} + +/** + * Synchronizes the current thread with the PushLock. + * + * @param PushLock + * Supplies a pointer to the push lock to wait on. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTFASTCALL +VOID +KE::PushLock::WaitOnPushLock(IN PKPUSH_LOCK PushLock) +{ + KPUSH_LOCK State; + + /* Capture the snapshot of the pointer */ + State.Ptr = PushLock->Ptr; + + /* Verify if the lock is actively held by any thread */ + if(State.Value & KPUSHLOCK_LOCK) + { + /* Force serialization by acquiring the lock */ + AcquireExclusivePushLock(PushLock); + + /* Immediately release the lock */ + ReleaseExclusivePushLock(PushLock); + } +}