Implement kernel push lock infrastructure

sdk/xtdk/ketypes.h

@@ -47,8 +47,20 @@
 /* APC pending state length */
 #define KAPC_STATE_LENGTH                           (FIELD_OFFSET(KAPC_STATE, UserApcPending) + sizeof(BOOLEAN))
 
-/* Push lock total bits */
+/* Indices used to access the PushLock data structure */
+#define KPUSHLOCK_INDEX                             ((ULONG_PTR)0x00)
+#define KPUSHLOCK_LOCK                              ((ULONG_PTR)0x01)
+#define KPUSHLOCK_WAITING                           ((ULONG_PTR)0x02)
+#define KPUSHLOCK_WAKING                            ((ULONG_PTR)0x04)
+#define KPUSHLOCK_MULTIPLE_SHARED                   ((ULONG_PTR)0x08)
+#define KPUSHLOCK_INCREMENT_SHARED                  ((ULONG_PTR)0x10)
+
+/* Mask for the pointer bits */
+#define KPUSHLOCK_PTR_BITS                          ((ULONG_PTR)0x0F)
+
+/* PushLock related definitions */
 #define KPUSH_LOCK_TOTAL_BITS                       (sizeof(ULONG_PTR) * 8)
+#define KPUSH_LOCK_SPIN_COUNT                       1024
 
 /* Kernel service descriptor tables count */
 #define KSERVICE_TABLES_COUNT                       4

xtoskrnl/CMakeLists.txt

@@ -58,6 +58,7 @@ list(APPEND XTOSKRNL_SOURCE
     ${XTOSKRNL_SOURCE_DIR}/ke/krnlinit.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/kthread.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/kubsan.cc
+    ${XTOSKRNL_SOURCE_DIR}/ke/pushlock.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/runlevel.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/schedule.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/semphore.cc

xtoskrnl/includes/ke.hh

@@ -26,6 +26,7 @@
 #include <ke/kthread.hh>
 #include <ke/kubsan.hh>
 #include <ke/proc.hh>
+#include <ke/pushlock.hh>
 #include <ke/runlevel.hh>
 #include <ke/schedule.hh>
 #include <ke/semphore.hh>

xtoskrnl/includes/ke/pushlock.hh

@@ -0,0 +1,42 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/ke/pushlock.hh
+ * DESCRIPTION:     XT Kernel Push Lock support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_KE_PUSHLOCK_HH
+#define __XTOSKRNL_KE_PUSHLOCK_HH
+
+#include <xtos.hh>
+
+
+/* Kernel Library */
+namespace KE
+{
+    class PushLock
+    {
+        public:
+            STATIC XTFASTCALL VOID AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID AcquireSharedPushLock(PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID AcquireWaitSharedPushLock(IN OUT PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleasePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL BOOLEAN TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID TryWakePushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID WaitOnPushLock(IN PKPUSH_LOCK PushLock);
+            STATIC XTFASTCALL VOID WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState);
+
+        private:
+            STATIC XTINLINE VOID OptimizePushLockList(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldValue);
+            STATIC XTINLINE XTFASTCALL VOID SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock);
+    };
+}
+
+#endif /* __XTOSKRNL_KE_PUSHLOCK_HH */

xtoskrnl/ke/pushlock.cc

@@ -0,0 +1,984 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/ke/pushlock.cc
+ * DESCRIPTION:     XT Kernel Push Lock support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Attempts to acquire a push lock for exclusive access via the fast-path.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be acquired.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::AcquireExclusivePushLock(IN PKPUSH_LOCK PushLock)
+{
+    /* Attempt to swap a NULL pointer with the EXCLUSIVE flag */
+    if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)KPUSHLOCK_LOCK))
+    {
+        /* Lock contention detected, enter the slow-path */
+        AcquireWaitExclusivePushLock(PushLock);
+    }
+}
+
+/**
+ * Attempts to acquire a push lock for shared access via the fast-path.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be acquired.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::AcquireSharedPushLock(PKPUSH_LOCK PushLock)
+{
+    /* Attempt to set the reader flag */
+    if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, NULLPTR, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED)))
+    {
+        /* Lock contention or existing readers detected, Enter the slow-path */
+        AcquireWaitSharedPushLock(PushLock);
+    }
+}
+
+/**
+ * Enqueues the current thread's wait block into the lock's wait list, conditionally assumes responsibility
+ * for list optimization and transitions the thread into a wait state until exclusive ownership is acquired.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be acquired.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::AcquireWaitExclusivePushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK OldState, NewState, TempState;
+    KPUSH_LOCK_WAIT_BLOCK WaitBlock;
+    PVOID ActualState;
+    BOOLEAN Optimize;
+
+    /* Initialize a synchronization event */
+    KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE);
+
+    /* Capture the snapshot of the push lock pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter contention loop */
+    while(TRUE)
+    {
+        /* Check if the lock is free */
+        if(!(OldState.Value & KPUSHLOCK_LOCK))
+        {
+            /* Assert the LOCK bit */
+            NewState.Value = OldState.Value | KPUSHLOCK_LOCK;
+
+            /* Commit the acquisition */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Lock acquired, return */
+                return;
+            }
+
+            /* The lock state changed, retry */
+            OldState.Ptr = ActualState;
+            continue;
+        }
+        else
+        {
+            /* Prepare the wait block for enqueuing */
+            WaitBlock.Flags = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING;
+            WaitBlock.Previous = NULLPTR;
+
+            /* Assume no list optimization is required initially */
+            Optimize = FALSE;
+
+            /* Check if other threads are already queued */
+            if(OldState.Value & KPUSHLOCK_WAITING)
+            {
+                /* Hook into the head of the existing LIFO list */
+                WaitBlock.Last = NULLPTR;
+                WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
+                WaitBlock.ShareCount = 0;
+
+                /* Construct the new pointer containing wait block and preserved flags */
+                NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) |
+                                 KPUSHLOCK_LOCK |
+                                 KPUSHLOCK_WAKING |
+                                 KPUSHLOCK_WAITING |
+                                 (ULONG_PTR)&WaitBlock;
+
+                /* Check if no one is currently organizing the list */
+                if(!(OldState.Value & KPUSHLOCK_WAKING))
+                {
+                    /* Take responsibility for optimizing the wait list */
+                    Optimize = TRUE;
+                }
+            }
+            else
+            {
+                /* Mark this block as the tail of the new wait list and preserve the shared reader count */
+                WaitBlock.Last = &WaitBlock;
+                WaitBlock.ShareCount = (ULONG)OldState.Shared;
+
+                /* Check if multiple readers currently hold the lock */
+                if(OldState.Shared > 1)
+                {
+                    /* Inject the wait block and preserve the multiple shared state */
+                    NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
+                }
+                else
+                {
+                    /* Clear the share count for a single owner */
+                    WaitBlock.ShareCount = 0;
+
+                    /* Inject the wait block for a single owner scenario */
+                    NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
+                }
+            }
+
+            /* insert wait block into the PushLock */
+            TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(TempState.Ptr != OldState.Ptr)
+            {
+                /* Lost the queue race, retry */
+                OldState.Ptr = TempState.Ptr;
+                continue;
+            }
+
+            /* Check if this thread took the optimization role */
+            if(Optimize)
+            {
+                /* Convert the LIFO wait list into a FIFO list */
+                OptimizePushLockList(PushLock, NewState);
+            }
+
+            /* Block the thread */
+            SpinPushLock(&WaitBlock);
+
+            /* Reset state and retry */
+            WaitBlock.ShareCount = 0;
+            OldState.Ptr = PushLock->Ptr;
+        }
+    }
+}
+
+/**
+ * Enqueues the current thread into the wait list if the lock is held exclusively or if a writer is already waiting.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be acquired.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::AcquireWaitSharedPushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK OldState, NewState, TempState;
+    KPUSH_LOCK_WAIT_BLOCK WaitBlock;
+    PVOID ActualState;
+    BOOLEAN Optimize;
+
+    /* Initialize a synchronization event */
+    KE::Event::InitializeEvent(&WaitBlock.WakeEvent, SynchronizationEvent, FALSE);
+
+    /* Capture the snapshot of the push lock pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter contention loop */
+    while(TRUE)
+    {
+        /* Check if the lock is free */
+        if(!(OldState.Value & (KPUSHLOCK_LOCK | KPUSHLOCK_WAITING)))
+        {
+            /* Increment the shared reader count */
+            NewState.Value = OldState.Value + KPUSHLOCK_INCREMENT_SHARED;
+
+            /* Commit the shared acquisition */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Lock acquired, return */
+                return;
+            }
+
+            /* The lock state changed, retry */
+            OldState.Ptr = ActualState;
+            continue;
+        }
+        else
+        {
+            /* Prepare the wait block for enqueuing as a reader */
+            WaitBlock.Flags = KPUSHLOCK_WAITING;
+            WaitBlock.Previous = NULLPTR;
+
+            /* Assume no list optimization is required */
+            Optimize = FALSE;
+
+            /* Check if other threads are already queued */
+            if(OldState.Value & KPUSHLOCK_WAITING)
+            {
+                /* Hook into the head of the existing LIFO wait list */
+                WaitBlock.Last = NULLPTR;
+                WaitBlock.Next = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
+                WaitBlock.ShareCount = 0;
+
+                /* Construct the new pointer containing wait block and preserved flags */
+                NewState.Value = (OldState.Value & KPUSHLOCK_MULTIPLE_SHARED) |
+                                 KPUSHLOCK_LOCK |
+                                 KPUSHLOCK_WAITING |
+                                 (ULONG_PTR)&WaitBlock;
+
+                /* Check if no one is currently organizing the list */
+                if(!(OldState.Value & KPUSHLOCK_WAKING))
+                {
+                    /* Take responsibility for optimizing the wait list */
+                    Optimize = TRUE;
+                }
+            }
+            else
+            {
+                /* Mark this block as the tail of the new wait list and preserve the shared reader count */
+                WaitBlock.Last = &WaitBlock;
+                WaitBlock.ShareCount = (ULONG)OldState.Shared;
+
+                /* Check if multiple readers currently hold the lock */
+                if(OldState.Shared > 1)
+                {
+                    /* Inject the wait block and preserve the multiple shared state */
+                    NewState.Value = KPUSHLOCK_MULTIPLE_SHARED | KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
+                }
+                else
+                {
+                    /* Inject the wait block for a single owner */
+                    NewState.Value = KPUSHLOCK_LOCK | KPUSHLOCK_WAITING | (ULONG_PTR)&WaitBlock;
+                }
+            }
+
+            /* Insert the wait block into the PushLock */
+            TempState.Ptr = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(TempState.Ptr != OldState.Ptr)
+            {
+                /* Lost the queue race, retry */
+                OldState.Ptr = TempState.Ptr;
+                continue;
+            }
+
+            /* Check if this thread took the optimization role */
+            if(Optimize)
+            {
+                /* Convert the LIFO wait list into a FIFO list */
+                OptimizePushLockList(PushLock, NewState);
+            }
+
+            /* Block the thread */
+            SpinPushLock(&WaitBlock);
+
+            /* Exit loop */
+            return;
+        }
+    }
+}
+
+/**
+ * Optimizes the push lock wait list by converting it from a singly-linked LIFO list into a doubly-linked FIFO list.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock whose list is being optimized.
+ *
+ * @param OldValue
+ *        Supplies the snapshot of the push lock state containing the pointer to the list head.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTINLINE
+VOID
+KE::PushLock::OptimizePushLockList(IN PKPUSH_LOCK PushLock,
+                                   IN KPUSH_LOCK OldValue)
+{
+    PKPUSH_LOCK_WAIT_BLOCK CurrentBlock, NextBlock;
+
+    /* Extract the pointer to the head of the wait block list */
+    CurrentBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~KPUSHLOCK_PTR_BITS);
+
+    /* Ensure the list is not empty */
+    if(CurrentBlock)
+    {
+        /* Traverse the list */
+        while((NextBlock = CurrentBlock->Next))
+        {
+            /* Link the next block back to the current block */
+            NextBlock->Previous = CurrentBlock;
+            CurrentBlock = NextBlock;
+        }
+    }
+}
+
+/**
+ * Releases a push lock that was acquired for exclusive access.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::ReleaseExclusivePushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK OldState;
+
+    /* Subtract the LOCK bit */
+    OldState.Value = RTL::Atomic::ExchangeAdd64((PLONG_PTR)&PushLock->Value, -(SSIZE_T)KPUSHLOCK_LOCK);
+
+    /* Check if waiters are queued and no one is currently waking them */
+    if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
+    {
+        /* Initiate the wake sequence */
+        TryWakePushLock(PushLock);
+    }
+}
+
+/**
+ * Attempts to release the lock immediately via the fast-path.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::ReleasePushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK NewState, OldState;
+
+    /* Capture the snapshot of the push lock pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Check if the push lock is held by multiple readers */
+    if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
+    {
+        /* Decrement the shared reader count */
+        NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
+    }
+    else
+    {
+        /* Clear the push lock state */
+        NewState.Value = 0;
+    }
+
+    /* Check if there are waiting threads or if the swap fails due to state changes */
+    if((OldState.Value & KPUSHLOCK_WAITING) ||
+       (RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr) != OldState.Ptr))
+    {
+        /* Contention or waiters present, execute the slow-path release */
+        ReleaseWaitPushLock(PushLock);
+    }
+}
+
+/**
+ * Releases a push lock that was acquired for shared access.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTINLINE
+XTFASTCALL
+VOID
+KE::PushLock::ReleaseSharedPushLock(IN PKPUSH_LOCK PushLock)
+{
+    /* Attempt to clear the lock */
+    if(RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED), NULLPTR)
+                                           != (PVOID)(KPUSHLOCK_LOCK | KPUSHLOCK_INCREMENT_SHARED))
+    {
+        /* Contention or multiple readers detected, call slow-path */
+        ReleaseWaitSharedPushLock(PushLock);
+    }
+}
+
+/**
+ * Drops the lock bit and transitions into the waking phase.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::ReleaseWaitExclusivePushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK NewState, OldState;
+    PVOID ActualState;
+
+    /* Capture the snapshot of the push lock pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter release loop */
+    while(TRUE)
+    {
+        /* Strip the lock bit from the current state */
+        NewState.Value = OldState.Value & ~KPUSHLOCK_LOCK;
+
+        /* Check if waiters are queued and no thread is waking them */
+        if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
+        {
+            /* Add the waking flag to the state */
+            NewState.Value |= KPUSHLOCK_WAKING;
+        }
+
+        /* Commit the modified state */
+        ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+        if(ActualState == OldState.Ptr)
+        {
+            /* Verify if acquired the waking role */
+            if((OldState.Value & KPUSHLOCK_WAITING) && !(OldState.Value & KPUSHLOCK_WAKING))
+            {
+                /* Execute the wake sequence */
+                WakePushLockWaiters(PushLock, NewState);
+            }
+            return;
+        }
+
+        /* State changed, update snapshot and retry */
+        OldState.Ptr = ActualState;
+    }
+}
+
+/**
+ * Handles decrements under contention and delegates to the wake mechanism if threads are queued
+ * and the lock becomes fully available.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::ReleaseWaitPushLock(IN PKPUSH_LOCK PushLock)
+{
+    PKPUSH_LOCK_WAIT_BLOCK WaitBlock;
+    KPUSH_LOCK NewState, OldState;
+    PVOID ActualState;
+
+    /* Capture the snapshot of the push lock pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter release loop */
+    while(TRUE)
+    {
+        /* Check if there are no waiters queued */
+        if(!(OldState.Value & KPUSHLOCK_WAITING))
+        {
+            /* Check if the lock is held by multiple readers */
+            if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
+            {
+                /* Decrement the shared reader count in the new state */
+                NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
+            }
+            else
+            {
+                /* Clear the new state */
+                NewState.Value = 0;
+            }
+
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Lock released, return */
+                return;
+            }
+
+            /* State changed, retry */
+            OldState.Ptr = ActualState;
+            continue;
+        }
+
+        /* Check if multiple readers hold the lock while waiters are queued */
+        if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
+        {
+            /* Extract the wait block list head */
+            WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
+
+            /* Traverse to find the last wait block */
+            while(WaitBlock->Next)
+            {
+                /* Check if the last block pointer is already cached */
+                if(WaitBlock->Last)
+                {
+                    /* Jump directly to the cached last block */
+                    WaitBlock = WaitBlock->Last;
+                    break;
+                }
+
+                /* Advance to the next wait block */
+                WaitBlock = WaitBlock->Next;
+            }
+
+            /* Check if the last block is a reader block */
+            if(WaitBlock->ShareCount > 0)
+            {
+                /* Decrement the block's reader share count */
+                if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0)
+                {
+                    /* There are still active readers, return */
+                    return;
+                }
+            }
+        }
+
+        /* Strip the lock and multiple shared flags from the current state */
+        NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED);
+
+        /* Check if another thread is already handling the wake process */
+        if(OldState.Value & KPUSHLOCK_WAKING)
+        {
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Lock released, return */
+                return;
+            }
+        }
+        else
+        {
+            /* Add the waking flag to the new state */
+            NewState.Value |= KPUSHLOCK_WAKING;
+
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Assumed the waking role, execute wake sequence and return */
+                WakePushLockWaiters(PushLock, NewState);
+                return;
+            }
+        }
+
+        /* State changed, update snapshot and retry */
+        OldState.Ptr = ActualState;
+    }
+}
+
+/**
+ * Handles decrements under contention and delegates to the wake mechanism if needed.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be released.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::ReleaseWaitSharedPushLock(IN PKPUSH_LOCK PushLock)
+{
+    PKPUSH_LOCK_WAIT_BLOCK WaitBlock;
+    KPUSH_LOCK NewState, OldState;
+    PVOID ActualState;
+
+    /* Capture the snapshot of the pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter release loop */
+    while(TRUE)
+    {
+        /* Check if there are no waiters currently queued */
+        if(!(OldState.Value & KPUSHLOCK_WAITING))
+        {
+            /* Check if the lock is held by multiple readers */
+            if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
+            {
+                /* Decrement the shared reader count in the new state */
+                NewState.Value = OldState.Value - KPUSHLOCK_INCREMENT_SHARED;
+            }
+            else
+            {
+                /* Clear the new state completely */
+                NewState.Value = 0;
+            }
+
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr) return;
+
+            /* State changed, retry */
+            OldState.Ptr = ActualState;
+            continue;
+        }
+
+        /* Check if multiple readers hold the lock while waiters are queued */
+        if(OldState.Value & KPUSHLOCK_MULTIPLE_SHARED)
+        {
+            /* Extract the wait block list head */
+            WaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
+
+            /* Traverse to find the last wait block */
+            while(WaitBlock->Next)
+            {
+                /* Check if the last block pointer is already cached */
+                if(WaitBlock->Last)
+                {
+                    /* Jump to the cached last block */
+                    WaitBlock = WaitBlock->Last;
+                    break;
+                }
+
+                /* Advance to the next wait block */
+                WaitBlock = WaitBlock->Next;
+            }
+
+            /* Check if the last block is a reader block */
+            if(WaitBlock->ShareCount > 0)
+            {
+                /* Decrement the block's reader share count */
+                if(RTL::Atomic::Decrement32((PLONG)&WaitBlock->ShareCount) > 0) return;
+            }
+        }
+
+        /* Strip the lock and multiple shared flags from the current state */
+        NewState.Value = OldState.Value & ~(KPUSHLOCK_LOCK | KPUSHLOCK_MULTIPLE_SHARED);
+
+        /* Check if another thread is already handling the wake process */
+        if(OldState.Value & KPUSHLOCK_WAKING)
+        {
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr) return;
+        }
+        else
+        {
+            /* Add the waking flag to the new state */
+            NewState.Value |= KPUSHLOCK_WAKING;
+
+            /* Commit the new state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Execute wake sequence */
+                WakePushLockWaiters(PushLock, NewState);
+                return;
+            }
+        }
+
+        /* State changed, update snapshot and retry */
+        OldState.Ptr = ActualState;
+    }
+}
+
+/**
+ * Handles the blocking phase for a thread waiting on a push lock.
+ *
+ * @param WaitBlock
+ *        Supplies a pointer to the thread's local wait block allocated on the stack.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTINLINE
+XTFASTCALL
+VOID
+KE::PushLock::SpinPushLock(IN PKPUSH_LOCK_WAIT_BLOCK WaitBlock)
+{
+    ULONG SpinCount;
+
+    /* Initialize the spin count */
+    SpinCount = KPUSH_LOCK_SPIN_COUNT;
+
+    /* Enter adaptive Spin loop */
+    while(SpinCount--)
+    {
+        /* Check if the unlocker cleared our wait flag */
+        if(!(WaitBlock->Flags & KPUSHLOCK_WAITING))
+        {
+            /* Wait flag cleared, return */
+            return;
+        }
+
+        /* Yield processor */
+        AR::CpuFunctions::YieldProcessor();
+    }
+
+    /* Test and clear the WAIT bit */
+    if(RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1))
+    {
+        /* Delegate to the Dispatcher */
+        KE::Dispatcher::WaitForSingleObject(&WaitBlock->WakeEvent, WrPushLock, KernelMode, FALSE, NULLPTR);
+    }
+}
+
+/**
+ * Attempts to acquire a push lock for exclusive access without entering a wait state.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to be acquired.
+ *
+ * @return This routine returns TRUE if the exclusive lock was successfully acquired, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+BOOLEAN
+KE::PushLock::TryAcquireExclusivePushLock(PKPUSH_LOCK PushLock)
+{
+    /* Acquire the lock */
+    if(RTL::Atomic::BitTestAndSet((PLONG)&PushLock->Value, KPUSHLOCK_INDEX))
+    {
+        /* Lock not acquired, return FALSE */
+        return FALSE;
+    }
+
+    /* Lock acquired, return TRUE */
+    return TRUE;
+}
+
+/**
+ * Initiates the wake sequence for queued waiters if the lock is free and no other thread is already performing
+ * the wake operation.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to evaluate.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::TryWakePushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK NewState, OldState;
+    PVOID ActualState;
+
+    /* Capture the snapshot of the pointer */
+    OldState.Ptr = PushLock->Ptr;
+
+    /* Enter evaluation loop */
+    while(TRUE)
+    {
+        /* Check if already waking, locked, or no one is waiting */
+        if((OldState.Value & KPUSHLOCK_WAKING) ||
+           (OldState.Value & KPUSHLOCK_LOCK) ||
+           !(OldState.Value & KPUSHLOCK_WAITING))
+        {
+            /* Abort */
+            return;
+        }
+
+        /* Add the waking flag to the state */
+        NewState.Value = OldState.Value | KPUSHLOCK_WAKING;
+
+        /* Commit the waking state */
+        ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+        if(ActualState == OldState.Ptr)
+        {
+            /* Assumed the waking role, execute wake sequence */
+            WakePushLockWaiters(PushLock, NewState);
+            return;
+        }
+
+        /* State changed, update snapshot and retry */
+        OldState.Ptr = ActualState;
+    }
+}
+
+/**
+ * Wakes threads waiting on a PushLock.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock being processed.
+ *
+ * @param OldState
+ *        Supplies the current snapshot of the push lock state.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::WakePushLockWaiters(IN PKPUSH_LOCK PushLock, IN KPUSH_LOCK OldState)
+{
+    PKPUSH_LOCK_WAIT_BLOCK FirstWaitBlock, PreviousWaitBlock, WaitBlock;
+    KPUSH_LOCK NewState;
+    PVOID ActualState;
+
+    /* Loop while the push lock is held by another thread */
+    while(OldState.Value & KPUSHLOCK_LOCK)
+    {
+        /* Strip the waking flag */
+        NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING;
+
+        /* Commit the modified state */
+        ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+        if(ActualState == OldState.Ptr)
+        {
+            /* Lock released, return */
+            return;
+        }
+
+        /* Update the snapshot pointer */
+        OldState.Ptr = ActualState;
+    }
+
+    /* Extract the first wait block pointer */
+    FirstWaitBlock = (PKPUSH_LOCK_WAIT_BLOCK)(OldState.Value & ~KPUSHLOCK_PTR_BITS);
+    WaitBlock = FirstWaitBlock;
+
+    /* Loop through the wait block list */
+    while(WaitBlock->Next)
+    {
+        /* Check if the last block pointer is cached */
+        if(WaitBlock->Last)
+        {
+            /* Jump to the cached last block */
+            WaitBlock = WaitBlock->Last;
+            break;
+        }
+
+        /* Store the current block as the previous block */
+        PreviousWaitBlock = WaitBlock;
+
+        /* Advance to the next block and set the back-pointer to the previous block */
+        WaitBlock = WaitBlock->Next;
+        WaitBlock->Previous = PreviousWaitBlock;
+    }
+
+    /* Point WaitBlock to the oldest waiter */
+    PreviousWaitBlock = WaitBlock->Previous;
+
+    /* Check if the target is a reader block */
+    if(!(WaitBlock->Flags & KPUSHLOCK_LOCK) || (PreviousWaitBlock == NULLPTR))
+    {
+        /* Enter clearing loop */
+        while(TRUE)
+        {
+            /* Clear the new state */
+            NewState.Value = 0;
+
+            /* Commit the cleared state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Cleared the lock state, break the loop */
+                break;
+            }
+
+            /* Update the snapshot pointer */
+            OldState.Ptr = ActualState;
+        }
+    }
+    else
+    {
+        /* Update the cached last block pointer and unlink the target block */
+        FirstWaitBlock->Last = PreviousWaitBlock;
+        WaitBlock->Previous = NULLPTR;
+
+        /* Enter clearing loop */
+        while(TRUE)
+        {
+            /* Strip the waking flag from the current state */
+            NewState.Value = OldState.Value & ~KPUSHLOCK_WAKING;
+
+            /* Commit the modified state */
+            ActualState = RTL::Atomic::CompareExchangePointer(&PushLock->Ptr, OldState.Ptr, NewState.Ptr);
+            if(ActualState == OldState.Ptr)
+            {
+                /* Cleared the waking flag, break the loop */
+                break;
+            }
+
+            /* Update the snapshot pointer */
+            OldState.Ptr = ActualState;
+        }
+    }
+
+    /* Raise runlevel to DISPATCH level if there are multiple blocks */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL, WaitBlock->Previous != NULLPTR);
+
+    /* Traverse the reversed list */
+    while(WaitBlock)
+    {
+        /* Save previous wait block */
+        PreviousWaitBlock = WaitBlock->Previous;
+
+        /* Clear the WAIT flag */
+        if(!RTL::Atomic::BitTestAndReset((PLONG)&WaitBlock->Flags, 1))
+        {
+            /* Thread is fully asleep, Signal its event and boost priority */
+            KE::Event::SetEventBoostPriority(&WaitBlock->WakeEvent, NULLPTR);
+        }
+
+        /* Go to the next block */
+        WaitBlock = PreviousWaitBlock;
+    }
+}
+
+/**
+ * Synchronizes the current thread with the PushLock.
+ *
+ * @param PushLock
+ *        Supplies a pointer to the push lock to wait on.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTFASTCALL
+VOID
+KE::PushLock::WaitOnPushLock(IN PKPUSH_LOCK PushLock)
+{
+    KPUSH_LOCK State;
+
+    /* Capture the snapshot of the pointer */
+    State.Ptr = PushLock->Ptr;
+
+    /* Verify if the lock is actively held by any thread */
+    if(State.Value & KPUSHLOCK_LOCK)
+    {
+        /* Force serialization by acquiring the lock */
+        AcquireExclusivePushLock(PushLock);
+
+        /* Immediately release the lock */
+        ReleaseExclusivePushLock(PushLock);
+    }
+}