Add per-processor lookaside list cache management functions

xtoskrnl/ex/laslist.cc

@@ -9,6 +9,190 @@
 #include <xtos.hh>
 
 
+/**
+ * Attempts to allocate a fixed-size buffer from the specified non-paged lookaside list.
+ *
+ * @param LookasideList
+ *        Supplies a pointer to the non-paged lookaside list.
+ *
+ * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+EX::LookasideList::AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList)
+{
+    PVOID Buffer;
+
+    /* Increment the total number of allocation requests */
+    LookasideList->Global.TotalAllocates++;
+
+    /* Attempt to pop an entry from the lookaside list */
+    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
+    if(!Buffer)
+    {
+        /* Increment the miss counter */
+        LookasideList->Global.AllocateMisses++;
+
+        /* Fallback to standard pool allocator */
+        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
+                                         &Buffer, LookasideList->Global.Tag);
+    }
+
+    /* Return the allocated buffer */
+    return Buffer;
+}
+
+/**
+ * Attempts to allocate a fixed-size buffer from the specified paged lookaside list.
+ *
+ * @param LookasideList
+ *        Supplies a pointer to the paged lookaside list.
+ *
+ * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+EX::LookasideList::AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList)
+{
+    PVOID Buffer;
+
+    /* Increment the total number of allocation requests */
+    LookasideList->Global.TotalAllocates++;
+
+    /* Attempt to pop an entry from the lookaside list */
+    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
+    if(!Buffer)
+    {
+        /* Increment the miss counter */
+        LookasideList->Global.AllocateMisses++;
+
+        /* Fallback to standard pool allocator */
+        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
+                                         &Buffer, LookasideList->Global.Tag);
+    }
+
+    /* Return the allocated buffer */
+    return Buffer;
+}
+
+
+/**
+ * Allocates a memory block from the specified per-processor lookaside list.
+ *
+ * @param Number
+ *        Supplies the index of the lookaside list to allocate from.
+ *
+ * @return This routine returns a pointer to the allocated memory block, or NULLPTR if allocation fails.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+EX::LookasideList::AllocateFromPerProcessorList(IN NONPAGED_LOOKASIDE_NUMBER Number)
+{
+    PKPROCESSOR_CONTROL_BLOCK Prcb;
+    PGENERAL_LOOKASIDE Lookaside;
+    PVOID Entry;
+
+    /* Retrieve the current Processor Control Block */
+    Prcb = KE::Processor::GetCurrentProcessorControlBlock();
+
+    /* Target the processor's local lookaside list and increment the tracking metric */
+    Lookaside = Prcb->LookasideList[Number].Local;
+    Lookaside->TotalAllocates++;
+
+    /* Attempt a pop from the local list */
+    Entry = RTL::Atomic::PopEntrySingleList(&Lookaside->ListHead);
+    if(Entry != NULLPTR)
+    {
+        /* Return the retrieved a pre-allocated block */
+        return Entry;
+    }
+
+    /* Record a local cache miss */
+    Lookaside->AllocateMisses++;
+    Lookaside->TotalAllocates++;
+
+    /* Switch the target to the shared global lookaside list */
+    Lookaside = Prcb->LookasideList[Number].Global;
+
+    /* Attempt a pop from the global list */
+    Entry = RTL::Atomic::PopEntrySingleList(&Lookaside->ListHead);
+    if(Entry != NULLPTR)
+    {
+        /* Return the retrieved a pre-allocated block */
+        return Entry;
+    }
+
+    /* Both caches are saturated, record a miss */
+    Lookaside->AllocateMisses++;
+
+    /* Invoke the pool allocator */
+    (Lookaside->Allocate)(Lookaside->Type, Lookaside->Size, &Entry, Lookaside->Tag);
+
+    /* Return the newly allocated block */
+    return Entry;
+}
+
+
+/**
+ * Frees a memory block back to the specified per-processor lookaside list.
+ *
+ * @param Number
+ *        Supplies the index of the lookaside list receiving the freed block.
+ *
+ * @param Entry
+ *        Supplies a pointer to the memory block being freed.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+EX::LookasideList::FreeToPerProcessorList(IN NONPAGED_LOOKASIDE_NUMBER Number,
+                                          IN PVOID Entry)
+{
+    PKPROCESSOR_CONTROL_BLOCK Prcb;
+    PGENERAL_LOOKASIDE Lookaside;
+
+    /* Retrieve the current Processor Control Block */
+    Prcb = KE::Processor::GetCurrentProcessorControlBlock();
+
+    /* Target the processor's local lookaside list and increment the tracking metric */
+    Lookaside = Prcb->LookasideList[Number].Local;
+    Lookaside->TotalFrees += 1;
+
+    /* Check if the lookaside list has reached its maximum capacity threshold */
+    if(RTL::SinglyList::QueryListDepth(&Lookaside->ListHead) >= Lookaside->Depth)
+    {
+        /* The local list is full, record a capacity miss */
+        Lookaside->FreeMisses++;
+        Lookaside->TotalFrees++;
+
+        /* Switch the target to the shared global lookaside list */
+        Lookaside = Prcb->LookasideList[Number].Global;
+
+        /* Check if the lookaside list has reached its maximum capacity threshold */
+        if(RTL::SinglyList::QueryListDepth(&Lookaside->ListHead) >= Lookaside->Depth)
+        {
+            /* Both caches are saturated, record a miss */
+            Lookaside->FreeMisses++;
+            (Lookaside->Free)(Entry);
+
+            /* Exit the routine */
+            return;
+        }
+    }
+
+    /* Push the block onto the selected lookaside list */
+    RTL::Atomic::PushEntrySingleList(&Lookaside->ListHead, (PSINGLE_LIST_ENTRY)Entry);
+}
+
 /**
  * Initializes a generic lookaside list and inserts it into the tracking list.
  *
@@ -141,73 +325,3 @@ EX::LookasideList::InitializeSystemLookasideLists(VOID)
                                 TAG_MM_MEMORY_POOL, 256, &PoolLookasideListHead);
     }
 }
-
-/**
- * Attempts to allocate a fixed-size buffer from the specified non-paged lookaside list.
- *
- * @param LookasideList
- *        Supplies a pointer to the non-paged lookaside list.
- *
- * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
- *
- * @since XT 1.0
- */
-XTAPI
-PVOID
-EX::LookasideList::AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList)
-{
-    PVOID Buffer;
-
-    /* Increment the total number of allocation requests */
-    LookasideList->Global.TotalAllocates++;
-
-    /* Attempt to pop an entry from the lookaside list */
-    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
-    if(!Buffer)
-    {
-        /* Increment the miss counter */
-        LookasideList->Global.AllocateMisses++;
-
-        /* Fallback to standard pool allocator */
-        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
-                                         &Buffer, LookasideList->Global.Tag);
-    }
-
-    /* Return the allocated buffer */
-    return Buffer;
-}
-
-/**
- * Attempts to allocate a fixed-size buffer from the specified paged lookaside list.
- *
- * @param LookasideList
- *        Supplies a pointer to the paged lookaside list.
- *
- * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
- *
- * @since XT 1.0
- */
-XTAPI
-PVOID
-EX::LookasideList::AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList)
-{
-    PVOID Buffer;
-
-    /* Increment the total number of allocation requests */
-    LookasideList->Global.TotalAllocates++;
-
-    /* Attempt to pop an entry from the lookaside list */
-    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
-    if(!Buffer)
-    {
-        /* Increment the miss counter */
-        LookasideList->Global.AllocateMisses++;
-
-        /* Fallback to standard pool allocator */
-        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
-                                         &Buffer, LookasideList->Global.Tag);
-    }
-
-    /* Return the allocated buffer */
-    return Buffer;
-}

xtoskrnl/includes/ex/laslist.hh

@@ -30,6 +30,9 @@ namespace EX
         public:
             STATIC XTAPI PVOID AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList);
             STATIC XTAPI PVOID AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList);
+            STATIC XTAPI PVOID AllocateFromPerProcessorList(IN NONPAGED_LOOKASIDE_NUMBER Number);
+            STATIC XTAPI VOID FreeToPerProcessorList(IN NONPAGED_LOOKASIDE_NUMBER Number,
+                                                     IN PVOID Entry);
             STATIC XTAPI VOID InitializeLookasideList(IN OUT PGENERAL_LOOKASIDE LookasideList,
                                                       IN MMPOOL_TYPE PoolType,
                                                       IN ULONG Size,