Implement kernel lookaside lists

sdk/xtdk/extypes.h

@@ -17,18 +17,61 @@
 /* Rundown protection flags */
 #define EX_RUNDOWN_ACTIVE                               0x1
 
+/* Number of lookaside lists */
+#define POOL_LOOKASIDE_LISTS                            32
 
 /* C/C++ specific code */
 #ifndef __XTOS_ASSEMBLER__
 
-/* Executive rundown protection structure definition */
+/* Kernel routine callbacks */
-typedef struct _EX_RUNDOWN_REFERENCE
+typedef XTSTATUS (XTAPI *PALLOCATE_FUNCTION)(IN MMPOOL_TYPE PoolType, IN SIZE_T Bytes, OUT PVOID *Memory, IN ULONG Tag);
+typedef XTSTATUS (XTAPI *PALLOCATE_FUNCTION_EX)(IN MMPOOL_TYPE PoolType, IN SIZE_T Bytes, OUT PVOID *Memory, IN ULONG Tag, IN PLOOKASIDE_LIST_EX Lookaside);
+typedef XTSTATUS (XTAPI *PFREE_FUNCTION)(IN PVOID Buffer);
+typedef XTSTATUS (XTAPI *PFREE_FUNCTION_EX)(IN PVOID Buffer, IN OUT PLOOKASIDE_LIST_EX Lookaside);
+
+/* Owner entry structure definition */
+typedef struct _OWNER_ENTRY
 {
+    ULONG_PTR OwnerThread;
     union
     {
-        ULONG_PTR Count;
+        struct
-        PVOID Ptr;
+        {
+            ULONG IoPriorityBoosted:1;
+            ULONG OwnerReferenced:1;
+            ULONG OwnerCount:30;
+        };
+        ULONG TableSize;
     };
+} OWNER_ENTRY, *POWNER_ENTRY;
+
+/* Exclusive resource structure definition */
+typedef struct _ERESOURCE
+{
+    LIST_ENTRY SystemResourcesList;
+    POWNER_ENTRY OwnerTable;
+    SHORT ActiveCount;
+    USHORT Flag;
+    VOLATILE PKSEMAPHORE SharedWaiters;
+    VOLATILE PKEVENT ExclusiveWaiters;
+    OWNER_ENTRY OwnerEntry;
+    ULONG ActiveEntries;
+    ULONG ContentionCount;
+    ULONG NumberOfSharedWaiters;
+    ULONG NumberOfExclusiveWaiters;
+    union
+    {
+        PVOID Address;
+        ULONG_PTR CreatorBackTraceIndex;
+    };
+    KSPIN_LOCK SpinLock;
+} ERESOURCE, *PERESOURCE;
+
+/* Executive rundown protection structure definition */
+typedef union _EX_RUNDOWN_REFERENCE
+{
+    ULONG_PTR Count;
+    PVOID Ptr;
 } EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE;
 
 /* Executive rundown wait block definition */
@@ -38,5 +81,74 @@ typedef struct _EX_RUNDOWN_WAIT_BLOCK
     KEVENT WakeEvent;
 } EX_RUNDOWN_WAIT_BLOCK, *PEX_RUNDOWN_WAIT_BLOCK;
 
+/* Lookaside list structure definition */
+typedef struct _GENERAL_LOOKASIDE
+{
+    union
+    {
+        SINGLE_LIST_HEADER ListHead;
+        SINGLE_LIST_ENTRY SingleListHead;
+    };
+    USHORT Depth;
+    USHORT MaximumDepth;
+    ULONG TotalAllocates;
+    union
+    {
+        ULONG AllocateMisses;
+        ULONG AllocateHits;
+    };
+    ULONG TotalFrees;
+    union {
+        ULONG FreeMisses;
+        ULONG FreeHits;
+    };
+    MMPOOL_TYPE Type;
+    ULONG Tag;
+    ULONG Size;
+    union
+    {
+        PALLOCATE_FUNCTION_EX AllocateEx;
+        PALLOCATE_FUNCTION Allocate;
+    };
+    union
+    {
+        PFREE_FUNCTION_EX FreeEx;
+        PFREE_FUNCTION Free;
+    };
+    LIST_ENTRY ListEntry;
+    ULONG LastTotalAllocates;
+    union
+    {
+        ULONG LastAllocateMisses;
+        ULONG LastAllocateHits;
+    };
+    ULONG Future[2];
+} GENERAL_LOOKASIDE, *PGENERAL_LOOKASIDE;
+
+/* Lookaside list pointers structure definition */
+typedef struct _LOOKASIDE_LIST
+{
+    PGENERAL_LOOKASIDE Local;
+    PGENERAL_LOOKASIDE Global;
+} LOOKASIDE_LIST, *PLOOKASIDE_LIST;
+
+/* Lookaside list extended structure definition */
+typedef struct _LOOKASIDE_LIST_EX
+{
+  GENERAL_LOOKASIDE Global;
+} LOOKASIDE_LIST_EX, *PLOOKASIDE_LIST_EX;
+
+/* Non-paged lookaside list structure definition */
+typedef struct _NONPAGED_LOOKASIDE_LIST
+{
+    GENERAL_LOOKASIDE Global;
+} NONPAGED_LOOKASIDE_LIST, *PNONPAGED_LOOKASIDE_LIST;
+
+/* Paged lookaside list structure definition */
+typedef struct _PAGED_LOOKASIDE_LIST
+{
+    GENERAL_LOOKASIDE Global;
+} PAGED_LOOKASIDE_LIST, *PPAGED_LOOKASIDE_LIST;
+
 #endif /* __XTOS_ASSEMBLER__ */
 #endif /* __XTDK_EXTYPES_H */

sdk/xtdk/xtstruct.h

@@ -242,12 +242,13 @@ typedef struct _EFI_VENDOR_DEVICE_PATH EFI_VENDOR_DEVICE_PATH, *PEFI_VENDOR_DEVI
 typedef struct _EFI_VLAN_DEVICE_PATH EFI_VLAN_DEVICE_PATH, *PEFI_VLAN_DEVICE_PATH;
 typedef struct _EFI_WORD_REGS EFI_WORD_REGS, *PEFI_WORD_REGS;
 typedef struct _EPROCESS EPROCESS, *PEPROCESS;
+typedef struct _ERESOURCE ERESOURCE, *PERESOURCE;
 typedef struct _ETHREAD ETHREAD, *PETHREAD;
-typedef struct _EX_RUNDOWN_REFERENCE EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE;
 typedef struct _EXCEPTION_RECORD EXCEPTION_RECORD, *PEXCEPTION_RECORD;
 typedef struct _EXCEPTION_REGISTRATION_RECORD EXCEPTION_REGISTRATION_RECORD, *PEXCEPTION_REGISTRATION_RECORD;
 typedef struct _FIRMWARE_INFORMATION_BLOCK FIRMWARE_INFORMATION_BLOCK, *PFIRMWARE_INFORMATION_BLOCK;
 typedef struct _FLOAT128 FLOAT128, *PFLOAT128;
+typedef struct _GENERAL_LOOKASIDE GENERAL_LOOKASIDE, *PGENERAL_LOOKASIDE;
 typedef struct _GENERIC_ADDRESS GENERIC_ADDRESS, *PGENERIC_ADDRESS;
 typedef struct _GUID GUID, *PGUID;
 typedef struct _HL_FRAMEBUFFER_DATA HL_FRAMEBUFFER_DATA, *PHL_FRAMEBUFFER_DATA;
@@ -290,12 +291,17 @@ typedef struct _LIST_ENTRY64 LIST_ENTRY64, *PLIST_ENTRY64;
 typedef struct _LOADER_GRAPHICS_INFORMATION_BLOCK LOADER_GRAPHICS_INFORMATION_BLOCK, *PLOADER_GRAPHICS_INFORMATION_BLOCK;
 typedef struct _LOADER_INFORMATION_BLOCK LOADER_INFORMATION_BLOCK, *PLOADER_INFORMATION_BLOCK;
 typedef struct _LOADER_MEMORY_DESCRIPTOR LOADER_MEMORY_DESCRIPTOR, *PLOADER_MEMORY_DESCRIPTOR;
+typedef struct _LOOKASIDE_LIST LOOKASIDE_LIST, *PLOOKASIDE_LIST;
+typedef struct _LOOKASIDE_LIST_EX LOOKASIDE_LIST_EX, *PLOOKASIDE_LIST_EX;
 typedef struct _M128 M128, *PM128;
 typedef struct _MMCOLOR_TABLES MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 typedef struct _MMFREE_POOL_ENTRY MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY;
 typedef struct _MMMEMORY_LAYOUT MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT;
 typedef struct _MMPFNENTRY MMPFNENTRY, *PMMPFNENTRY;
 typedef struct _MMPFNLIST MMPFNLIST, *PMMPFNLIST;
+typedef struct _NONPAGED_LOOKASIDE_LIST NONPAGED_LOOKASIDE_LIST, *PNONPAGED_LOOKASIDE_LIST;
+typedef struct _OWNER_ENTRY OWNER_ENTRY, *POWNER_ENTRY;
+typedef struct _PAGED_LOOKASIDE_LIST PAGED_LOOKASIDE_LIST, *PPAGED_LOOKASIDE_LIST;
 typedef struct _PCAT_FIRMWARE_INFORMATION PCAT_FIRMWARE_INFORMATION, *PPCAT_FIRMWARE_INFORMATION;
 typedef struct _PCI_BRIDGE_CONTROL_REGISTER PCI_BRIDGE_CONTROL_REGISTER, *PPCI_BRIDGE_CONTROL_REGISTER;
 typedef struct _PCI_COMMON_CONFIG PCI_COMMON_CONFIG, *PPCI_COMMON_CONFIG;
@@ -378,6 +384,7 @@ typedef union _EFI_HASH_OUTPUT EFI_HASH_OUTPUT, *PEFI_HASH_OUTPUT;
 typedef union _EFI_IA32_REGISTER_SET EFI_IA32_REGISTER_SET, *PEFI_IA32_REGISTER_SET;
 typedef union _EFI_IP_ADDRESS EFI_IP_ADDRESS, *PEFI_IP_ADDRESS;
 typedef union _EFI_PXE_BASE_CODE_PACKET EFI_PXE_BASE_CODE_PACKET, *PEFI_PXE_BASE_CODE_PACKET;
+typedef union _EX_RUNDOWN_REFERENCE EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE;
 typedef union _LARGE_INTEGER LARGE_INTEGER, *PLARGE_INTEGER;
 typedef union _SINGLE_LIST_HEADER SINGLE_LIST_HEADER, *PSINGLE_LIST_HEADER;
 typedef union _ULARGE_INTEGER ULARGE_INTEGER, *PULARGE_INTEGER;

xtoskrnl/CMakeLists.txt

@@ -14,7 +14,9 @@ list(APPEND XTOSKRNL_SOURCE
     ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/data.cc
     ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/procsup.cc
     ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/traps.cc
+    ${XTOSKRNL_SOURCE_DIR}/ex/data.cc
     ${XTOSKRNL_SOURCE_DIR}/ex/exports.cc
+    ${XTOSKRNL_SOURCE_DIR}/ex/laslist.cc
     ${XTOSKRNL_SOURCE_DIR}/ex/rundown.cc
     ${XTOSKRNL_SOURCE_DIR}/hl/${ARCH}/cpu.cc
     ${XTOSKRNL_SOURCE_DIR}/hl/${ARCH}/firmware.cc

xtoskrnl/ex/data.cc

@@ -0,0 +1,34 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/ex/data.cc
+ * DESCRIPTION:     Kernel Executive global and static data
+ * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
+ */
+
+#include <xtos.hh>
+
+
+/* Tracks all active non-paged lookaside lists in the system */
+LIST_ENTRY EX::LookasideList::NonPagedLookasideListHead;
+
+/* Spinlock protecting the integrity of the NonPagedLookasideListHead */
+KSPIN_LOCK EX::LookasideList::NonPagedLookasideListLock;
+
+/* Array of standard system lookaside lists for non-paged pool allocations */
+GENERAL_LOOKASIDE EX::LookasideList::NonPagedPoolLookasideLists[POOL_LOOKASIDE_LISTS];
+
+/* Tracks all active paged lookaside lists in the system */
+LIST_ENTRY EX::LookasideList::PagedLookasideListHead;
+
+/* Spinlock protecting the integrity of the PagedLookasideListHead */
+KSPIN_LOCK EX::LookasideList::PagedLookasideListLock;
+
+/* Array of standard system lookaside lists for paged pool allocations */
+GENERAL_LOOKASIDE EX::LookasideList::PagedPoolLookasideLists[POOL_LOOKASIDE_LISTS];
+
+/* Tracks all standard pool lookaside lists */
+LIST_ENTRY EX::LookasideList::PoolLookasideListHead;
+
+/* Tracks dynamic system lookaside lists */
+LIST_ENTRY EX::LookasideList::SystemLookasideListHead;

xtoskrnl/ex/laslist.cc

@@ -0,0 +1,213 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/ex/laslist.cc
+ * DESCRIPTION:     Lookaside Lists support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Initializes a generic lookaside list and inserts it into the tracking list.
+ *
+ * @param LookasideList
+ *        Supplies a pointer to the general lookaside list structure to initialize.
+ *
+ * @param PoolType
+ *        Supplies the type of memory pool to be allocated.
+ *
+ * @param Size
+ *        Supplies the size, in bytes, of the entries to be allocated from the lookaside list.
+ *
+ * @param Tag
+ *        Supplies the pool tag to be used for allocations.
+ *
+ * @param MaxDepth
+ *        Supplies the maximum number of entries the lookaside list should hold.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the linked list header used to track this lookaside list.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+EX::LookasideList::InitializeLookasideList(IN OUT PGENERAL_LOOKASIDE LookasideList,
+                                           IN MMPOOL_TYPE PoolType,
+                                           IN ULONG Size,
+                                           IN ULONG Tag,
+                                           IN USHORT MaxDepth,
+                                           IN PLIST_ENTRY ListHead)
+{
+    /* Initialize the singly linked list header and insert it into the tracking list */
+    RTL::SinglyList::InitializeListHead(&LookasideList->ListHead);
+    RTL::LinkedList::InsertTailList(ListHead, &LookasideList->ListEntry);
+
+    /* Initialize lookaside list properties */
+    LookasideList->Allocate = &MM::Allocator::AllocatePool;
+    LookasideList->AllocateHits = 0;
+    LookasideList->Depth = 2;
+    LookasideList->Free = &MM::Allocator::FreePool;
+    LookasideList->FreeHits = 0;
+    LookasideList->LastAllocateHits = 0;
+    LookasideList->LastTotalAllocates = 0;
+    LookasideList->MaximumDepth = MaxDepth;
+    LookasideList->Size = Size;
+    LookasideList->Tag = Tag;
+    LookasideList->TotalAllocates = 0;
+    LookasideList->TotalFrees = 0;
+    LookasideList->Type = PoolType;
+}
+
+/**
+ * Initializes the per-processor lookaside list pointers.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+EX::LookasideList::InitializePointers(VOID)
+{
+    PGENERAL_LOOKASIDE LookasideEntry;
+    PKPROCESSOR_CONTROL_BLOCK Prcb;
+    ULONG Index;
+
+    /* Retrieve the processor control block */
+    Prcb = KE::Processor::GetCurrentProcessorControlBlock();
+
+    /* Initialize both non-paged and paged lookaside lists for all pool indexes */
+    for(Index = 0; Index < POOL_LOOKASIDE_LISTS; Index++)
+    {
+        /* Get the non-paged lookaside list entry */
+        LookasideEntry = &NonPagedPoolLookasideLists[Index];
+
+        /* Initialize the singly linked list header for the non-paged entry */
+        RTL::SinglyList::InitializeListHead(&LookasideEntry->ListHead);
+
+        /* Assign the initialized non-paged entry to the PRCB */
+        Prcb->NonPagedLookasideList[Index].Global = LookasideEntry;
+        Prcb->NonPagedLookasideList[Index].Local = LookasideEntry;
+
+        /* Get the paged lookaside list entry */
+        LookasideEntry = &PagedPoolLookasideLists[Index];
+
+        /* Initialize the singly linked list header for the paged entry */
+        RTL::SinglyList::InitializeListHead(&LookasideEntry->ListHead);
+
+        /* Assign the initialized paged entry to the PRCB */
+        Prcb->PagedLookasideList[Index].Global = LookasideEntry;
+        Prcb->PagedLookasideList[Index].Local = LookasideEntry;
+    }
+}
+
+/**
+ * Initializes global system lookaside lists and their synchronization primitives.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+EX::LookasideList::InitializeSystemLookasideLists(VOID)
+{
+    ULONG Index;
+
+    /* Initialize spinlocks for protecting lookaside list operations */
+    KE::SpinLock::InitializeSpinLock(&NonPagedLookasideListLock);
+    KE::SpinLock::InitializeSpinLock(&PagedLookasideListLock);
+
+    /* Initialize global list heads for tracking various types of lookaside lists */
+    RTL::LinkedList::InitializeListHead(&NonPagedLookasideListHead);
+    RTL::LinkedList::InitializeListHead(&PagedLookasideListHead);
+    RTL::LinkedList::InitializeListHead(&PoolLookasideListHead);
+    RTL::LinkedList::InitializeListHead(&SystemLookasideListHead);
+
+    /* Initialize standard pool lookaside lists */
+    for(Index = 0; Index < POOL_LOOKASIDE_LISTS; Index++)
+    {
+        /* Initialize non-paged pool lookaside list */
+        InitializeLookasideList(&NonPagedPoolLookasideLists[Index], NonPagedPool, (Index + 1) * 8,
+                                TAG_MM_MEMORY_POOL, 256, &PoolLookasideListHead);
+
+        /* Initialize paged pool lookaside list */
+        InitializeLookasideList(&PagedPoolLookasideLists[Index], PagedPool, (Index + 1) * 8,
+                                TAG_MM_MEMORY_POOL, 256, &PoolLookasideListHead);
+    }
+}
+
+/**
+ * Attempts to allocate a fixed-size buffer from the specified non-paged lookaside list.
+ *
+ * @param LookasideList
+ *        Supplies a pointer to the non-paged lookaside list.
+ *
+ * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+EX::LookasideList::AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList)
+{
+    PVOID Buffer;
+
+    /* Increment the total number of allocation requests */
+    LookasideList->Global.TotalAllocates++;
+
+    /* Attempt to pop an entry from the lookaside list */
+    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
+    if(!Buffer)
+    {
+        /* Increment the miss counter */
+        LookasideList->Global.AllocateMisses++;
+
+        /* Fallback to standard pool allocator */
+        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
+                                         &Buffer, LookasideList->Global.Tag);
+    }
+
+    /* Return the allocated buffer */
+    return Buffer;
+}
+
+/**
+ * Attempts to allocate a fixed-size buffer from the specified paged lookaside list.
+ *
+ * @param LookasideList
+ *        Supplies a pointer to the paged lookaside list.
+ *
+ * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+EX::LookasideList::AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList)
+{
+    PVOID Buffer;
+
+    /* Increment the total number of allocation requests */
+    LookasideList->Global.TotalAllocates++;
+
+    /* Attempt to pop an entry from the lookaside list */
+    Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead);
+    if(!Buffer)
+    {
+        /* Increment the miss counter */
+        LookasideList->Global.AllocateMisses++;
+
+        /* Fallback to standard pool allocator */
+        (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size,
+                                         &Buffer, LookasideList->Global.Tag);
+    }
+
+    /* Return the allocated buffer */
+    return Buffer;
+}

xtoskrnl/includes/ex.hh

@@ -11,6 +11,7 @@
 
 #include <xtos.hh>
 
+#include <ex/laslist.hh>
 #include <ex/rundown.hh>
 
 #endif /* __XTOSKRNL_EX_HH */

xtoskrnl/includes/ex/laslist.hh

@@ -0,0 +1,44 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/ex/laslist.hh
+ * DESCRIPTION:     Lookaside Lists support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_EX_LASLIST_HH
+#define __XTOSKRNL_EX_LASLIST_HH
+
+#include <xtos.hh>
+
+
+/* Architecture-specific Library */
+namespace EX
+{
+    class LookasideList
+    {
+        private:
+            STATIC LIST_ENTRY NonPagedLookasideListHead;
+            STATIC KSPIN_LOCK NonPagedLookasideListLock;
+            STATIC GENERAL_LOOKASIDE NonPagedPoolLookasideLists[POOL_LOOKASIDE_LISTS];
+            STATIC LIST_ENTRY PagedLookasideListHead;
+            STATIC KSPIN_LOCK PagedLookasideListLock;
+            STATIC GENERAL_LOOKASIDE PagedPoolLookasideLists[POOL_LOOKASIDE_LISTS];
+            STATIC LIST_ENTRY PoolLookasideListHead;
+            STATIC LIST_ENTRY SystemLookasideListHead;
+
+        public:
+            STATIC XTAPI PVOID AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList);
+            STATIC XTAPI PVOID AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList);
+            STATIC XTAPI VOID InitializeLookasideList(IN OUT PGENERAL_LOOKASIDE LookasideList,
+                                                      IN MMPOOL_TYPE PoolType,
+                                                      IN ULONG Size,
+                                                      IN ULONG Tag,
+                                                      IN USHORT MaxDepth,
+                                                      IN PLIST_ENTRY ListHead);
+            STATIC XTAPI VOID InitializePointers(VOID);
+            STATIC XTAPI VOID InitializeSystemLookasideLists(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_EX_LASLIST_HH */