From 8479c95e82a4dd54bc517ed5e96a628f339311ec Mon Sep 17 00:00:00 2001 From: Aiken Harris Date: Tue, 23 Jun 2026 19:05:04 +0200 Subject: [PATCH] Implement kernel lookaside lists --- sdk/xtdk/extypes.h | 120 +++++++++++++++++- sdk/xtdk/xtstruct.h | 9 +- xtoskrnl/CMakeLists.txt | 2 + xtoskrnl/ex/data.cc | 34 +++++ xtoskrnl/ex/laslist.cc | 213 ++++++++++++++++++++++++++++++++ xtoskrnl/includes/ex.hh | 1 + xtoskrnl/includes/ex/laslist.hh | 44 +++++++ 7 files changed, 418 insertions(+), 5 deletions(-) create mode 100644 xtoskrnl/ex/data.cc create mode 100644 xtoskrnl/ex/laslist.cc create mode 100644 xtoskrnl/includes/ex/laslist.hh diff --git a/sdk/xtdk/extypes.h b/sdk/xtdk/extypes.h index 8e66c94..447c543 100644 --- a/sdk/xtdk/extypes.h +++ b/sdk/xtdk/extypes.h @@ -17,18 +17,61 @@ /* Rundown protection flags */ #define EX_RUNDOWN_ACTIVE 0x1 +/* Number of lookaside lists */ +#define POOL_LOOKASIDE_LISTS 32 /* C/C++ specific code */ #ifndef __XTOS_ASSEMBLER__ -/* Executive rundown protection structure definition */ -typedef struct _EX_RUNDOWN_REFERENCE +/* Kernel routine callbacks */ +typedef XTSTATUS (XTAPI *PALLOCATE_FUNCTION)(IN MMPOOL_TYPE PoolType, IN SIZE_T Bytes, OUT PVOID *Memory, IN ULONG Tag); +typedef XTSTATUS (XTAPI *PALLOCATE_FUNCTION_EX)(IN MMPOOL_TYPE PoolType, IN SIZE_T Bytes, OUT PVOID *Memory, IN ULONG Tag, IN PLOOKASIDE_LIST_EX Lookaside); +typedef XTSTATUS (XTAPI *PFREE_FUNCTION)(IN PVOID Buffer); +typedef XTSTATUS (XTAPI *PFREE_FUNCTION_EX)(IN PVOID Buffer, IN OUT PLOOKASIDE_LIST_EX Lookaside); + +/* Owner entry structure definition */ +typedef struct _OWNER_ENTRY { + ULONG_PTR OwnerThread; union { - ULONG_PTR Count; - PVOID Ptr; + struct + { + ULONG IoPriorityBoosted:1; + ULONG OwnerReferenced:1; + ULONG OwnerCount:30; + }; + ULONG TableSize; }; +} OWNER_ENTRY, *POWNER_ENTRY; + +/* Exclusive resource structure definition */ +typedef struct _ERESOURCE +{ + LIST_ENTRY SystemResourcesList; + POWNER_ENTRY OwnerTable; + SHORT ActiveCount; + USHORT Flag; + VOLATILE PKSEMAPHORE SharedWaiters; + VOLATILE PKEVENT ExclusiveWaiters; + OWNER_ENTRY OwnerEntry; + ULONG ActiveEntries; + ULONG ContentionCount; + ULONG NumberOfSharedWaiters; + ULONG NumberOfExclusiveWaiters; + union + { + PVOID Address; + ULONG_PTR CreatorBackTraceIndex; + }; + KSPIN_LOCK SpinLock; +} ERESOURCE, *PERESOURCE; + +/* Executive rundown protection structure definition */ +typedef union _EX_RUNDOWN_REFERENCE +{ + ULONG_PTR Count; + PVOID Ptr; } EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE; /* Executive rundown wait block definition */ @@ -38,5 +81,74 @@ typedef struct _EX_RUNDOWN_WAIT_BLOCK KEVENT WakeEvent; } EX_RUNDOWN_WAIT_BLOCK, *PEX_RUNDOWN_WAIT_BLOCK; +/* Lookaside list structure definition */ +typedef struct _GENERAL_LOOKASIDE +{ + union + { + SINGLE_LIST_HEADER ListHead; + SINGLE_LIST_ENTRY SingleListHead; + }; + USHORT Depth; + USHORT MaximumDepth; + ULONG TotalAllocates; + union + { + ULONG AllocateMisses; + ULONG AllocateHits; + }; + ULONG TotalFrees; + union { + ULONG FreeMisses; + ULONG FreeHits; + }; + MMPOOL_TYPE Type; + ULONG Tag; + ULONG Size; + union + { + PALLOCATE_FUNCTION_EX AllocateEx; + PALLOCATE_FUNCTION Allocate; + }; + union + { + PFREE_FUNCTION_EX FreeEx; + PFREE_FUNCTION Free; + }; + LIST_ENTRY ListEntry; + ULONG LastTotalAllocates; + union + { + ULONG LastAllocateMisses; + ULONG LastAllocateHits; + }; + ULONG Future[2]; +} GENERAL_LOOKASIDE, *PGENERAL_LOOKASIDE; + +/* Lookaside list pointers structure definition */ +typedef struct _LOOKASIDE_LIST +{ + PGENERAL_LOOKASIDE Local; + PGENERAL_LOOKASIDE Global; +} LOOKASIDE_LIST, *PLOOKASIDE_LIST; + +/* Lookaside list extended structure definition */ +typedef struct _LOOKASIDE_LIST_EX +{ + GENERAL_LOOKASIDE Global; +} LOOKASIDE_LIST_EX, *PLOOKASIDE_LIST_EX; + +/* Non-paged lookaside list structure definition */ +typedef struct _NONPAGED_LOOKASIDE_LIST +{ + GENERAL_LOOKASIDE Global; +} NONPAGED_LOOKASIDE_LIST, *PNONPAGED_LOOKASIDE_LIST; + +/* Paged lookaside list structure definition */ +typedef struct _PAGED_LOOKASIDE_LIST +{ + GENERAL_LOOKASIDE Global; +} PAGED_LOOKASIDE_LIST, *PPAGED_LOOKASIDE_LIST; + #endif /* __XTOS_ASSEMBLER__ */ #endif /* __XTDK_EXTYPES_H */ diff --git a/sdk/xtdk/xtstruct.h b/sdk/xtdk/xtstruct.h index 04e80f0..351d8c7 100644 --- a/sdk/xtdk/xtstruct.h +++ b/sdk/xtdk/xtstruct.h @@ -242,12 +242,13 @@ typedef struct _EFI_VENDOR_DEVICE_PATH EFI_VENDOR_DEVICE_PATH, *PEFI_VENDOR_DEVI typedef struct _EFI_VLAN_DEVICE_PATH EFI_VLAN_DEVICE_PATH, *PEFI_VLAN_DEVICE_PATH; typedef struct _EFI_WORD_REGS EFI_WORD_REGS, *PEFI_WORD_REGS; typedef struct _EPROCESS EPROCESS, *PEPROCESS; +typedef struct _ERESOURCE ERESOURCE, *PERESOURCE; typedef struct _ETHREAD ETHREAD, *PETHREAD; -typedef struct _EX_RUNDOWN_REFERENCE EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE; typedef struct _EXCEPTION_RECORD EXCEPTION_RECORD, *PEXCEPTION_RECORD; typedef struct _EXCEPTION_REGISTRATION_RECORD EXCEPTION_REGISTRATION_RECORD, *PEXCEPTION_REGISTRATION_RECORD; typedef struct _FIRMWARE_INFORMATION_BLOCK FIRMWARE_INFORMATION_BLOCK, *PFIRMWARE_INFORMATION_BLOCK; typedef struct _FLOAT128 FLOAT128, *PFLOAT128; +typedef struct _GENERAL_LOOKASIDE GENERAL_LOOKASIDE, *PGENERAL_LOOKASIDE; typedef struct _GENERIC_ADDRESS GENERIC_ADDRESS, *PGENERIC_ADDRESS; typedef struct _GUID GUID, *PGUID; typedef struct _HL_FRAMEBUFFER_DATA HL_FRAMEBUFFER_DATA, *PHL_FRAMEBUFFER_DATA; @@ -290,12 +291,17 @@ typedef struct _LIST_ENTRY64 LIST_ENTRY64, *PLIST_ENTRY64; typedef struct _LOADER_GRAPHICS_INFORMATION_BLOCK LOADER_GRAPHICS_INFORMATION_BLOCK, *PLOADER_GRAPHICS_INFORMATION_BLOCK; typedef struct _LOADER_INFORMATION_BLOCK LOADER_INFORMATION_BLOCK, *PLOADER_INFORMATION_BLOCK; typedef struct _LOADER_MEMORY_DESCRIPTOR LOADER_MEMORY_DESCRIPTOR, *PLOADER_MEMORY_DESCRIPTOR; +typedef struct _LOOKASIDE_LIST LOOKASIDE_LIST, *PLOOKASIDE_LIST; +typedef struct _LOOKASIDE_LIST_EX LOOKASIDE_LIST_EX, *PLOOKASIDE_LIST_EX; typedef struct _M128 M128, *PM128; typedef struct _MMCOLOR_TABLES MMCOLOR_TABLES, *PMMCOLOR_TABLES; typedef struct _MMFREE_POOL_ENTRY MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY; typedef struct _MMMEMORY_LAYOUT MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT; typedef struct _MMPFNENTRY MMPFNENTRY, *PMMPFNENTRY; typedef struct _MMPFNLIST MMPFNLIST, *PMMPFNLIST; +typedef struct _NONPAGED_LOOKASIDE_LIST NONPAGED_LOOKASIDE_LIST, *PNONPAGED_LOOKASIDE_LIST; +typedef struct _OWNER_ENTRY OWNER_ENTRY, *POWNER_ENTRY; +typedef struct _PAGED_LOOKASIDE_LIST PAGED_LOOKASIDE_LIST, *PPAGED_LOOKASIDE_LIST; typedef struct _PCAT_FIRMWARE_INFORMATION PCAT_FIRMWARE_INFORMATION, *PPCAT_FIRMWARE_INFORMATION; typedef struct _PCI_BRIDGE_CONTROL_REGISTER PCI_BRIDGE_CONTROL_REGISTER, *PPCI_BRIDGE_CONTROL_REGISTER; typedef struct _PCI_COMMON_CONFIG PCI_COMMON_CONFIG, *PPCI_COMMON_CONFIG; @@ -378,6 +384,7 @@ typedef union _EFI_HASH_OUTPUT EFI_HASH_OUTPUT, *PEFI_HASH_OUTPUT; typedef union _EFI_IA32_REGISTER_SET EFI_IA32_REGISTER_SET, *PEFI_IA32_REGISTER_SET; typedef union _EFI_IP_ADDRESS EFI_IP_ADDRESS, *PEFI_IP_ADDRESS; typedef union _EFI_PXE_BASE_CODE_PACKET EFI_PXE_BASE_CODE_PACKET, *PEFI_PXE_BASE_CODE_PACKET; +typedef union _EX_RUNDOWN_REFERENCE EX_RUNDOWN_REFERENCE, *PEX_RUNDOWN_REFERENCE; typedef union _LARGE_INTEGER LARGE_INTEGER, *PLARGE_INTEGER; typedef union _SINGLE_LIST_HEADER SINGLE_LIST_HEADER, *PSINGLE_LIST_HEADER; typedef union _ULARGE_INTEGER ULARGE_INTEGER, *PULARGE_INTEGER; diff --git a/xtoskrnl/CMakeLists.txt b/xtoskrnl/CMakeLists.txt index 76e92db..99f8b7a 100644 --- a/xtoskrnl/CMakeLists.txt +++ b/xtoskrnl/CMakeLists.txt @@ -14,7 +14,9 @@ list(APPEND XTOSKRNL_SOURCE ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/data.cc ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/procsup.cc ${XTOSKRNL_SOURCE_DIR}/ar/${ARCH}/traps.cc + ${XTOSKRNL_SOURCE_DIR}/ex/data.cc ${XTOSKRNL_SOURCE_DIR}/ex/exports.cc + ${XTOSKRNL_SOURCE_DIR}/ex/laslist.cc ${XTOSKRNL_SOURCE_DIR}/ex/rundown.cc ${XTOSKRNL_SOURCE_DIR}/hl/${ARCH}/cpu.cc ${XTOSKRNL_SOURCE_DIR}/hl/${ARCH}/firmware.cc diff --git a/xtoskrnl/ex/data.cc b/xtoskrnl/ex/data.cc new file mode 100644 index 0000000..1e796a3 --- /dev/null +++ b/xtoskrnl/ex/data.cc @@ -0,0 +1,34 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/ex/data.cc + * DESCRIPTION: Kernel Executive global and static data + * DEVELOPERS: Rafal Kupiec + */ + +#include + + +/* Tracks all active non-paged lookaside lists in the system */ +LIST_ENTRY EX::LookasideList::NonPagedLookasideListHead; + +/* Spinlock protecting the integrity of the NonPagedLookasideListHead */ +KSPIN_LOCK EX::LookasideList::NonPagedLookasideListLock; + +/* Array of standard system lookaside lists for non-paged pool allocations */ +GENERAL_LOOKASIDE EX::LookasideList::NonPagedPoolLookasideLists[POOL_LOOKASIDE_LISTS]; + +/* Tracks all active paged lookaside lists in the system */ +LIST_ENTRY EX::LookasideList::PagedLookasideListHead; + +/* Spinlock protecting the integrity of the PagedLookasideListHead */ +KSPIN_LOCK EX::LookasideList::PagedLookasideListLock; + +/* Array of standard system lookaside lists for paged pool allocations */ +GENERAL_LOOKASIDE EX::LookasideList::PagedPoolLookasideLists[POOL_LOOKASIDE_LISTS]; + +/* Tracks all standard pool lookaside lists */ +LIST_ENTRY EX::LookasideList::PoolLookasideListHead; + +/* Tracks dynamic system lookaside lists */ +LIST_ENTRY EX::LookasideList::SystemLookasideListHead; diff --git a/xtoskrnl/ex/laslist.cc b/xtoskrnl/ex/laslist.cc new file mode 100644 index 0000000..aad8fb6 --- /dev/null +++ b/xtoskrnl/ex/laslist.cc @@ -0,0 +1,213 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/ex/laslist.cc + * DESCRIPTION: Lookaside Lists support + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Initializes a generic lookaside list and inserts it into the tracking list. + * + * @param LookasideList + * Supplies a pointer to the general lookaside list structure to initialize. + * + * @param PoolType + * Supplies the type of memory pool to be allocated. + * + * @param Size + * Supplies the size, in bytes, of the entries to be allocated from the lookaside list. + * + * @param Tag + * Supplies the pool tag to be used for allocations. + * + * @param MaxDepth + * Supplies the maximum number of entries the lookaside list should hold. + * + * @param ListHead + * Supplies a pointer to the linked list header used to track this lookaside list. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +EX::LookasideList::InitializeLookasideList(IN OUT PGENERAL_LOOKASIDE LookasideList, + IN MMPOOL_TYPE PoolType, + IN ULONG Size, + IN ULONG Tag, + IN USHORT MaxDepth, + IN PLIST_ENTRY ListHead) +{ + /* Initialize the singly linked list header and insert it into the tracking list */ + RTL::SinglyList::InitializeListHead(&LookasideList->ListHead); + RTL::LinkedList::InsertTailList(ListHead, &LookasideList->ListEntry); + + /* Initialize lookaside list properties */ + LookasideList->Allocate = &MM::Allocator::AllocatePool; + LookasideList->AllocateHits = 0; + LookasideList->Depth = 2; + LookasideList->Free = &MM::Allocator::FreePool; + LookasideList->FreeHits = 0; + LookasideList->LastAllocateHits = 0; + LookasideList->LastTotalAllocates = 0; + LookasideList->MaximumDepth = MaxDepth; + LookasideList->Size = Size; + LookasideList->Tag = Tag; + LookasideList->TotalAllocates = 0; + LookasideList->TotalFrees = 0; + LookasideList->Type = PoolType; +} + +/** + * Initializes the per-processor lookaside list pointers. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +EX::LookasideList::InitializePointers(VOID) +{ + PGENERAL_LOOKASIDE LookasideEntry; + PKPROCESSOR_CONTROL_BLOCK Prcb; + ULONG Index; + + /* Retrieve the processor control block */ + Prcb = KE::Processor::GetCurrentProcessorControlBlock(); + + /* Initialize both non-paged and paged lookaside lists for all pool indexes */ + for(Index = 0; Index < POOL_LOOKASIDE_LISTS; Index++) + { + /* Get the non-paged lookaside list entry */ + LookasideEntry = &NonPagedPoolLookasideLists[Index]; + + /* Initialize the singly linked list header for the non-paged entry */ + RTL::SinglyList::InitializeListHead(&LookasideEntry->ListHead); + + /* Assign the initialized non-paged entry to the PRCB */ + Prcb->NonPagedLookasideList[Index].Global = LookasideEntry; + Prcb->NonPagedLookasideList[Index].Local = LookasideEntry; + + /* Get the paged lookaside list entry */ + LookasideEntry = &PagedPoolLookasideLists[Index]; + + /* Initialize the singly linked list header for the paged entry */ + RTL::SinglyList::InitializeListHead(&LookasideEntry->ListHead); + + /* Assign the initialized paged entry to the PRCB */ + Prcb->PagedLookasideList[Index].Global = LookasideEntry; + Prcb->PagedLookasideList[Index].Local = LookasideEntry; + } +} + +/** + * Initializes global system lookaside lists and their synchronization primitives. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +EX::LookasideList::InitializeSystemLookasideLists(VOID) +{ + ULONG Index; + + /* Initialize spinlocks for protecting lookaside list operations */ + KE::SpinLock::InitializeSpinLock(&NonPagedLookasideListLock); + KE::SpinLock::InitializeSpinLock(&PagedLookasideListLock); + + /* Initialize global list heads for tracking various types of lookaside lists */ + RTL::LinkedList::InitializeListHead(&NonPagedLookasideListHead); + RTL::LinkedList::InitializeListHead(&PagedLookasideListHead); + RTL::LinkedList::InitializeListHead(&PoolLookasideListHead); + RTL::LinkedList::InitializeListHead(&SystemLookasideListHead); + + /* Initialize standard pool lookaside lists */ + for(Index = 0; Index < POOL_LOOKASIDE_LISTS; Index++) + { + /* Initialize non-paged pool lookaside list */ + InitializeLookasideList(&NonPagedPoolLookasideLists[Index], NonPagedPool, (Index + 1) * 8, + TAG_MM_MEMORY_POOL, 256, &PoolLookasideListHead); + + /* Initialize paged pool lookaside list */ + InitializeLookasideList(&PagedPoolLookasideLists[Index], PagedPool, (Index + 1) * 8, + TAG_MM_MEMORY_POOL, 256, &PoolLookasideListHead); + } +} + +/** + * Attempts to allocate a fixed-size buffer from the specified non-paged lookaside list. + * + * @param LookasideList + * Supplies a pointer to the non-paged lookaside list. + * + * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails. + * + * @since XT 1.0 + */ +XTAPI +PVOID +EX::LookasideList::AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList) +{ + PVOID Buffer; + + /* Increment the total number of allocation requests */ + LookasideList->Global.TotalAllocates++; + + /* Attempt to pop an entry from the lookaside list */ + Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead); + if(!Buffer) + { + /* Increment the miss counter */ + LookasideList->Global.AllocateMisses++; + + /* Fallback to standard pool allocator */ + (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size, + &Buffer, LookasideList->Global.Tag); + } + + /* Return the allocated buffer */ + return Buffer; +} + +/** + * Attempts to allocate a fixed-size buffer from the specified paged lookaside list. + * + * @param LookasideList + * Supplies a pointer to the paged lookaside list. + * + * @return Returns a pointer to the allocated buffer, or NULLPTR if the allocation fails. + * + * @since XT 1.0 + */ +XTAPI +PVOID +EX::LookasideList::AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList) +{ + PVOID Buffer; + + /* Increment the total number of allocation requests */ + LookasideList->Global.TotalAllocates++; + + /* Attempt to pop an entry from the lookaside list */ + Buffer = RTL::Atomic::PopEntrySingleList(&LookasideList->Global.ListHead); + if(!Buffer) + { + /* Increment the miss counter */ + LookasideList->Global.AllocateMisses++; + + /* Fallback to standard pool allocator */ + (LookasideList->Global.Allocate)(LookasideList->Global.Type, LookasideList->Global.Size, + &Buffer, LookasideList->Global.Tag); + } + + /* Return the allocated buffer */ + return Buffer; +} diff --git a/xtoskrnl/includes/ex.hh b/xtoskrnl/includes/ex.hh index 747bd2f..c29ac45 100644 --- a/xtoskrnl/includes/ex.hh +++ b/xtoskrnl/includes/ex.hh @@ -11,6 +11,7 @@ #include +#include #include #endif /* __XTOSKRNL_EX_HH */ diff --git a/xtoskrnl/includes/ex/laslist.hh b/xtoskrnl/includes/ex/laslist.hh new file mode 100644 index 0000000..78f7054 --- /dev/null +++ b/xtoskrnl/includes/ex/laslist.hh @@ -0,0 +1,44 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/ex/laslist.hh + * DESCRIPTION: Lookaside Lists support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_EX_LASLIST_HH +#define __XTOSKRNL_EX_LASLIST_HH + +#include + + +/* Architecture-specific Library */ +namespace EX +{ + class LookasideList + { + private: + STATIC LIST_ENTRY NonPagedLookasideListHead; + STATIC KSPIN_LOCK NonPagedLookasideListLock; + STATIC GENERAL_LOOKASIDE NonPagedPoolLookasideLists[POOL_LOOKASIDE_LISTS]; + STATIC LIST_ENTRY PagedLookasideListHead; + STATIC KSPIN_LOCK PagedLookasideListLock; + STATIC GENERAL_LOOKASIDE PagedPoolLookasideLists[POOL_LOOKASIDE_LISTS]; + STATIC LIST_ENTRY PoolLookasideListHead; + STATIC LIST_ENTRY SystemLookasideListHead; + + public: + STATIC XTAPI PVOID AllocateFromLookasideList(IN PNONPAGED_LOOKASIDE_LIST LookasideList); + STATIC XTAPI PVOID AllocateFromLookasideList(IN PPAGED_LOOKASIDE_LIST LookasideList); + STATIC XTAPI VOID InitializeLookasideList(IN OUT PGENERAL_LOOKASIDE LookasideList, + IN MMPOOL_TYPE PoolType, + IN ULONG Size, + IN ULONG Tag, + IN USHORT MaxDepth, + IN PLIST_ENTRY ListHead); + STATIC XTAPI VOID InitializePointers(VOID); + STATIC XTAPI VOID InitializeSystemLookasideLists(VOID); + }; +} + +#endif /* __XTOSKRNL_EX_LASLIST_HH */