From 03727a61d30d61c2e6dc26d6573b50c30846deba Mon Sep 17 00:00:00 2001 From: Rafal Kupiec Date: Wed, 22 May 2024 18:51:09 +0200 Subject: [PATCH] Implement hardware layer pool memory management --- sdk/xtdk/amd64/mmtypes.h | 14 +- sdk/xtdk/i686/mmtypes.h | 15 +- xtoskrnl/CMakeLists.txt | 2 + xtoskrnl/includes/globals.h | 9 + xtoskrnl/includes/mmi.h | 34 ++++ xtoskrnl/mm/globals.c | 9 + xtoskrnl/mm/hlpool.c | 383 ++++++++++++++++++++++++++++++++++++ xtoskrnl/mm/pages.c | 62 ++++++ 8 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 xtoskrnl/mm/hlpool.c create mode 100644 xtoskrnl/mm/pages.c diff --git a/sdk/xtdk/amd64/mmtypes.h b/sdk/xtdk/amd64/mmtypes.h index 39b18fb..96e7aa4 100644 --- a/sdk/xtdk/amd64/mmtypes.h +++ b/sdk/xtdk/amd64/mmtypes.h @@ -14,7 +14,7 @@ /* Pages related definitions */ -#define MM_PAGE_MASK 0xFFF +#define MM_PAGE_MASK (MM_PAGE_SIZE - 1) #define MM_PAGE_SHIFT 12L #define MM_PAGE_SIZE 4096 @@ -38,6 +38,18 @@ /* Default number of secondary colors */ #define MM_DEFAULT_SECONDARY_COLORS 64 +/* Number of HAL allocation descriptors */ +#define MM_HAL_ALLOCATION_DESCRIPTORS 64 + +/* Kernel HAL heap initial start address */ +#define MM_HAL_HEAP_START_ADDRESS ((PVOID)(((ULONG_PTR)MM_HAL_VA_START) + 1024 * 1024)) + +/* HAL memory pool virtual address start */ +#define MM_HAL_VA_START 0xFFFFFFFFFFC00000ULL + +/* Maximum physical address used by HAL allocations */ +#define MM_MAXIMUM_PHYSICAL_ADDRESS 0x00000000FFFFFFFF + /* Page size enumeration list */ typedef enum _PAGE_SIZE { diff --git a/sdk/xtdk/i686/mmtypes.h b/sdk/xtdk/i686/mmtypes.h index 3174ece..fc7a750 100644 --- a/sdk/xtdk/i686/mmtypes.h +++ b/sdk/xtdk/i686/mmtypes.h @@ -14,7 +14,7 @@ /* Pages related definitions */ -#define MM_PAGE_MASK 0xFFF +#define MM_PAGE_MASK (MM_PAGE_SIZE - 1) #define MM_PAGE_SHIFT 12 #define MM_PAGE_SIZE 4096 @@ -24,6 +24,19 @@ /* Default number of secondary colors */ #define MM_DEFAULT_SECONDARY_COLORS 64 +/* Number of HAL allocation descriptors */ +#define MM_HAL_ALLOCATION_DESCRIPTORS 64 + +/* Kernel HAL heap initial start address */ +#define MM_HAL_HEAP_START_ADDRESS ((PVOID)(((ULONG_PTR)MM_HAL_VA_START) + 1024 * 1024)) + +/* HAL memory pool virtual address start */ +#define MM_HAL_VA_START 0xFFC00000 + +/* Maximum physical address used by HAL allocations */ +#define MM_MAXIMUM_PHYSICAL_ADDRESS 0xFFFFFFFF + + /* Page size enumeration list */ typedef enum _PAGE_SIZE { diff --git a/xtoskrnl/CMakeLists.txt b/xtoskrnl/CMakeLists.txt index f61f9d5..ad03b96 100644 --- a/xtoskrnl/CMakeLists.txt +++ b/xtoskrnl/CMakeLists.txt @@ -39,8 +39,10 @@ list(APPEND XTOSKRNL_SOURCE ${XTOSKRNL_SOURCE_DIR}/ke/${ARCH}/kthread.c ${XTOSKRNL_SOURCE_DIR}/ke/${ARCH}/proc.c ${XTOSKRNL_SOURCE_DIR}/mm/globals.c + ${XTOSKRNL_SOURCE_DIR}/mm/hlpool.c ${XTOSKRNL_SOURCE_DIR}/mm/init.c ${XTOSKRNL_SOURCE_DIR}/mm/kpools.c + ${XTOSKRNL_SOURCE_DIR}/mm/pages.c ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/init.c ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pages.c ${XTOSKRNL_SOURCE_DIR}/po/idle.c diff --git a/xtoskrnl/includes/globals.h b/xtoskrnl/includes/globals.h index 5ca422b..196a3a8 100644 --- a/xtoskrnl/includes/globals.h +++ b/xtoskrnl/includes/globals.h @@ -69,4 +69,13 @@ EXTERN ULONG MmPageMapLevel; /* Processor structures data (THIS IS A TEMPORARY HACK) */ EXTERN UCHAR MmProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE]; +/* Allocation descriptors dedicated for HAL */ +EXTERN LOADER_MEMORY_MAPPING MmpHalAllocationDescriptors[MM_HAL_ALLOCATION_DESCRIPTORS]; + +/* Live address of kernel HAL heap */ +EXTERN PVOID MmpHalHeapStart; + +/* Number of used HAL allocation descriptors */ +EXTERN ULONG MmpUsedHalAllocationDescriptors; + #endif /* __XTOSKRNL_GLOBALS_H */ diff --git a/xtoskrnl/includes/mmi.h b/xtoskrnl/includes/mmi.h index d027abd..f72ef2f 100644 --- a/xtoskrnl/includes/mmi.h +++ b/xtoskrnl/includes/mmi.h @@ -13,6 +13,12 @@ /* Memory Manager routines forward references */ +XTAPI +XTSTATUS +MmAllocateHalMemory(IN PFN_NUMBER PageCount, + IN BOOLEAN Aligned, + OUT PULONG_PTR Buffer); + XTAPI XTSTATUS MmAllocateKernelStack(IN PVOID *Stack, @@ -24,6 +30,10 @@ XTSTATUS MmAllocateProcessorStructures(IN ULONG CpuNumber, OUT PVOID *StructuresData); +XTAPI +VOID +MmFlushTlb(VOID); + XTAPI VOID MmFreeKernelStack(IN PVOID Stack, @@ -37,6 +47,30 @@ XTAPI VOID MmInitializeMemoryManager(VOID); +XTAPI +XTSTATUS +MmMapHalMemory(IN PHYSICAL_ADDRESS PhysicalAddress, + IN PFN_NUMBER PageCount, + IN BOOLEAN FlushTlb, + OUT PVOID *VirtualAddress); + +XTAPI +VOID +MmMarkHalMemoryWriteThrough(IN PVOID VirtualAddress, + IN PFN_NUMBER PageCount); + +XTAPI +VOID +MmRemapHalMemory(IN PVOID VirtualAddress, + IN PHYSICAL_ADDRESS PhysicalAddress, + IN BOOLEAN FlushTlb); + +XTAPI +XTSTATUS +MmUnmapHalMemory(IN PVOID VirtualAddress, + IN PFN_NUMBER PageCount, + IN BOOLEAN FlushTlb); + XTAPI VOID MmpScanMemoryDescriptors(VOID); diff --git a/xtoskrnl/mm/globals.c b/xtoskrnl/mm/globals.c index dac2a17..c3f10fa 100644 --- a/xtoskrnl/mm/globals.c +++ b/xtoskrnl/mm/globals.c @@ -29,3 +29,12 @@ ULONG MmPageMapLevel; /* Processor structures data (THIS IS A TEMPORARY HACK) */ UCHAR MmProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE] = {0}; + +/* Allocation descriptors dedicated for HAL */ +LOADER_MEMORY_MAPPING MmpHalAllocationDescriptors[MM_HAL_ALLOCATION_DESCRIPTORS]; + +/* Live address of kernel HAL heap */ +PVOID MmpHalHeapStart = MM_HAL_HEAP_START_ADDRESS; + +/* Number of used HAL allocation descriptors */ +ULONG MmpUsedHalAllocationDescriptors = 0; diff --git a/xtoskrnl/mm/hlpool.c b/xtoskrnl/mm/hlpool.c new file mode 100644 index 0000000..c525232 --- /dev/null +++ b/xtoskrnl/mm/hlpool.c @@ -0,0 +1,383 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/hlpool.c + * DESCRIPTION: Hardware layer pool memory management + * DEVELOPERS: Rafal Kupiec + */ + +#include + + +/** + * Allocates physical memory for kernel hardware layer before memory manager gets initialized. + * + * @param PageCount + * Supplies the number of pages to be allocated. + * + * @param Aligned + * Specifies whether allocated memory should be aligned to 64k boundary or not. + * + * @param Buffer + * Supplies a buffer that receives the physical address. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MmAllocateHalMemory(IN PFN_NUMBER PageCount, + IN BOOLEAN Aligned, + OUT PULONG_PTR Buffer) +{ + PLOADER_MEMORY_MAPPING Descriptor, ExtraDescriptor, HalDescriptor; + PFN_NUMBER Alignment, MaxPage; + ULONGLONG PhysicalAddress; + PLIST_ENTRY ListEntry; + + /* Assume failure */ + *Buffer = 0; + + /* Calculate maximum page address */ + MaxPage = MM_MAXIMUM_PHYSICAL_ADDRESS >> MM_PAGE_SHIFT; + + /* Make sure there are at least 2 descriptors available */ + if((MmpUsedHalAllocationDescriptors + 2) > MM_HAL_ALLOCATION_DESCRIPTORS) + { + /* Not enough descriptors, return error */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Scan memory descriptors provided by the boot loader */ + ListEntry = KeInitializationBlock->MemoryDescriptorListHead.Flink; + while(ListEntry != &KeInitializationBlock->MemoryDescriptorListHead) + { + Descriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_MAPPING, ListEntry); + + /* Align memory to 64KB if needed */ + Alignment = Aligned ? (((Descriptor->BasePage + 0x0F) & ~0x0F) - Descriptor->BasePage) : 0; + + /* Ensure that memory type is free for this descriptor */ + if(Descriptor->MemoryType == LoaderFree) + { + /* Check if descriptor is big enough and if it fits under the maximum physical address */ + if(Descriptor->BasePage && + ((Descriptor->BasePage + PageCount + Alignment) < MaxPage) && + (Descriptor->PageCount >= (PageCount + Alignment))) + { + /* Set physical address */ + PhysicalAddress = (Descriptor->BasePage + Alignment) << MM_PAGE_SHIFT; + break; + } + } + + /* Move to next descriptor */ + ListEntry = ListEntry->Flink; + } + + /* Make sure we found a descriptor */ + if(ListEntry == &KeInitializationBlock->MemoryDescriptorListHead) + { + /* Descriptor not found, return error */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Allocate new descriptor */ + HalDescriptor = &MmpHalAllocationDescriptors[MmpUsedHalAllocationDescriptors]; + HalDescriptor->BasePage = Descriptor->BasePage + Alignment; + HalDescriptor->MemoryType = LoaderHALCachedMemory; + HalDescriptor->PageCount = PageCount; + + /* Update HAL allocation descriptors count */ + MmpUsedHalAllocationDescriptors++; + + /* Check if alignment was done */ + if(Alignment) + { + /* Check if extra descriptor is needed to describe the allocation */ + if(Descriptor->PageCount > (PageCount + Alignment)) + { + /* Initialize extra descriptor */ + ExtraDescriptor = &MmpHalAllocationDescriptors[MmpUsedHalAllocationDescriptors]; + ExtraDescriptor->BasePage = Descriptor->BasePage + Alignment + (ULONG)PageCount; + ExtraDescriptor->MemoryType = LoaderFree; + ExtraDescriptor->PageCount = Descriptor->PageCount - (Alignment + (ULONG)PageCount); + + /* Update HAL allocation descriptors count */ + MmpUsedHalAllocationDescriptors++; + + /* Insert extra descriptor in the list */ + RtlInsertHeadList(&Descriptor->ListEntry, &ExtraDescriptor->ListEntry); + } + + /* Trim source descriptor to the alignment */ + Descriptor->PageCount = Alignment; + + /* Insert new descriptor in the list */ + RtlInsertHeadList(&Descriptor->ListEntry, &HalDescriptor->ListEntry); + } + else + { + /* Consume pages from the source descriptor */ + Descriptor->BasePage += (ULONG)PageCount; + Descriptor->PageCount -= (ULONG)PageCount; + + /* Insert new descriptor in the list */ + RtlInsertTailList(&Descriptor->ListEntry, &HalDescriptor->ListEntry); + + /* Check if source descriptor is fully consumed */ + if(Descriptor->PageCount == 0) + { + /* Remove descriptor from the list */ + RtlRemoveEntryList(&Descriptor->ListEntry); + } + } + + /* Return physical address */ + *Buffer = PhysicalAddress; + return STATUS_SUCCESS; +} + +/** + * Maps physical address to the virtual memory area used by kernel hardware layer (HAL). + * + * @param PhysicalAddress + * Supplies the physical address to map. + * + * @param PageCount + * Supplies the number of pages to be mapped. + * + * @param FlushTlb + * Specifies whether to flush the TLB or not. + * + * @param VirtualAddress + * Supplies a buffer that receives the virtual address of the mapped pages. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MmMapHalMemory(IN PHYSICAL_ADDRESS PhysicalAddress, + IN PFN_NUMBER PageCount, + IN BOOLEAN FlushTlb, + OUT PVOID *VirtualAddress) +{ + PVOID BaseAddress, ReturnAddress; + PFN_NUMBER MappedPages; + PHARDWARE_PTE PtePointer; + + /* Initialize variables */ + BaseAddress = MmpHalHeapStart; + MappedPages = 0; + ReturnAddress = BaseAddress; + *VirtualAddress = NULL; + + /* Iterate through all pages */ + while(MappedPages < PageCount) + { + /* Check if address overflows */ + if(BaseAddress == NULL) + { + /* Not enough free pages, return error */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Get PTE pointer and advance to next page */ + PtePointer = (PHARDWARE_PTE)MmpGetPteAddress(VirtualAddress); + VirtualAddress = (PVOID)(ULONG_PTR)VirtualAddress + MM_PAGE_SIZE; + + /* Check if PTE is valid */ + if(PtePointer->Valid) + { + /* PTE is not available, go to the next one */ + BaseAddress = VirtualAddress; + MappedPages = 0; + continue; + } + + /* Increase number of mapped pages */ + MappedPages++; + } + + /* Take the actual base address with an offset */ + VirtualAddress = (PVOID)(ULONG_PTR)(BaseAddress + PAGE_OFFSET(PhysicalAddress.LowPart)); + + /* Check if base address starts at the beginning of the heap */ + if(BaseAddress == MmpHalHeapStart) + { + /* Move heap beyond base address */ + MmpHalHeapStart = (PVOID)((ULONG_PTR)BaseAddress + ((ULONG_PTR)PageCount << MM_PAGE_SHIFT)); + } + + /* Iterate through mapped pages */ + while(MappedPages--) + { + /* Get PTE pointer */ + PtePointer = (PHARDWARE_PTE)MmpGetPteAddress(BaseAddress); + + /* Fill the PTE */ + PtePointer->PageFrameNumber = (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT); + PtePointer->Valid = 1; + PtePointer->Writable = 1; + + /* Advance to the next address */ + PhysicalAddress.QuadPart += MM_PAGE_SIZE; + BaseAddress = (PVOID)((ULONG_PTR)BaseAddress + MM_PAGE_SIZE); + } + + /* Check if TLB needs to be flushed */ + if(FlushTlb) + { + /* Flush the TLB */ + MmFlushTlb(); + } + + /* Return virtual address */ + *VirtualAddress = ReturnAddress; + return STATUS_SUCCESS; +} + +/** + * Marks existing mapping as CD/WT to avoid delays in write-back cache. + * + * @param VirtualAddress + * Supplies the virtual address region to mark as CD/WT. + * + * @param PageCount + * Supplies the number of mapped pages. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MmMarkHalMemoryWriteThrough(IN PVOID VirtualAddress, + IN PFN_NUMBER PageCount) +{ + PHARDWARE_PTE PtePointer; + PFN_NUMBER Page; + + /* Get PTE address from virtual address */ + PtePointer = (PHARDWARE_PTE)MmpGetPteAddress(VirtualAddress); + + /* Iterate through mapped pages */ + for(Page = 0; Page < PageCount; Page++) + { + /* Mark pages as CD/WT */ + PtePointer->CacheDisable = 1; + PtePointer->WriteThrough = 1; + PtePointer++; + } +} + +/** + * Remaps the PTE to new physical address. + * + * @param VirtualAddress + * Supplies the virtual address to remap. + * + * @param PhysicalAddress + * Supplies a new physical address. + * + * @param FlushTlb + * Specifies whether to flush the TLB or not. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MmRemapHalMemory(IN PVOID VirtualAddress, + IN PHYSICAL_ADDRESS PhysicalAddress, + IN BOOLEAN FlushTlb) +{ + PHARDWARE_PTE PtePointer; + + /* Get PTE address from virtual address */ + PtePointer = (PHARDWARE_PTE)MmpGetPteAddress(VirtualAddress); + + /* Remap the PTE */ + PtePointer->PageFrameNumber = (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT); + PtePointer->Valid = 1; + PtePointer->Writable = 1; + + /* Check if TLB needs to be flushed */ + if(FlushTlb) + { + /* Flush the TLB */ + MmFlushTlb(); + } +} + +/** + * Unmaps a Page Table Entry corresponding to the given virtual address. + * + * @param VirtualAddress + * Supplies the virtual address to unmap. + * + * @param PageCount + * Supplies the number of mapped pages. + * + * @param FlushTlb + * Specifies whether to flush the TLB or not. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MmUnmapHalMemory(IN PVOID VirtualAddress, + IN PFN_NUMBER PageCount, + IN BOOLEAN FlushTlb) +{ + PHARDWARE_PTE PtePointer; + PFN_NUMBER Page; + + /* Check if address is valid HAL memory */ + if(VirtualAddress < (PVOID)MM_HAL_VA_START) + { + /* Invalid address, return error */ + return STATUS_INVALID_PARAMETER; + } + + /* Align virtual address down to page boundary */ + VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress & ~(MM_PAGE_SIZE - 1)); + + /* Get PTE address from virtual address */ + PtePointer = (PHARDWARE_PTE)MmpGetPteAddress(VirtualAddress); + + /* Iterate through mapped pages */ + for(Page = 0; Page < PageCount; Page++) + { + /* Unmap the PTE and get the next one */ + PtePointer->CacheDisable = 0; + PtePointer->Valid = 0; + PtePointer->Writable = 0; + PtePointer->WriteThrough = 0; + PtePointer->PageFrameNumber = 0; + PtePointer++; + } + + /* Check if TLB needs to be flushed */ + if(FlushTlb) + { + /* Flush the TLB */ + MmFlushTlb(); + } + + /* Check if heap can be reused */ + if(MmpHalHeapStart > VirtualAddress) + { + /* Free VA space */ + MmpHalHeapStart = VirtualAddress; + } + + /* Return success */ + return STATUS_SUCCESS; +} diff --git a/xtoskrnl/mm/pages.c b/xtoskrnl/mm/pages.c new file mode 100644 index 0000000..239314a --- /dev/null +++ b/xtoskrnl/mm/pages.c @@ -0,0 +1,62 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/pages.c + * DESCRIPTION: Low level page management support + * DEVELOPERS: Rafal Kupiec + */ + +#include + + +/** + * Flushes current Translation Lookaside Buffer (TLB) + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MmFlushTlb(VOID) +{ + CPUID_REGISTERS CpuRegisters; + BOOLEAN Interrupts; + ULONG_PTR Cr4; + + /* Save interrupts state and disable them */ + Interrupts = ArInterruptsEnabled(); + ArClearInterruptFlag(); + + /* Get CPU features */ + CpuRegisters.Leaf = CPUID_GET_CPU_FEATURES; + ArCpuId(&CpuRegisters); + + /* Check if Paging Global Extensions (PGE) is supported */ + if(CpuRegisters.Edx & CPUID_FEATURES_EDX_PGE) + { + /* Read CR4 */ + Cr4 = ArReadControlRegister(4); + + /* Disable PGE */ + ArWriteControlRegister(4, Cr4 & ~CR4_PGE); + + /* Flush the TLB */ + ArFlushTlb(); + + /* Restore CR4 */ + ArWriteControlRegister(4, Cr4); + } + else + { + /* Simply flush the TLB */ + ArFlushTlb(); + } + + /* Check if interrupts should be enabled */ + if(Interrupts) + { + /* Re-enable interrupts */ + ArSetInterruptFlag(); + } +}