diff --git a/boot/xtldr/arch/amd64/memory.cc b/boot/xtldr/arch/amd64/memory.cc index ed12532..8f3659a 100644 --- a/boot/xtldr/arch/amd64/memory.cc +++ b/boot/xtldr/arch/amd64/memory.cc @@ -44,6 +44,14 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, return Status; } + /* Add new memory mapping for the page map itself */ + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData); + if(Status != STATUS_EFI_SUCCESS) + { + /* Memory mapping failure */ + return Status; + } + /* Assign and zero-fill memory used by page mappings */ PageMap->PtePointer = (PVOID)(UINT_PTR)Address; RTL::Memory::ZeroMemory(PageMap->PtePointer, EFI_PAGE_SIZE); @@ -57,7 +65,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, } /* Map the trampoline code area */ - Status = MapVirtualMemory(PageMap, (PVOID)MM_TRAMPOLINE_ADDRESS,(PVOID)MM_TRAMPOLINE_ADDRESS, + Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS,MM_TRAMPOLINE_ADDRESS, 1, LoaderFirmwareTemporary); if(Status != STATUS_EFI_SUCCESS) { @@ -74,7 +82,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, ModuleInfo = CONTAIN_RECORD(ModulesListEntry, XTBL_MODULE_INFO, Flink); /* Map module code */ - Status = MapVirtualMemory(PageMap, ModuleInfo->ModuleBase, ModuleInfo->ModuleBase, + Status = MapVirtualMemory(PageMap, (ULONGLONG)ModuleInfo->ModuleBase, (ULONGLONG)ModuleInfo->ModuleBase, EFI_SIZE_TO_PAGES(ModuleInfo->ModuleSize), LoaderFirmwareTemporary); /* Check if mapping succeeded */ @@ -95,7 +103,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, if(LoaderBase && LoaderSize) { /* Map boot loader code as well */ - Status = MapVirtualMemory(PageMap, LoaderBase, LoaderBase, + Status = MapVirtualMemory(PageMap, (ULONGLONG)LoaderBase, (ULONGLONG)LoaderBase, EFI_SIZE_TO_PAGES(LoaderSize), LoaderFirmwareTemporary); if(Status != STATUS_EFI_SUCCESS) { @@ -193,7 +201,7 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap, } /* Add new memory mapping */ - Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)(UINT_PTR)Address, 1, LoaderMemoryData); + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData); if(Status != STATUS_EFI_SUCCESS) { /* Memory mapping failure */ @@ -239,9 +247,9 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap, XTCDECL EFI_STATUS Memory::MapPage(IN PXTBL_PAGE_MAPPING PageMap, - IN ULONG_PTR VirtualAddress, - IN ULONG_PTR PhysicalAddress, - IN ULONG NumberOfPages) + IN ULONGLONG VirtualAddress, + IN ULONGLONG PhysicalAddress, + IN ULONGLONG NumberOfPages) { PVOID Pml1, Pml2, Pml3, Pml4, Pml5; SIZE_T Pml1Entry, Pml2Entry, Pml3Entry, Pml4Entry, Pml5Entry; diff --git a/boot/xtldr/arch/i686/memory.cc b/boot/xtldr/arch/i686/memory.cc index a194d2d..8250b79 100644 --- a/boot/xtldr/arch/i686/memory.cc +++ b/boot/xtldr/arch/i686/memory.cc @@ -93,8 +93,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, } /* Map the trampoline code area */ - Status = MapVirtualMemory(PageMap, (PVOID)MM_TRAMPOLINE_ADDRESS,(PVOID)MM_TRAMPOLINE_ADDRESS, - 1, LoaderFirmwareTemporary); + Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS, MM_TRAMPOLINE_ADDRESS, 1, LoaderFirmwareTemporary); if(Status != STATUS_EFI_SUCCESS) { /* Mapping trampoline code failed */ @@ -110,7 +109,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, ModuleInfo = CONTAIN_RECORD(ModulesListEntry, XTBL_MODULE_INFO, Flink); /* Map module code */ - Status = MapVirtualMemory(PageMap, ModuleInfo->ModuleBase, ModuleInfo->ModuleBase, + Status = MapVirtualMemory(PageMap, (ULONGLONG)ModuleInfo->ModuleBase, (ULONGLONG)ModuleInfo->ModuleBase, EFI_SIZE_TO_PAGES(ModuleInfo->ModuleSize), LoaderFirmwareTemporary); /* Check if mapping succeeded */ @@ -131,7 +130,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, if(LoaderBase && LoaderSize) { /* Map boot loader code as well */ - Status = MapVirtualMemory(PageMap, LoaderBase, LoaderBase, + Status = MapVirtualMemory(PageMap, (ULONGLONG)LoaderBase, (ULONGLONG)LoaderBase, EFI_SIZE_TO_PAGES(LoaderSize), LoaderFirmwareTemporary); if(Status != STATUS_EFI_SUCCESS) { @@ -157,8 +156,9 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap, if(Mapping->VirtualAddress) { /* Dump memory mapping */ - Debug::Print(L" Type=%02lu, PhysicalBase=%.8P, VirtualBase=%.8P, Pages=%llu\n", Mapping->MemoryType, - Mapping->PhysicalAddress, Mapping->VirtualAddress, Mapping->NumberOfPages); + Debug::Print(L" Type=%02lu, PhysicalBase=0x%.8llX, VirtualBase=0x%.8llX, Pages=%llu\n", + Mapping->MemoryType, Mapping->PhysicalAddress, + Mapping->VirtualAddress, Mapping->NumberOfPages); /* Map memory */ Status = MapPage(PageMap, (UINT_PTR)Mapping->VirtualAddress, @@ -252,7 +252,7 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap, } /* Add new memory mapping */ - Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)(UINT_PTR)Address, 1, LoaderMemoryData); + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData); if(Status != STATUS_EFI_SUCCESS) { /* Memory mapping failure */ @@ -313,11 +313,11 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap, XTCDECL EFI_STATUS Memory::MapPage(IN PXTBL_PAGE_MAPPING PageMap, - IN ULONG_PTR VirtualAddress, - IN ULONG_PTR PhysicalAddress, - IN ULONG NumberOfPages) + IN ULONGLONG VirtualAddress, + IN ULONGLONG PhysicalAddress, + IN ULONGLONG NumberOfPages) { - SIZE_T PageFrameNumber; + ULONGLONG PageFrameNumber; PVOID Pml1, Pml2, Pml3; SIZE_T Pml1Entry, Pml2Entry, Pml3Entry; PHARDWARE_LEGACY_PTE LegacyPmlTable; diff --git a/boot/xtldr/includes/xtldr.hh b/boot/xtldr/includes/xtldr.hh index c8f71bd..bc42aab 100644 --- a/boot/xtldr/includes/xtldr.hh +++ b/boot/xtldr/includes/xtldr.hh @@ -177,15 +177,16 @@ class Memory IN SHORT PageMapLevel, IN PAGE_SIZE PageSize); STATIC XTCDECL EFI_STATUS MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, - IN OUT PVOID *MemoryMapAddress, + IN OUT PVOID *BaseAddress, + IN BOOLEAN IdentityMapping, IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine); STATIC XTCDECL EFI_STATUS MapPage(IN PXTBL_PAGE_MAPPING PageMap, - IN ULONG_PTR VirtualAddress, - IN ULONG_PTR PhysicalAddress, - IN ULONG NumberOfPages); + IN ULONGLONG VirtualAddress, + IN ULONGLONG PhysicalAddress, + IN ULONGLONG NumberOfPages); STATIC XTCDECL EFI_STATUS MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, - IN PVOID VirtualAddress, - IN PVOID PhysicalAddress, + IN ULONGLONG VirtualAddress, + IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType); STATIC XTCDECL PVOID PhysicalAddressToVirtual(IN PVOID PhysicalAddress, diff --git a/boot/xtldr/memory.cc b/boot/xtldr/memory.cc index 91ff66d..f3cc773 100644 --- a/boot/xtldr/memory.cc +++ b/boot/xtldr/memory.cc @@ -314,9 +314,12 @@ Memory::InitializePageMap(OUT PXTBL_PAGE_MAPPING PageMap, * @param PageMap * Supplies a pointer to the page mapping structure. * - * @param MemoryMapAddress + * @param BaseAddress * Supplies a virtual address, where EFI memory will be mapped. * + * @param IdentityMapping + * Specifies whether EFI non-free memory should be mapped by identity or sequential mapping. + * * @param GetMemoryTypeRoutine * Supplies a pointer to the routine which will be used to match EFI memory type to the OS memory type. * @@ -327,19 +330,20 @@ Memory::InitializePageMap(OUT PXTBL_PAGE_MAPPING PageMap, XTCDECL EFI_STATUS Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, - IN OUT PVOID *MemoryMapAddress, + IN OUT PVOID *BaseAddress, + IN BOOLEAN IdentityMapping, IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine) { + ULONGLONG MaxAddress, VirtualAddress; PEFI_MEMORY_DESCRIPTOR Descriptor; LOADER_MEMORY_TYPE MemoryType; PEFI_MEMORY_MAP MemoryMap; SIZE_T DescriptorCount; - PUCHAR VirtualAddress; EFI_STATUS Status; SIZE_T Index; /* Set virtual address as specified in argument */ - VirtualAddress = (PUCHAR)*MemoryMapAddress; + VirtualAddress = (ULONGLONG)*BaseAddress; /* Check if custom memory type routine is specified */ if(GetMemoryTypeRoutine == NULLPTR) @@ -367,8 +371,37 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, /* Iterate through all descriptors from the memory map */ for(Index = 0; Index < DescriptorCount; Index++) { - /* Make sure descriptor does not start beyond lowest physical page */ - if(Descriptor->PhysicalStart <= MAXUINT_PTR) + /* Check page map level */ + if(PageMap->PageMapLevel == 2) + { + /* Limit physical address to 4GB in legacy mode */ + MaxAddress = 0xFFFFFFFF; + } + else if(PageMap->PageMapLevel == 3) + { + /* Limit physical address to 64GB in PAE mode */ + MaxAddress = 0xFFFFFFFFFULL; + } + + /* Check page map level */ + if(PageMap->PageMapLevel == 2 || PageMap->PageMapLevel == 3) + { + /* Check if physical address starts beyond limit */ + if(Descriptor->PhysicalStart >= MaxAddress) + { + /* Go to the next descriptor */ + Descriptor = (PEFI_MEMORY_DESCRIPTOR)((PUCHAR)Descriptor + MemoryMap->DescriptorSize); + continue; + } + + /* Check if memory descriptor exceeds the lowest physical page */ + if(Descriptor->PhysicalStart + (Descriptor->NumberOfPages << EFI_PAGE_SHIFT) > MaxAddress) + { + /* Truncate memory descriptor to the lowest supported physical page */ + Descriptor->NumberOfPages = (MaxAddress - Descriptor->PhysicalStart) >> EFI_PAGE_SHIFT; + } + } + { /* Skip EFI reserved memory */ if(Descriptor->Type == EfiReservedMemoryType) @@ -378,25 +411,6 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, continue; } - /* Check if preparing page map level 2 (non-PAE i686) */ - if(PageMap->PageMapLevel == 2) - { - /* Check if physical address starts beyond 4GB */ - if(Descriptor->PhysicalStart > 0xFFFFFFFF) - { - /* Go to the next descriptor */ - Descriptor = (PEFI_MEMORY_DESCRIPTOR)((PUCHAR)Descriptor + MemoryMap->DescriptorSize); - continue; - } - - /* Check if memory descriptor exceeds the lowest physical page */ - if(Descriptor->PhysicalStart + (Descriptor->NumberOfPages << EFI_PAGE_SHIFT) > MAXULONG) - { - /* Truncate memory descriptor to the 4GB */ - Descriptor->NumberOfPages = (((ULONGLONG)MAXULONG + 1) - Descriptor->PhysicalStart) >> EFI_PAGE_SHIFT; - } - } - /* Convert EFI memory type into XTLDR memory type */ MemoryType = GetMemoryTypeRoutine((EFI_MEMORY_TYPE)Descriptor->Type); @@ -404,22 +418,32 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, if(MemoryType == LoaderFirmwareTemporary) { /* Map EFI firmware code */ - Status = MapVirtualMemory(PageMap, (PVOID)Descriptor->PhysicalStart, - (PVOID)Descriptor->PhysicalStart, Descriptor->NumberOfPages, MemoryType); + Status = MapVirtualMemory(PageMap, Descriptor->PhysicalStart, + Descriptor->PhysicalStart, Descriptor->NumberOfPages, MemoryType); } else if(MemoryType != LoaderFree) { - /* Add any non-free memory mapping */ - Status = MapVirtualMemory(PageMap, VirtualAddress, (PVOID)Descriptor->PhysicalStart, - Descriptor->NumberOfPages, MemoryType); + /* Check mapping strategy */ + if(IdentityMapping) + { + /* Add any non-free memory using identity mapping */ + Status = MapVirtualMemory(PageMap, Descriptor->PhysicalStart + KSEG0_BASE, Descriptor->PhysicalStart, + Descriptor->NumberOfPages, MemoryType); + } + else + { + /* Add any non-free memory using sequential mapping */ + Status = MapVirtualMemory(PageMap, VirtualAddress, Descriptor->PhysicalStart, + Descriptor->NumberOfPages, MemoryType); - /* Calculate next valid virtual address */ - VirtualAddress += Descriptor->NumberOfPages * EFI_PAGE_SIZE; + /* Update virtual address */ + VirtualAddress = VirtualAddress + (Descriptor->NumberOfPages * MM_PAGE_SIZE); + } } else { /* Map all other memory as loader free */ - Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)Descriptor->PhysicalStart, + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Descriptor->PhysicalStart, Descriptor->NumberOfPages, LoaderFree); } @@ -436,7 +460,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, } /* Always map first page */ - Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)0, 1, LoaderFirmwarePermanent); + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, 0, 1, LoaderFirmwarePermanent); if(Status != STATUS_EFI_SUCCESS) { /* Mapping failed */ @@ -444,7 +468,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, } /* Map BIOS ROM and VRAM */ - Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)0xA0000, 0x60, LoaderFirmwarePermanent); + Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, 0xA0000, 0x60, LoaderFirmwarePermanent); if(Status != STATUS_EFI_SUCCESS) { /* Mapping failed */ @@ -452,7 +476,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, } /* Store next valid virtual address and return success */ - *MemoryMapAddress = VirtualAddress; + *BaseAddress = (PVOID)VirtualAddress; return STATUS_EFI_SUCCESS; } @@ -481,13 +505,13 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, XTCDECL EFI_STATUS Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, - IN PVOID VirtualAddress, - IN PVOID PhysicalAddress, + IN ULONGLONG VirtualAddress, + IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType) { PXTBL_MEMORY_MAPPING Mapping1, Mapping2, Mapping3; - PVOID PhysicalAddressEnd, PhysicalAddress2End; + ULONGLONG PhysicalAddressEnd, PhysicalAddress2End; PLIST_ENTRY ListEntry, MappingListEntry; SIZE_T NumberOfMappedPages; EFI_STATUS Status; @@ -507,7 +531,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, Mapping1->MemoryType = MemoryType; /* Calculate the end of the physical address */ - PhysicalAddressEnd = (PVOID)((ULONG_PTR)PhysicalAddress + (NumberOfPages * EFI_PAGE_SIZE) - 1); + PhysicalAddressEnd = PhysicalAddress + (NumberOfPages * EFI_PAGE_SIZE) - 1; /* Iterate through all the mappings already set to insert new mapping at the correct place */ ListEntry = PageMap->MemoryMap.Flink; @@ -515,7 +539,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, { /* Take a mapping from the list and calculate its end of physical address */ Mapping2 = CONTAIN_RECORD(ListEntry, XTBL_MEMORY_MAPPING, ListEntry); - PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1); + PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1; /* Check if new mapping is a subset of an existing mapping */ if(Mapping1->PhysicalAddress >= Mapping2->PhysicalAddress && PhysicalAddressEnd <= PhysicalAddress2End) @@ -539,7 +563,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, } /* Calculate number of pages for this mapping */ - NumberOfMappedPages = ((PUCHAR)PhysicalAddress2End - (PUCHAR)PhysicalAddressEnd) / EFI_PAGE_SIZE; + NumberOfMappedPages = (PhysicalAddress2End - PhysicalAddressEnd) / EFI_PAGE_SIZE; if(NumberOfMappedPages > 0) { /* Pages associated to the mapping, allocate memory for it */ @@ -551,8 +575,8 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, } /* Set mapping fields and insert it on the top */ - Mapping3->PhysicalAddress = (PUCHAR)PhysicalAddressEnd + 1; - Mapping3->VirtualAddress = NULLPTR; + Mapping3->PhysicalAddress = PhysicalAddressEnd + 1; + Mapping3->VirtualAddress = (ULONGLONG)NULLPTR; Mapping3->NumberOfPages = NumberOfMappedPages; Mapping3->MemoryType = Mapping2->MemoryType; RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry); @@ -561,7 +585,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, /* Calculate number of pages and the end of the physical address */ Mapping2->NumberOfPages = ((PUCHAR)PhysicalAddressEnd + 1 - (PUCHAR)Mapping2->PhysicalAddress) / EFI_PAGE_SIZE; - PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1); + PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1; } /* Check if they overlap */ @@ -588,7 +612,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, /* Set mapping fields and insert it on the top */ Mapping3->PhysicalAddress = Mapping1->PhysicalAddress; - Mapping3->VirtualAddress = NULLPTR; + Mapping3->VirtualAddress = (ULONGLONG)NULLPTR; Mapping3->NumberOfPages = NumberOfMappedPages; Mapping3->MemoryType = Mapping2->MemoryType; RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry); @@ -597,7 +621,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap, /* Calculate number of pages and the end of the physical address */ Mapping2->NumberOfPages = ((PUCHAR)Mapping1->PhysicalAddress - (PUCHAR)Mapping2->PhysicalAddress) / EFI_PAGE_SIZE; - PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1); + PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1; } /* Check if mapping is really needed */ diff --git a/boot/xtldr/modules/pecoff/pecoff.cc b/boot/xtldr/modules/pecoff/pecoff.cc index 00f105a..7d9692c 100644 --- a/boot/xtldr/modules/pecoff/pecoff.cc +++ b/boot/xtldr/modules/pecoff/pecoff.cc @@ -729,11 +729,11 @@ PeCoff::RelocateLoadedImage(IN PPECOFF_IMAGE_CONTEXT Image) } else { - /* Check if loaded 32-bit PE32 image should be relocated */ /* Set relocation data directory and image base address */ DataDirectory = &Image->PeHeader->OptionalHeader32.DataDirectory[PECOFF_IMAGE_DIRECTORY_ENTRY_BASERELOC]; ImageBase = Image->PeHeader->OptionalHeader32.ImageBase; + /* Check if loaded 32-bit PE32 image should be relocated */ if(Image->PeHeader->OptionalHeader32.NumberOfRvaAndSizes <= PECOFF_IMAGE_DIRECTORY_ENTRY_BASERELOC || DataDirectory->VirtualAddress == 0 || DataDirectory->Size < sizeof(PECOFF_IMAGE_BASE_RELOCATION)) { diff --git a/boot/xtldr/modules/xtos_o/amd64/memory.cc b/boot/xtldr/modules/xtos_o/amd64/memory.cc index 02335aa..4ec271c 100644 --- a/boot/xtldr/modules/xtos_o/amd64/memory.cc +++ b/boot/xtldr/modules/xtos_o/amd64/memory.cc @@ -10,6 +10,21 @@ #include +/** + * Determines the appropriate EFI memory mapping strategy for the AMD64 architecture. + * + * @return This routine returns TRUE, what results in an identity mapping. + * + * @since XT 1.0 + */ +XTCDECL +BOOLEAN +Xtos::DetermineMappingStrategy() +{ + /* Use an identity mapping strategy */ + return TRUE; +} + /** * Determines the appropriate paging level (PML) for the AMD64 architecture. * diff --git a/boot/xtldr/modules/xtos_o/i686/memory.cc b/boot/xtldr/modules/xtos_o/i686/memory.cc index c2b2e1c..e4fbdf3 100644 --- a/boot/xtldr/modules/xtos_o/i686/memory.cc +++ b/boot/xtldr/modules/xtos_o/i686/memory.cc @@ -9,6 +9,21 @@ #include +/** + * Determines the appropriate EFI memory mapping strategy for the i686 architecture. + * + * @return This routine returns FALSE, what results in a sequential mapping. + * + * @since XT 1.0 + */ +XTCDECL +BOOLEAN +Xtos::DetermineMappingStrategy() +{ + /* Use a sequential mapping strategy */ + return FALSE; +} + /** * Determines the appropriate paging level (PML) for the i686 architecture. * @@ -59,9 +74,22 @@ EFI_STATUS Xtos::EnablePaging(IN PXTBL_PAGE_MAPPING PageMap) { EFI_STATUS Status; + ULONG_PTR SelfMapAddress; + + /* Initialize self map address */ + if(PageMap->PageMapLevel == 3) + { + /* For PML3 (PAE) use PTE base address */ + SelfMapAddress = MM_PTE_BASE; + } + else + { + /* For PML2 (PAE disabled) use legacy PDE base address */ + SelfMapAddress = MM_PDE_LEGACY_BASE; + } /* Build page map */ - Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, MM_PTE_BASE); + Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, SelfMapAddress); if(Status != STATUS_EFI_SUCCESS) { /* Failed to build page map */ diff --git a/boot/xtldr/modules/xtos_o/includes/xtos.hh b/boot/xtldr/modules/xtos_o/includes/xtos.hh index 4d70606..708ce14 100644 --- a/boot/xtldr/modules/xtos_o/includes/xtos.hh +++ b/boot/xtldr/modules/xtos_o/includes/xtos.hh @@ -39,6 +39,7 @@ class Xtos IN UINT NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType); STATIC XTCDECL LOADER_MEMORY_TYPE ConvertEfiMemoryType(IN EFI_MEMORY_TYPE EfiMemoryType); + STATIC XTCDECL BOOLEAN DetermineMappingStrategy(); STATIC XTCDECL ULONG DeterminePagingLevel(IN CONST PWCHAR Parameters); STATIC XTCDECL EFI_STATUS EnablePaging(IN PXTBL_PAGE_MAPPING PageMap); STATIC XTCDECL VOID GetDisplayInformation(OUT PSYSTEM_RESOURCE_FRAMEBUFFER FrameBufferResource, diff --git a/boot/xtldr/modules/xtos_o/xtos.cc b/boot/xtldr/modules/xtos_o/xtos.cc index 2003161..5b2b8a0 100644 --- a/boot/xtldr/modules/xtos_o/xtos.cc +++ b/boot/xtldr/modules/xtos_o/xtos.cc @@ -194,46 +194,63 @@ Xtos::GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap, IN PVOID *VirtualAddress, OUT PLIST_ENTRY MemoryDescriptorList) { + PLOADER_MEMORY_DESCRIPTOR Descriptor; + PXTBL_MEMORY_MAPPING MemoryMapping; EFI_PHYSICAL_ADDRESS Address; + PLIST_ENTRY ListEntry; EFI_STATUS Status; ULONGLONG Pages; + /* Calculate the number of pages required to store the memory descriptor array */ Pages = (ULONGLONG)EFI_SIZE_TO_PAGES((PageMap->MapSize + 1) * sizeof(LOADER_MEMORY_DESCRIPTOR)); + /* Allocate physical pages to hold the memory descriptor list */ Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, Pages, &Address); if(Status != STATUS_EFI_SUCCESS) { + /* Page allocation failed, return the status code */ return Status; } - Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)Address, Pages, LoaderMemoryData); + /* Create a virtual memory mapping for the allocated descriptor buffer */ + Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderMemoryData); if(Status != STATUS_EFI_SUCCESS) { + /* Release the allocated pages as the virtual mapping failed and return status code */ XtLdrProtocol->Memory.FreePages(Address, Pages); return Status; } - PVOID PhysicalBase = (PVOID)Address; + /* Initialize the descriptor pointer to the start of the allocated physical buffer */ + Descriptor = (PLOADER_MEMORY_DESCRIPTOR)Address; - PLIST_ENTRY ListEntry; + /* Get the first entry from the internal boot loader memory map */ ListEntry = PageMap->MemoryMap.Flink; + + /* Iterate through the internal memory map and populate the loader descriptor list */ while(ListEntry != &PageMap->MemoryMap) { - PXTBL_MEMORY_MAPPING MemoryMapping = CONTAIN_RECORD(ListEntry, XTBL_MEMORY_MAPPING, ListEntry); - PLOADER_MEMORY_DESCRIPTOR MemoryDescriptor = (PLOADER_MEMORY_DESCRIPTOR)Address; + /* Retrieve the internal memory mapping record from the current list entry */ + MemoryMapping = CONTAIN_RECORD(ListEntry, XTBL_MEMORY_MAPPING, ListEntry); - MemoryDescriptor->MemoryType = MemoryMapping->MemoryType; - MemoryDescriptor->BasePage = (UINT_PTR)MemoryMapping->PhysicalAddress / EFI_PAGE_SIZE; - MemoryDescriptor->PageCount = MemoryMapping->NumberOfPages; + /* Transfer memory type and address information to the kernel descriptor */ + Descriptor->MemoryType = MemoryMapping->MemoryType; + Descriptor->BasePage = (UINT_PTR)(MemoryMapping->PhysicalAddress / EFI_PAGE_SIZE); + Descriptor->PageCount = (ULONG)MemoryMapping->NumberOfPages; - XtLdrProtocol->LinkedList.InsertTail(MemoryDescriptorList, &MemoryDescriptor->ListEntry); + /* Link the entry */ + XtLdrProtocol->LinkedList.InsertTail(MemoryDescriptorList, &Descriptor->ListEntry); - Address = Address + sizeof(LOADER_MEMORY_DESCRIPTOR); + /* Move to the next slot in the allocated buffer */ + Descriptor++; ListEntry = ListEntry->Flink; } - XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, MemoryDescriptorList, PhysicalBase, *VirtualAddress); + /* Convert all physical link pointers in the list to their corresponding virtual addresses */ + XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, MemoryDescriptorList, (PVOID)Address, *VirtualAddress); + /* Advance the virtual address pointer to the next available free region and return success */ + *VirtualAddress = (PUINT8)*VirtualAddress + (Pages * EFI_PAGE_SIZE); return STATUS_EFI_SUCCESS; } @@ -266,7 +283,7 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap, { return Status; } - Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)Address, Pages, LoaderFirmwarePermanent); + Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderFirmwarePermanent); if(Status != STATUS_EFI_SUCCESS) { XtLdrProtocol->Memory.FreePages(Address, Pages); @@ -336,8 +353,8 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap, FrameBufferResource->Header.VirtualAddress = *VirtualAddress; /* Map frame buffer memory */ - XtLdrProtocol->Memory.MapVirtualMemory(PageMap, FrameBufferResource->Header.VirtualAddress, - FrameBufferResource->Header.PhysicalAddress, + XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)FrameBufferResource->Header.VirtualAddress, + (ULONGLONG)FrameBufferResource->Header.PhysicalAddress, FrameBufferPages, LoaderFirmwarePermanent); /* Close FrameBuffer protocol */ @@ -389,7 +406,7 @@ Xtos::InitializeApicBase(IN PXTBL_PAGE_MAPPING PageMap) } /* Map APIC base address */ - XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (PVOID)APIC_BASE, ApicBaseAddress, 1, LoaderFirmwarePermanent); + XtLdrProtocol->Memory.MapVirtualMemory(PageMap, APIC_BASE, (ULONGLONG)ApicBaseAddress, 1, LoaderFirmwarePermanent); return STATUS_EFI_SUCCESS; } @@ -456,7 +473,7 @@ Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap, ParametersSize); /* Map kernel initialization block */ - XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)LoaderBlock, + XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, (ULONGLONG)LoaderBlock, BlockPages, LoaderSystemBlock); /* Calculate next valid virtual address */ @@ -469,6 +486,9 @@ Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap, XtLdrProtocol->LinkedList.InitializeHead(&LoaderBlock->MemoryDescriptorListHead); GetMemoryDescriptorList(PageMap, VirtualAddress, &LoaderBlock->MemoryDescriptorListHead); + /* Set boot image size */ + LoaderBlock->BootImageSize = (PFN_NUMBER)(((ULONGLONG)*VirtualAddress - KSEG0_BASE) / EFI_PAGE_SIZE); + /* Return success */ return STATUS_EFI_SUCCESS; } @@ -607,11 +627,12 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir, PXTBL_FRAMEBUFFER_PROTOCOL FrameBufProtocol; PPECOFF_IMAGE_CONTEXT ImageContext = NULLPTR; PEFI_LOADED_IMAGE_PROTOCOL ImageProtocol; - PVOID VirtualAddress, VirtualMemoryArea; + PVOID VirtualAddress; PXT_ENTRY_POINT KernelEntryPoint; EFI_HANDLE ProtocolHandle; EFI_STATUS Status; XTBL_PAGE_MAPPING PageMap; + BOOLEAN IdentityMapping; /* Initialize XTOS startup sequence */ XtLdrProtocol->Debug.Print(L"Initializing XTOS startup sequence\n"); @@ -628,19 +649,30 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir, /* Close FrameBuffer protocol */ XtLdrProtocol->Protocol.Close(&ProtocolHandle, &FrameBufGuid); + /* Determine whether to use a sequential or an identity mapping strategy */ + IdentityMapping = DetermineMappingStrategy(); + /* Set base virtual memory area for the kernel mappings */ - VirtualMemoryArea = (PVOID)KSEG0_BASE; - VirtualAddress = (PVOID)(KSEG0_BASE + KSEG0_KERNEL_BASE); + VirtualAddress = (PVOID)(KSEG0_BASE); /* Initialize virtual memory mappings */ XtLdrProtocol->Memory.InitializePageMap(&PageMap, DeterminePagingLevel(Parameters->Parameters), Size4K); - Status = XtLdrProtocol->Memory.MapEfiMemory(&PageMap, &VirtualMemoryArea, NULLPTR); + /* Map all EFI memory regions */ + Status = XtLdrProtocol->Memory.MapEfiMemory(&PageMap, &VirtualAddress, IdentityMapping, NULLPTR); if(Status != STATUS_EFI_SUCCESS) { + /* Mapping failed */ return Status; } + /* Check mapping strategy */ + if(IdentityMapping) + { + /* Adjust virtual address to skip the identity-mapped physical range */ + VirtualAddress = (PVOID)((ULONGLONG)VirtualAddress + 0x800000000); + } + /* Load the kernel */ Status = LoadModule(BootDir, Parameters->KernelFile, VirtualAddress, LoaderSystemCode, &ImageContext); if(Status != STATUS_EFI_SUCCESS) @@ -650,8 +682,8 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir, } /* Add kernel image memory mapping */ - Status = XtLdrProtocol->Memory.MapVirtualMemory(&PageMap, ImageContext->VirtualAddress, - ImageContext->PhysicalAddress, ImageContext->ImagePages, + Status = XtLdrProtocol->Memory.MapVirtualMemory(&PageMap, (ULONGLONG)ImageContext->VirtualAddress, + (ULONGLONG)ImageContext->PhysicalAddress, ImageContext->ImagePages, LoaderSystemCode); if(Status != STATUS_EFI_SUCCESS) { diff --git a/sdk/xtdk/amd64/ketypes.h b/sdk/xtdk/amd64/ketypes.h index 1299161..ccb7d9d 100644 --- a/sdk/xtdk/amd64/ketypes.h +++ b/sdk/xtdk/amd64/ketypes.h @@ -108,9 +108,6 @@ /* Static Kernel-Mode address start */ #define KSEG0_BASE 0xFFFFF80000000000 -/* XTOS Kernel address base */ -#define KSEG0_KERNEL_BASE 0x0000000800000000 - /* XTOS Kernel stack size */ #define KERNEL_STACK_SIZE 0x8000 diff --git a/sdk/xtdk/amd64/mmtypes.h b/sdk/xtdk/amd64/mmtypes.h index 3b8db1f..7f7e84a 100644 --- a/sdk/xtdk/amd64/mmtypes.h +++ b/sdk/xtdk/amd64/mmtypes.h @@ -25,11 +25,11 @@ #define MM_PXE_BASE 0xFFFFF6FB7DBED000ULL /* Page directory and page base addresses for 5-level paging */ -#define MM_PTE_LA57_BASE 0xFFFF000000000000ULL -#define MM_PDE_LA57_BASE 0xFFFF010000000000ULL -#define MM_PPE_LA57_BASE 0xFFFF010800000000ULL -#define MM_PXE_LA57_BASE 0xFFFF010840000000ULL -#define MM_P5E_LA57_BASE 0xFFFF010840200000ULL +#define MM_PTE_LA57_BASE 0xFFED000000000000ULL +#define MM_PDE_LA57_BASE 0xFFEDF68000000000ULL +#define MM_PPE_LA57_BASE 0xFFEDF6FB40000000ULL +#define MM_PXE_LA57_BASE 0xFFEDF6FB7DA00000ULL +#define MM_P5E_LA57_BASE 0xFFEDF6FB7DBED000ULL /* PTE shift values */ #define MM_PTE_SHIFT 3 @@ -39,15 +39,56 @@ #define MM_PXI_SHIFT 39 #define MM_P5I_SHIFT 48 -/* Number of PTEs per page */ -#define MM_PTE_PER_PAGE 512 -#define MM_PDE_PER_PAGE 512 -#define MM_PPE_PER_PAGE 512 -#define MM_PXE_PER_PAGE 512 +/* PTE state flags */ +#define MM_PTE_VALID 0x0000000000000001ULL +#define MM_PTE_ACCESSED 0x0000000000000020ULL +#define MM_PTE_DIRTY 0x0000000000000040ULL + +/* PTE scope flags */ +#define MM_PTE_LARGE_PAGE 0x0000000000000080ULL +#define MM_PTE_GLOBAL 0x0000000000000100ULL + +/* PTE access flags */ +#define MM_PTE_NOACCESS 0x0000000000000000ULL +#define MM_PTE_READONLY 0x0000000000000000ULL +#define MM_PTE_EXECUTE 0x0000000000000000ULL +#define MM_PTE_EXECUTE_READ 0x0000000000000000ULL +#define MM_PTE_READWRITE 0x8000000000000002ULL +#define MM_PTE_WRITECOPY 0x8000000000000200ULL +#define MM_PTE_EXECUTE_READWRITE 0x0000000000000002ULL +#define MM_PTE_EXECUTE_WRITECOPY 0x0000000000000200ULL + +/* PTE protection flags */ +#define MM_PTE_NOEXECUTE 0x8000000000000000ULL +#define MM_PTE_GUARDED 0x8000000000000018ULL +#define MM_PTE_PROTECT 0x8000000000000612ULL + +/* PTE cache flags */ +#define MM_PTE_CACHE_ENABLE 0x0000000000000000ULL +#define MM_PTE_CACHE_DISABLE 0x0000000000000010ULL +#define MM_PTE_CACHE_WRITECOMBINED 0x0000000000000010ULL +#define MM_PTE_CACHE_WRITETHROUGH 0x0000000000000008ULL + +/* PTE software flags */ +#define MM_PTE_COPY_ON_WRITE 0x0000000000000200ULL +#define MM_PTE_PROTOTYPE 0x0000000000000400ULL +#define MM_PTE_TRANSITION 0x0000000000000800ULL + +/* PTE frame bits */ +#define MM_PTE_FRAME_BITS 57 + +/* PTE protection bits */ +#define MM_PTE_PROTECTION_BITS 5 + +/* Base address of the system page table */ +#define MM_SYSTEM_PTE_BASE KSEG0_BASE /* Minimum number of physical pages needed by the system */ #define MM_MINIMUM_PHYSICAL_PAGES 2048 +/* Number of system PTEs */ +#define MM_DEFAULT_NUMBER_SYSTEM_PTES 22000 + /* Default number of secondary colors */ #define MM_DEFAULT_SECONDARY_COLORS 64 @@ -63,6 +104,9 @@ /* Maximum physical address used by HAL allocations */ #define MM_MAXIMUM_PHYSICAL_ADDRESS 0x00000000FFFFFFFFULL +/* Highest system address */ +#define MM_HIGHEST_SYSTEM_ADDRESS 0xFFFFFFFFFFFFFFFFULL + /* Trampoline code address */ #define MM_TRAMPOLINE_ADDRESS 0x80000 @@ -252,6 +296,7 @@ typedef struct _MMPFN USHORT ReferenceCount; } e2; } u3; + ULONG UsedPageTableEntries; union { MMPTE OriginalPte; @@ -262,12 +307,11 @@ typedef struct _MMPFN ULONG_PTR EntireFrame; struct { - ULONG_PTR PteFrame:58; + ULONG_PTR PteFrame:57; ULONG_PTR InPageError:1; ULONG_PTR VerifierAllocation:1; ULONG_PTR AweAllocation:1; - ULONG_PTR LockCharged:1; - ULONG_PTR KernelStack:1; + ULONG_PTR Priority:3; ULONG_PTR MustBeCached:1; }; } u4; diff --git a/sdk/xtdk/bltypes.h b/sdk/xtdk/bltypes.h index 3ea3355..3cc2378 100644 --- a/sdk/xtdk/bltypes.h +++ b/sdk/xtdk/bltypes.h @@ -100,9 +100,9 @@ typedef VOID (XTCDECL *PBL_LLIST_INITIALIZE_HEAD)(IN PLIST_ENTRY ListHead); typedef VOID (XTCDECL *PBL_LLIST_INSERT_HEAD)(IN OUT PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry); typedef VOID (XTCDECL *PBL_LLIST_INSERT_TAIL)(IN OUT PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry); typedef VOID (XTCDECL *PBL_LLIST_REMOVE_ENTRY)(IN PLIST_ENTRY Entry); -typedef EFI_STATUS (XTCDECL *PBL_MAP_EFI_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN OUT PVOID *MemoryMapAddress, IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine); -typedef EFI_STATUS (XTCDECL *PBL_MAP_PAGE)(IN PXTBL_PAGE_MAPPING PageMap, IN ULONG_PTR VirtualAddress, IN ULONG_PTR PhysicalAddress, IN ULONG NumberOfPages); -typedef EFI_STATUS (XTCDECL *PBL_MAP_VIRTUAL_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN PVOID VirtualAddress, IN PVOID PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType); +typedef EFI_STATUS (XTCDECL *PBL_MAP_EFI_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN OUT PVOID *BaseAddress, IN BOOLEAN IdentityMapping, IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine); +typedef EFI_STATUS (XTCDECL *PBL_MAP_PAGE)(IN PXTBL_PAGE_MAPPING PageMap, IN ULONGLONG VirtualAddress, IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages); +typedef EFI_STATUS (XTCDECL *PBL_MAP_VIRTUAL_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN ULONGLONG VirtualAddress, IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType); typedef VOID (XTAPI *PBL_MOVE_MEMORY)(IN OUT PVOID Destination, IN PCVOID Source, IN SIZE_T Length); typedef EFI_STATUS (XTCDECL *PBL_OPEN_VOLUME)(IN PEFI_DEVICE_PATH_PROTOCOL DevicePath, OUT PEFI_HANDLE DiskHandle, OUT PEFI_FILE_HANDLE *FsHandle); typedef EFI_STATUS (XTCDECL *PBL_OPEN_PROTOCOL)(OUT PEFI_HANDLE Handle, OUT PVOID *ProtocolHandler, IN PEFI_GUID ProtocolGuid); @@ -232,8 +232,8 @@ typedef struct _XTBL_KNOWN_BOOT_PROTOCOL typedef struct _XTBL_MEMORY_MAPPING { LIST_ENTRY ListEntry; - PVOID VirtualAddress; - PVOID PhysicalAddress; + ULONGLONG VirtualAddress; + ULONGLONG PhysicalAddress; ULONGLONG NumberOfPages; LOADER_MEMORY_TYPE MemoryType; } XTBL_MEMORY_MAPPING, *PXTBL_MEMORY_MAPPING; diff --git a/sdk/xtdk/i686/ketypes.h b/sdk/xtdk/i686/ketypes.h index f62ff74..32697c4 100644 --- a/sdk/xtdk/i686/ketypes.h +++ b/sdk/xtdk/i686/ketypes.h @@ -128,9 +128,6 @@ /* Static Kernel-Mode address start */ #define KSEG0_BASE 0x80000000 -/* XTOS Kernel address base */ -#define KSEG0_KERNEL_BASE 0x01800000 - /* XTOS Kernel stack size */ #define KERNEL_STACK_SIZE 0x4000 diff --git a/sdk/xtdk/i686/mmtypes.h b/sdk/xtdk/i686/mmtypes.h index 027fc1a..4445543 100644 --- a/sdk/xtdk/i686/mmtypes.h +++ b/sdk/xtdk/i686/mmtypes.h @@ -35,9 +35,58 @@ #define MM_PTE_LEGACY_SHIFT 2 #define MM_PDI_LEGACY_SHIFT 22 +/* PTE state flags */ +#define MM_PTE_VALID 0x00000001 +#define MM_PTE_ACCESSED 0x00000020 +#define MM_PTE_DIRTY 0x00000040 + +/* PTE scope flags */ +#define MM_PTE_LARGE_PAGE 0x00000080 +#define MM_PTE_GLOBAL 0x00000100 + +/* PTE access flags */ +#define MM_PTE_NOACCESS 0x00000000 +#define MM_PTE_READONLY 0x00000000 +#define MM_PTE_EXECUTE 0x00000000 +#define MM_PTE_EXECUTE_READ 0x00000000 +#define MM_PTE_READWRITE 0x00000002 +#define MM_PTE_WRITECOPY 0x00000200 +#define MM_PTE_EXECUTE_READWRITE 0x00000002 +#define MM_PTE_EXECUTE_WRITECOPY 0x00000200 + +/* PTE protection flags */ +#define MM_PTE_NOEXECUTE 0x00000000 +#define MM_PTE_GUARDED 0x00000018 +#define MM_PTE_PROTECT 0x00000612 + +/* PTE cache flags */ +#define MM_PTE_CACHE_ENABLE 0x00000000 +#define MM_PTE_CACHE_DISABLE 0x00000010 +#define MM_PTE_CACHE_WRITECOMBINED 0x00000010 +#define MM_PTE_CACHE_WRITETHROUGH 0x00000008 + +/* PTE software flags */ +#define MM_PTE_COPY_ON_WRITE 0x00000200 +#define MM_PTE_PROTOTYPE 0x00000400 +#define MM_PTE_TRANSITION 0x00000800 + +/* PTE frame bits */ +#define MM_PTE_FRAME_BITS 25 + +/* PTE protection bits */ +#define MM_PTE_PROTECTION_BITS 5 + +/* Base address of the system page table */ +#define MM_SYSTEM_PTE_BASE NULLPTR + /* Minimum number of physical pages needed by the system */ #define MM_MINIMUM_PHYSICAL_PAGES 1100 +/* Number of system PTEs */ +#define MM_MINIMUM_NUMBER_SYSTEM_PTES 7000 +#define MM_DEFAULT_NUMBER_SYSTEM_PTES 11000 +#define MM_MAXIMUM_NUMBER_SYSTEM_PTES 22000 + /* Default number of secondary colors */ #define MM_DEFAULT_SECONDARY_COLORS 64 @@ -53,6 +102,9 @@ /* Maximum physical address used by HAL allocations */ #define MM_MAXIMUM_PHYSICAL_ADDRESS 0xFFFFFFFF +/* Highest system address */ +#define MM_HIGHEST_SYSTEM_ADDRESS 0xFFFFFFFF + /* Trampoline code address */ #define MM_TRAMPOLINE_ADDRESS 0x80000 @@ -106,7 +158,6 @@ typedef struct _HARDWARE_MODERN_PTE /* Generic Page Table entry union to abstract PML2 and PML3 formats */ typedef union _HARDWARE_PTE { - ULONGLONG Long; HARDWARE_LEGACY_PTE Pml2; HARDWARE_MODERN_PTE Pml3; } HARDWARE_PTE, *PHARDWARE_PTE; @@ -201,12 +252,12 @@ typedef struct _MMPML2_PTE_TRANSITION typedef union _MMPML2_PTE { ULONG Long; - HARDWARE_PTE Flush; - MMPML2_PTE_HARDWARE Hard; - MMPML2_PTE_PROTOTYPE Proto; - MMPML2_PTE_SOFTWARE Soft; - MMPML2_PTE_TRANSITION Trans; - MMPML2_PTE_SUBSECTION Subsect; + HARDWARE_LEGACY_PTE Flush; + MMPML2_PTE_HARDWARE Hardware; + MMPML2_PTE_PROTOTYPE Prototype; + MMPML2_PTE_SOFTWARE Software; + MMPML2_PTE_TRANSITION Transition; + MMPML2_PTE_SUBSECTION Subsection; MMPML2_PTE_LIST List; } MMPML2_PTE, *PMMPML2_PTE; @@ -296,7 +347,7 @@ typedef struct _MMPML3_PTE_TRANSITION typedef union _MMPML3_PTE { ULONGLONG Long; - HARDWARE_PTE Flush; + HARDWARE_MODERN_PTE Flush; MMPML3_PTE_HARDWARE Hardware; MMPML3_PTE_PROTOTYPE Prototype; MMPML3_PTE_SOFTWARE Software; @@ -308,7 +359,6 @@ typedef union _MMPML3_PTE /* Generic Page Table Entry union to abstract PML2 and PML3 formats */ typedef union _MMPTE { - ULONGLONG Long; MMPML2_PTE Pml2; MMPML3_PTE Pml3; } MMPTE, *PMMPTE; @@ -339,6 +389,7 @@ typedef struct _MMPFN USHORT ReferenceCount; } e2; } u3; + ULONG UsedPageTableEntries; union { MMPTE OriginalPte; @@ -349,12 +400,11 @@ typedef struct _MMPFN ULONG_PTR EntireFrame; struct { - ULONG_PTR PteFrame:26; + ULONG_PTR PteFrame:25; ULONG_PTR InPageError:1; ULONG_PTR VerifierAllocation:1; ULONG_PTR AweAllocation:1; - ULONG_PTR LockCharged:1; - ULONG_PTR KernelStack:1; + ULONG_PTR Priority:3; ULONG_PTR MustBeCached:1; }; } u4; diff --git a/sdk/xtdk/mmtypes.h b/sdk/xtdk/mmtypes.h index 2cddd97..22dac32 100644 --- a/sdk/xtdk/mmtypes.h +++ b/sdk/xtdk/mmtypes.h @@ -4,6 +4,7 @@ * FILE: sdk/xtdk/mmtypes.h * DESCRIPTION: Memory management data structures * DEVELOPERS: Rafal Kupiec + * Aiken Harris */ #ifndef __XTDK_MMTYPES_H @@ -13,6 +14,72 @@ #include ARCH_HEADER(xtstruct.h) +/* Number of hyper space pages */ +#define MM_HYPERSPACE_PAGE_COUNT 255 + +/* Number of free page list heads */ +#define MM_MAX_FREE_PAGE_LIST_HEADS 4 + +/* Number of paging colors */ +#define MM_PAGING_COLORS 64 + +/* PTE frame mask definition */ +#define MM_PFN_PTE_FRAME (((ULONG_PTR)1 << MM_PTE_FRAME_BITS) - 1) + +/* Memory manager pool type mask definition */ +#define MM_POOL_TYPE_MASK 1 + +/* Number of reserved zeroed PTEs */ +#define MM_RESERVED_ZERO_PTES 32 + +/* Memory manager page lists */ +typedef enum _MMPAGELISTS +{ + ZeroedPageList = 0, + FreePageList = 1, + StandbyPageList = 2, + ModifiedPageList = 3, + ModifiedReadOnlyPageList = 4, + BadPageList = 5, + ActiveAndValid = 6, + TransitionPage = 7 +} MMPAGELISTS, *PMMPAGELISTS; + +/* Page cache attributes */ +typedef enum _MMPFN_CACHE_ATTRIBUTE +{ + PfnNonCached, + PfnCached, + PfnWriteCombined, + PfnNotMapped +} MMPFN_CACHE_ATTRIBUTE, *PMMPFN_CACHE_ATTRIBUTE; + +/* Memory Manager pool types */ +typedef enum _MMPOOL_TYPE +{ + NonPagedPool = 0, + PagedPool = 1, + NonPagedPoolMustSucceed = 2, + NonPagedPoolCacheAligned = 4, + PagedPoolCacheAligned = 5, + NonPagedPoolCacheAlignedMustS = 6, + MaxPoolType = 7, + NonPagedPoolSession = 32, + PagedPoolSession = 33, + NonPagedPoolMustSucceedSession = 34, + NonPagedPoolCacheAlignedSession = 36, + PagedPoolCacheAlignedSession = 37, + NonPagedPoolCacheAlignedMustSSession = 38 +} MMPOOL_TYPE, *PMMPOOL_TYPE; + +/* Page table pool types */ +typedef enum _MMSYSTEM_PTE_POOL_TYPE +{ + SystemPteSpace, + NonPagedPoolExpansion, + MaximumPtePoolTypes +} MMSYSTEM_PTE_POOL_TYPE, *PMMSYSTEM_PTE_POOL_TYPE; + /* Page map routines structure definition */ typedef CONST STRUCT _CMMPAGEMAP_ROUTINES { @@ -30,6 +97,57 @@ typedef struct _MMCOLOR_TABLES ULONG_PTR Count; } MMCOLOR_TABLES, *PMMCOLOR_TABLES; +/* Free pool entry structure definition */ +typedef struct _MMFREE_POOL_ENTRY +{ + LIST_ENTRY List; + PFN_COUNT Size; +} MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY; + +/* Memory layout structure definition */ +typedef struct _MMMEMORY_LAYOUT +{ + PMMPFN PfnDatabase; + PFN_NUMBER PfnDatabaseSize; + PVOID SelfMapAddress; + PVOID HardwarePoolStart; + PVOID HardwarePoolEnd; + PVOID HyperSpaceStart; + PVOID HyperSpaceEnd; + PVOID LoaderMappingsStart; + PVOID LoaderMappingsEnd; + PFN_NUMBER LoaderMappingsSize; + PVOID NonCanonicalStart; + PVOID NonCanonicalEnd; + PVOID NonPagedPoolStart; + PVOID NonPagedPoolEnd; + PFN_NUMBER NonPagedPoolSize; + PVOID NonPagedExpansionPoolStart; + PVOID NonPagedExpansionPoolEnd; + PFN_NUMBER NonPagedExpansionPoolSize; + PVOID NonPagedSystemPoolStart; + PVOID NonPagedSystemPoolEnd; + PFN_NUMBER NonPagedSystemPoolSize; + PVOID PagedPoolStart; + PVOID PagedPoolEnd; + PFN_NUMBER PagedPoolSize; + PVOID ReservedSystemPoolStart; + PVOID ReservedSystemPoolEnd; + PVOID SessionSpaceStart; + PVOID SessionSpaceEnd; + PFN_NUMBER SessionSpaceSize; + PVOID SharedSystemPageStart; + PVOID SharedSystemPageEnd; + PVOID SystemCacheStart; + PVOID SystemCacheEnd; + PVOID SystemWorkingSetStart; + PVOID SystemWorkingSetEnd; + PVOID UserSpaceStart; + PVOID UserSpaceEnd; + PVOID PteSpaceStart; + PVOID PteSpaceEnd; +} MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT; + /* Page Frame Entry structure definition */ typedef struct _MMPFNENTRY { @@ -45,4 +163,13 @@ typedef struct _MMPFNENTRY USHORT ParityError:1; } MMPFNENTRY, *PMMPFNENTRY; +/* Page Frame List structure definition */ +typedef struct _MMPFNLIST +{ + PFN_NUMBER Total; + MMPAGELISTS ListName; + PFN_NUMBER Flink; + PFN_NUMBER Blink; +} MMPFNLIST, *PMMPFNLIST; + #endif /* __XTDK_MMTYPES_H */ diff --git a/sdk/xtdk/xtbase.h b/sdk/xtdk/xtbase.h index eb6790d..96282c7 100644 --- a/sdk/xtdk/xtbase.h +++ b/sdk/xtdk/xtbase.h @@ -29,6 +29,9 @@ typedef UCHAR KRUNLEVEL, *PKRUNLEVEL; /* Spin locks synchronization mechanism */ typedef ULONG_PTR KSPIN_LOCK, *PKSPIN_LOCK; +/* Page Frame Number count */ +typedef ULONG PFN_COUNT; + /* Page Frame Number */ typedef ULONG_PTR PFN_NUMBER, *PPFN_NUMBER; diff --git a/sdk/xtdk/xtdefs.h b/sdk/xtdk/xtdefs.h index a573847..62e3012 100644 --- a/sdk/xtdk/xtdefs.h +++ b/sdk/xtdk/xtdefs.h @@ -74,6 +74,10 @@ /* Macro for calculating size of a field in the structure */ #define FIELD_SIZE(Structure, Field) (sizeof(((Structure *)0)->Field)) +/* Macros for calculating minimum and maximum of two values */ +#define MIN(A, B) (((A) < (B)) ? (A) : (B)) +#define MAX(A, B) (((A) > (B)) ? (A) : (B)) + /* Macro that page-aligns a virtual address */ #define PAGE_ALIGN(VirtualAddress) ((PVOID)((ULONG_PTR)VirtualAddress & ~MM_PAGE_MASK)) diff --git a/sdk/xtdk/xtfw.h b/sdk/xtdk/xtfw.h index 1a4642a..fbe3965 100644 --- a/sdk/xtdk/xtfw.h +++ b/sdk/xtdk/xtfw.h @@ -107,6 +107,7 @@ typedef struct _KERNEL_INITIALIZATION_BLOCK ULONG BlockVersion; ULONG ProtocolVersion; PWCHAR KernelParameters; + PFN_NUMBER BootImageSize; LIST_ENTRY LoadOrderListHead; LIST_ENTRY MemoryDescriptorListHead; LIST_ENTRY BootDriverListHead; diff --git a/sdk/xtdk/xtstruct.h b/sdk/xtdk/xtstruct.h index 62fa2f8..7f95d8e 100644 --- a/sdk/xtdk/xtstruct.h +++ b/sdk/xtdk/xtstruct.h @@ -48,6 +48,10 @@ typedef enum _KTHREAD_STATE KTHREAD_STATE, *PKTHREAD_STATE; typedef enum _KTIMER_TYPE KTIMER_TYPE, *PKTIMER_TYPE; typedef enum _KUBSAN_DATA_TYPE KUBSAN_DATA_TYPE, *PKUBSAN_DATA_TYPE; typedef enum _LOADER_MEMORY_TYPE LOADER_MEMORY_TYPE, *PLOADER_MEMORY_TYPE; +typedef enum _MMPAGELISTS MMPAGELISTS, *PMMPAGELISTS; +typedef enum _MMPFN_CACHE_ATTRIBUTE MMPFN_CACHE_ATTRIBUTE, *PMMPFN_CACHE_ATTRIBUTE; +typedef enum _MMPOOL_TYPE MMPOOL_TYPE, *PMMPOOL_TYPE; +typedef enum _MMSYSTEM_PTE_POOL_TYPE MMSYSTEM_PTE_POOL_TYPE, *PMMSYSTEM_PTE_POOL_TYPE; typedef enum _MODE MODE, *PMODE; typedef enum _RTL_VARIABLE_TYPE RTL_VARIABLE_TYPE, *PRTL_VARIABLE_TYPE; typedef enum _SYSTEM_FIRMWARE_TYPE SYSTEM_FIRMWARE_TYPE, *PSYSTEM_FIRMWARE_TYPE; @@ -274,7 +278,10 @@ typedef struct _LOADER_INFORMATION_BLOCK LOADER_INFORMATION_BLOCK, *PLOADER_INFO typedef struct _LOADER_MEMORY_DESCRIPTOR LOADER_MEMORY_DESCRIPTOR, *PLOADER_MEMORY_DESCRIPTOR; typedef struct _M128 M128, *PM128; typedef struct _MMCOLOR_TABLES MMCOLOR_TABLES, *PMMCOLOR_TABLES; +typedef struct _MMFREE_POOL_ENTRY MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY; +typedef struct _MMMEMORY_LAYOUT MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT; typedef struct _MMPFNENTRY MMPFNENTRY, *PMMPFNENTRY; +typedef struct _MMPFNLIST MMPFNLIST, *PMMPFNLIST; typedef struct _PCAT_FIRMWARE_INFORMATION PCAT_FIRMWARE_INFORMATION, *PPCAT_FIRMWARE_INFORMATION; typedef struct _PCI_BRIDGE_CONTROL_REGISTER PCI_BRIDGE_CONTROL_REGISTER, *PPCI_BRIDGE_CONTROL_REGISTER; typedef struct _PCI_COMMON_CONFIG PCI_COMMON_CONFIG, *PPCI_COMMON_CONFIG; diff --git a/xtoskrnl/CMakeLists.txt b/xtoskrnl/CMakeLists.txt index 1080ff6..3e378b2 100644 --- a/xtoskrnl/CMakeLists.txt +++ b/xtoskrnl/CMakeLists.txt @@ -51,14 +51,22 @@ list(APPEND XTOSKRNL_SOURCE ${XTOSKRNL_SOURCE_DIR}/ke/spinlock.cc ${XTOSKRNL_SOURCE_DIR}/ke/sysres.cc ${XTOSKRNL_SOURCE_DIR}/ke/timer.cc - ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/init.cc + ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/alloc.cc + ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/mmgr.cc ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pagemap.cc ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/paging.cc + ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfault.cc + ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfn.cc + ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pte.cc + ${XTOSKRNL_SOURCE_DIR}/mm/alloc.cc + ${XTOSKRNL_SOURCE_DIR}/mm/colors.cc ${XTOSKRNL_SOURCE_DIR}/mm/data.cc ${XTOSKRNL_SOURCE_DIR}/mm/hlpool.cc - ${XTOSKRNL_SOURCE_DIR}/mm/init.cc ${XTOSKRNL_SOURCE_DIR}/mm/kpool.cc + ${XTOSKRNL_SOURCE_DIR}/mm/mmgr.cc ${XTOSKRNL_SOURCE_DIR}/mm/paging.cc + ${XTOSKRNL_SOURCE_DIR}/mm/pfn.cc + ${XTOSKRNL_SOURCE_DIR}/mm/pte.cc ${XTOSKRNL_SOURCE_DIR}/po/idle.cc ${XTOSKRNL_SOURCE_DIR}/rtl/${ARCH}/dispatch.cc ${XTOSKRNL_SOURCE_DIR}/rtl/${ARCH}/exsup.cc @@ -84,11 +92,12 @@ add_library(libxtos ${XTOSKRNL_SOURCE}) # Link kernel executable add_executable(xtoskrnl - ${XTOSKRNL_SOURCE} ${CMAKE_CURRENT_BINARY_DIR}/xtoskrnl.def) # Add linker libraries -target_link_libraries(xtoskrnl) +target_link_libraries(xtoskrnl + PRIVATE + libxtos) # Set proper binary name and install target set_target_properties(xtoskrnl PROPERTIES SUFFIX .exe) diff --git a/xtoskrnl/README.md b/xtoskrnl/README.md index 9112f64..e9600bf 100644 --- a/xtoskrnl/README.md +++ b/xtoskrnl/README.md @@ -4,6 +4,18 @@ within the XTOS kernel space. It is responsible for various core services, such management, and process scheduling. The kernel contains the scheduler (sometimes referred to as the Dispatcher), the cache, object, and memory managers, the security manager, and other executive components described below. + +## Kernel Parameters +Kernel parameters are XTOS boot-time options used to ensure proper initialization and handling of hardware peripherals. +These parameters can be configured either temporarily by editing the boot entry in the bootloader’s selection menu, or +permanently by modifying the XTLDR configuration file. + +The following is a consolidated list of available kernel parameters: + * **NOXPA**: Disables PAE or LA57 support, depending on the CPU architecture. This parameter is handled by the + bootloader, which configures paging and selects the appropriate Page Map Level (PML) before transferring control to + the kernel. + +## Source Code The source code of the kernel is organized into subsystem-specific directories. Each directory name also defines the corresponding C++ namespace in which the subsystem's classes and routines reside. These subsystems include: @@ -68,8 +80,8 @@ routine: For all C++ code inside the kernel the naming model has evolved. Consider the **KE::KThread::InitializeThread()** routine: * **KE** - The namespace replaces the prefix and indicates the subsystem. Namespaces are written in uppercase and no - longer use the trailing p for private routines, because classes use C++ visibility to control access. + longer use the trailing p for private routines, because classes use C++ visibility to control access. * **KThread** - Within each namespace, related functionality is grouped into classes, which encapsulate variables and - methods. + methods. * **InitializeThread** - Method names follow the `` pattern. \ No newline at end of file diff --git a/xtoskrnl/includes/mm.hh b/xtoskrnl/includes/mm.hh index 8dbdfb5..6045afe 100644 --- a/xtoskrnl/includes/mm.hh +++ b/xtoskrnl/includes/mm.hh @@ -12,10 +12,15 @@ #include #include XTOS_ARCH_HEADER(mm, pagemap.hh) +#include XTOS_ARCH_HEADER(mm, paging.hh) +#include XTOS_ARCH_HEADER(mm, pte.hh) +#include +#include #include -#include #include -#include +#include +#include +#include #endif /* __XTOSKRNL_MM_HH */ diff --git a/xtoskrnl/includes/mm/alloc.hh b/xtoskrnl/includes/mm/alloc.hh new file mode 100644 index 0000000..2f3dc5b --- /dev/null +++ b/xtoskrnl/includes/mm/alloc.hh @@ -0,0 +1,39 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/alloc.hh + * DESCRIPTION: Memory manager pool allocation + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_ALLOC_HH +#define __XTOSKRNL_MM_ALLOC_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Allocator + { + private: + STATIC LIST_ENTRY NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS]; + + public: + STATIC XTAPI XTSTATUS AllocatePages(IN MMPOOL_TYPE PoolType, + IN SIZE_T Bytes, + OUT PVOID *Memory); + STATIC XTAPI VOID InitializeNonPagedPool(VOID); + STATIC XTAPI VOID InitializePagedPool(VOID); + + private: + STATIC XTAPI XTSTATUS AllocateNonPagedPoolPages(IN PFN_COUNT Pages, + OUT PVOID *Memory); + STATIC XTAPI XTSTATUS AllocatePagedPoolPages(IN PFN_COUNT Pages, + OUT PVOID *Memory); + STATIC XTAPI VOID MapNonPagedPool(VOID); + }; +} + +#endif /* __XTOSKRNL_MM_ALLOC_HH */ diff --git a/xtoskrnl/includes/mm/amd64/pagemap.hh b/xtoskrnl/includes/mm/amd64/pagemap.hh index 79a4418..0d56e85 100644 --- a/xtoskrnl/includes/mm/amd64/pagemap.hh +++ b/xtoskrnl/includes/mm/amd64/pagemap.hh @@ -1,13 +1,13 @@ /** * PROJECT: ExectOS * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/includes/mm/pagemap.hh + * FILE: xtoskrnl/includes/mm/amd64/pagemap.hh * DESCRIPTION: Low-level support for page map manipulation * DEVELOPERS: Aiken Harris */ -#ifndef __XTOSKRNL_MM_PAGEMAP_HH -#define __XTOSKRNL_MM_PAGEMAP_HH +#ifndef __XTOSKRNL_MM_AMD64_PAGEMAP_HH +#define __XTOSKRNL_MM_AMD64_PAGEMAP_HH #include @@ -21,33 +21,76 @@ namespace MM MMPAGEMAP_INFO PageMapInfo; public: - XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer); - XTAPI PMMP5E GetP5eAddress(PVOID Address); - XTAPI PMMPDE GetPdeAddress(PVOID Address); - XTAPI PMMPPE GetPpeAddress(PVOID Address); - XTAPI PMMPTE GetPteAddress(PVOID Address); - XTAPI PMMPXE GetPxeAddress(PVOID Address); - virtual XTAPI VOID InitializePageMapInfo(VOID) = 0; - XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer); - XTAPI VOID SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable); - XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough); + XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN LONG Count); + XTAPI VOID ClearPte(IN PMMPTE PtePointer); + XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte); + XTAPI PMMPTE GetNextPte(IN PMMPTE Pte); + XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte); + XTAPI PMMP5E GetP5eAddress(IN PVOID Address); + XTAPI ULONG GetP5eOffset(IN PVOID Address); + XTAPI PVOID GetP5eVirtualAddress(IN PMMP5E P5ePointer); + XTAPI USHORT GetPageMapLevel(); + XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte); + XTAPI PMMPDE GetPdeAddress(IN PVOID Address); + XTAPI ULONG GetPdeOffset(IN PVOID Address); + VIRTUAL XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer) = 0; + XTAPI PMMPPE GetPpeAddress(IN PVOID Address); + XTAPI ULONG GetPpeOffset(IN PVOID Address); + VIRTUAL XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer) = 0; + XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer); + XTAPI PMMPTE GetPteAddress(IN PVOID Address); + XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte); + XTAPI ULONG GetPteOffset(IN PVOID Address); + XTAPI ULONG GetPteSize(VOID); + XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer); + VIRTUAL XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer) = 0; + XTAPI PMMPXE GetPxeAddress(IN PVOID Address); + XTAPI ULONG GetPxeOffset(IN PVOID Address); + VIRTUAL XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer) = 0; + XTAPI BOOLEAN GetXpaStatus(); + VIRTUAL XTAPI VOID InitializePageMapInfo(VOID) = 0; + XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer); + XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value); + XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes); + XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough); + XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection); + XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value); } PAGEMAP, *PPAGEMAP; class PageMapBasic final : public PageMap { public: + XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer); + XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); + XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer); XTAPI VOID InitializePageMapInfo(VOID); }; class PageMapXpa final : public PageMap { public: + XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer); + XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); + XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer); XTAPI VOID InitializePageMapInfo(VOID); }; } -#endif /* __XTOSKRNL_MM_PAGEMAP_HH */ +#endif /* __XTOSKRNL_MM_AMD64_PAGEMAP_HH */ diff --git a/xtoskrnl/includes/mm/amd64/paging.hh b/xtoskrnl/includes/mm/amd64/paging.hh new file mode 100644 index 0000000..a0e5062 --- /dev/null +++ b/xtoskrnl/includes/mm/amd64/paging.hh @@ -0,0 +1,79 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/amd64/paging.hh + * DESCRIPTION: Low level page management support for AMD64 + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_AMD64_PAGING_HH +#define __XTOSKRNL_MM_AMD64_PAGING_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Paging + { + private: + STATIC PPAGEMAP PmlRoutines; + + public: + STATIC XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN LONG Count); + STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer); + STATIC XTAPI VOID FlushTlb(VOID); + STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte); + STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte); + STATIC XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte); + STATIC XTAPI PMMP5E GetP5eAddress(IN PVOID Address); + STATIC XTAPI PVOID GetP5eVirtualAddress(IN PMMP5E P5ePointer); + STATIC XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte); + STATIC XTAPI USHORT GetPageMapLevel(); + STATIC XTAPI PMMPDE GetPdeAddress(IN PVOID Address); + STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address); + STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer); + STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer); + STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address); + STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte); + STATIC XTAPI ULONG GetPteSize(VOID); + STATIC XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer); + STATIC XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer); + STATIC XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer); + STATIC XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); + STATIC XTAPI PMMPXE GetPxeAddress(IN PVOID Address); + STATIC XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer); + STATIC XTAPI BOOLEAN GetXpaStatus(VOID); + STATIC XTAPI VOID InitializePageMapSupport(VOID); + STATIC XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer); + STATIC XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value); + STATIC XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value); + STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask); + STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes); + STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough); + STATIC XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value); + STATIC XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection); + STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address, + IN ULONG Size); + + private: + STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID); + STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID); + STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID); + }; +} + +#endif /* __XTOSKRNL_MM_AMD64_PAGING_HH */ diff --git a/xtoskrnl/includes/mm/amd64/pte.hh b/xtoskrnl/includes/mm/amd64/pte.hh new file mode 100644 index 0000000..3f9c3ff --- /dev/null +++ b/xtoskrnl/includes/mm/amd64/pte.hh @@ -0,0 +1,69 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/amd64/pte.hh + * DESCRIPTION: Page Table Entry (PTE) for AMD64 support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_AMD64_PTE_HH +#define __XTOSKRNL_MM_AMD64_PTE_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Pte + { + private: + STATIC MMPTE FirstSystemFreePte[MaximumPtePoolTypes]; + STATIC PMMPTE SystemPteBase; + STATIC PMMPTE SystemPtesEnd[MaximumPtePoolTypes]; + STATIC PMMPTE SystemPtesStart[MaximumPtePoolTypes]; + STATIC PFN_COUNT TotalSystemFreePtes[MaximumPtePoolTypes]; + STATIC MMPTE ValidPte; + + public: + STATIC XTAPI BOOLEAN AddressValid(IN PVOID VirtualAddress); + STATIC XTAPI PFN_COUNT GetPtesPerPage(VOID); + STATIC XTAPI PMMPTE GetSystemPteBaseAddress(VOID); + STATIC XTAPI PMMPTE GetValidPte(VOID); + STATIC XTAPI VOID InitializePageTable(VOID); + STATIC XTAPI VOID InitializeSystemPte(VOID); + STATIC XTAPI VOID InitializeSystemPteSpace(VOID); + STATIC XTAPI VOID MapP5E(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMP5E TemplateP5e); + STATIC XTAPI VOID MapPDE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPDE TemplatePde); + STATIC XTAPI VOID MapPPE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPPE TemplatePpe); + STATIC XTAPI VOID MapPTE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPTE TemplatePte); + STATIC XTAPI VOID MapPXE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPXE TemplatePxe); + STATIC XTAPI VOID ReleaseSystemPtes(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType); + STATIC XTAPI PMMPTE ReserveSystemPtes(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType); + + private: + STATIC XTAPI BOOLEAN FindFreeCluster(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType, + OUT PMMPTE *FoundCluster, + OUT PMMPTE *PreviousClusterNode); + STATIC XTAPI ULONG GetClusterSize(IN PMMPTE Pte); + STATIC XTAPI VOID InitializeSystemPtePool(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE PoolType); + }; +} + +#endif /* __XTOSKRNL_MM_AMD64_PTE_HH */ diff --git a/xtoskrnl/includes/mm/colors.hh b/xtoskrnl/includes/mm/colors.hh new file mode 100644 index 0000000..fda6934 --- /dev/null +++ b/xtoskrnl/includes/mm/colors.hh @@ -0,0 +1,38 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/colors.hh + * DESCRIPTION: Memory manager page coloring subsystem + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_COLORS_HH +#define __XTOSKRNL_MM_COLORS_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Colors + { + private: + STATIC PMMCOLOR_TABLES FreePages[FreePageList + 1]; + STATIC MMPFNLIST ModifiedPages[MM_PAGING_COLORS]; + STATIC ULONG PagingColors; + STATIC ULONG PagingColorsMask; + + public: + STATIC XTAPI VOID ComputePageColoring(VOID); + STATIC XTAPI PMMCOLOR_TABLES GetFreePages(IN MMPAGELISTS PageList, + IN ULONG Color); + STATIC XTAPI PMMPFNLIST GetModifiedPages(IN ULONG Color); + STATIC XTAPI ULONG GetNextColor(VOID); + STATIC XTAPI ULONG GetPagingColors(VOID); + STATIC XTAPI ULONG GetPagingColorsMask(VOID); + STATIC XTAPI VOID InitializeColorTables(VOID); + }; +} + +#endif /* __XTOSKRNL_MM_COLORS_HH */ diff --git a/xtoskrnl/includes/mm/i686/pagemap.hh b/xtoskrnl/includes/mm/i686/pagemap.hh index 0ca1616..3552cda 100644 --- a/xtoskrnl/includes/mm/i686/pagemap.hh +++ b/xtoskrnl/includes/mm/i686/pagemap.hh @@ -1,13 +1,13 @@ /** * PROJECT: ExectOS * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/includes/mm/pagemap.hh + * FILE: xtoskrnl/includes/mm/i686/pagemap.hh * DESCRIPTION: Low-level support for page map manipulation * DEVELOPERS: Aiken Harris */ -#ifndef __XTOSKRNL_MM_PAGEMAP_HH -#define __XTOSKRNL_MM_PAGEMAP_HH +#ifndef __XTOSKRNL_MM_I686_PAGEMAP_HH +#define __XTOSKRNL_MM_I686_PAGEMAP_HH #include @@ -21,45 +21,129 @@ namespace MM MMPAGEMAP_INFO PageMapInfo; public: - XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer); - XTAPI PMMPDE GetPdeAddress(PVOID Address); - XTAPI PMMPPE GetPpeAddress(PVOID Address); - XTAPI PMMPTE GetPteAddress(PVOID Address); - virtual XTAPI VOID InitializePageMapInfo(VOID) = 0; - virtual XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer) = 0; - virtual XTAPI VOID SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable) = 0; - virtual XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough) = 0; + VIRTUAL XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN ULONG Count) = 0; + VIRTUAL XTAPI VOID ClearPte(IN PMMPTE PtePointer) = 0; + VIRTUAL XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte) = 0; + VIRTUAL XTAPI PMMPTE GetNextPte(IN PMMPTE Pte) = 0; + VIRTUAL XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte) = 0; + VIRTUAL XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte) = 0; + XTAPI USHORT GetPageMapLevel(); + XTAPI PMMPDE GetPdeAddress(IN PVOID Address); + XTAPI ULONG GetPdeOffset(IN PVOID Address); + VIRTUAL XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer) = 0; + XTAPI PMMPPE GetPpeAddress(IN PVOID Address); + XTAPI ULONG GetPpeOffset(IN PVOID Address); + XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer); + VIRTUAL XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer) = 0; + XTAPI PMMPTE GetPteAddress(IN PVOID Address); + XTAPI ULONG GetPteOffset(IN PVOID Address); + VIRTUAL XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte) = 0; + VIRTUAL XTAPI ULONG GetPteSize(VOID) = 0; + VIRTUAL XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer) = 0; + VIRTUAL XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer) = 0; + VIRTUAL XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer) = 0; + VIRTUAL XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer) = 0; + XTAPI BOOLEAN GetXpaStatus(); + VIRTUAL XTAPI VOID InitializePageMapInfo(VOID) = 0; + VIRTUAL XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer) = 0; + VIRTUAL XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value) = 0; + VIRTUAL XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value) = 0; + VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask) = 0; + VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes) = 0; + VIRTUAL XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough) = 0; + VIRTUAL XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection) = 0; + VIRTUAL XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value) = 0; + } PAGEMAP, *PPAGEMAP; class PageMapBasic final : public PageMap { public: + XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN ULONG Count); + XTAPI VOID ClearPte(IN PMMPTE PtePointer); + XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte); + XTAPI PMMPTE GetNextPte(IN PMMPTE Pte); + XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte); + XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte); + XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer); + XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte); + XTAPI ULONG GetPteSize(VOID); + XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer); + XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); XTAPI VOID InitializePageMapInfo(VOID); - XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer); - XTAPI VOID SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable); - XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough); + XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer); + XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value); + XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes); + XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough); + XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection); + XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value); }; class PageMapXpa final : public PageMap { public: + XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN ULONG Count); + XTAPI VOID ClearPte(IN PMMPTE PtePointer); + XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte); + XTAPI PMMPTE GetNextPte(IN PMMPTE Pte); + XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte); + XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte); + XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer); + XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte); + XTAPI ULONG GetPteSize(VOID); + XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer); + XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer); + XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); XTAPI VOID InitializePageMapInfo(VOID); - XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer); - XTAPI VOID SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable); - XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough); + XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer); + XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value); + XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask); + XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes); + XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough); + XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection); + XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value); }; } -#endif /* __XTOSKRNL_MM_PAGEMAP_HH */ +#endif /* __XTOSKRNL_MM_I686_PAGEMAP_HH */ diff --git a/xtoskrnl/includes/mm/i686/paging.hh b/xtoskrnl/includes/mm/i686/paging.hh new file mode 100644 index 0000000..cdc95ab --- /dev/null +++ b/xtoskrnl/includes/mm/i686/paging.hh @@ -0,0 +1,75 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/i686/paging.hh + * DESCRIPTION: Low level page management support for i686 + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_I686_PAGING_HH +#define __XTOSKRNL_MM_I686_PAGING_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Paging + { + private: + STATIC PPAGEMAP PmlRoutines; + + public: + STATIC XTAPI PMMPTE AdvancePte(IN PMMPTE Pte, + IN LONG Count); + STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer); + STATIC XTAPI VOID FlushTlb(VOID); + STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte); + STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte); + STATIC XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte); + STATIC XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte); + STATIC XTAPI USHORT GetPageMapLevel(); + STATIC XTAPI PMMPDE GetPdeAddress(IN PVOID Address); + STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer); + STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address); + STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer); + STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer); + STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address); + STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte); + STATIC XTAPI ULONG GetPteSize(VOID); + STATIC XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer); + STATIC XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer); + STATIC XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer); + STATIC XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer); + STATIC XTAPI BOOLEAN GetXpaStatus(VOID); + STATIC XTAPI VOID InitializePageMapSupport(VOID); + STATIC XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer); + STATIC XTAPI VOID SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value); + STATIC XTAPI VOID SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value); + STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask); + STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes); + STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough); + STATIC XTAPI VOID WritePte(IN PMMPTE Pte, + IN MMPTE Value); + STATIC XTAPI VOID TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection); + STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address, + IN ULONG Size); + + private: + STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID); + STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID); + STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID); + }; +} + +#endif /* __XTOSKRNL_MM_I686_PAGING_HH */ diff --git a/xtoskrnl/includes/mm/i686/pte.hh b/xtoskrnl/includes/mm/i686/pte.hh new file mode 100644 index 0000000..116a022 --- /dev/null +++ b/xtoskrnl/includes/mm/i686/pte.hh @@ -0,0 +1,63 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/i686/pte.hh + * DESCRIPTION: Page Table Entry (PTE) for i686 support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_I686_PTE_HH +#define __XTOSKRNL_MM_I686_PTE_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Pte + { + private: + STATIC MMPTE FirstSystemFreePte[MaximumPtePoolTypes]; + STATIC PMMPTE SystemPteBase; + STATIC PMMPTE SystemPtesEnd[MaximumPtePoolTypes]; + STATIC PMMPTE SystemPtesStart[MaximumPtePoolTypes]; + STATIC PFN_COUNT TotalSystemFreePtes[MaximumPtePoolTypes]; + STATIC MMPTE ValidPte; + + public: + STATIC XTAPI BOOLEAN AddressValid(IN PVOID VirtualAddress); + STATIC XTAPI PFN_COUNT GetPtesPerPage(VOID); + STATIC XTAPI PMMPTE GetSystemPteBaseAddress(VOID); + STATIC XTAPI PMMPTE GetValidPte(VOID); + STATIC XTAPI VOID InitializePageTable(VOID); + STATIC XTAPI VOID InitializeSystemPte(VOID); + STATIC XTAPI VOID InitializeSystemPteSpace(VOID); + STATIC XTAPI VOID MapPDE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPDE TemplatePde); + STATIC XTAPI VOID MapPPE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPPE TemplatePpe); + STATIC XTAPI VOID MapPTE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPTE TemplatePte); + STATIC XTAPI VOID ReleaseSystemPtes(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType); + STATIC XTAPI PMMPTE ReserveSystemPtes(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType); + + private: + STATIC XTAPI BOOLEAN FindFreeCluster(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType, + OUT PMMPTE *FoundCluster, + OUT PMMPTE *PreviousClusterNode); + STATIC XTAPI ULONG GetClusterSize(IN PMMPTE Pte); + STATIC XTAPI VOID InitializeSystemPtePool(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE PoolType); + }; +} + +#endif /* __XTOSKRNL_MM_I686_PTE_HH */ diff --git a/xtoskrnl/includes/mm/init.hh b/xtoskrnl/includes/mm/init.hh deleted file mode 100644 index e606876..0000000 --- a/xtoskrnl/includes/mm/init.hh +++ /dev/null @@ -1,39 +0,0 @@ -/** - * PROJECT: ExectOS - * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/includes/mm/init.hh - * DESCRIPTION: Memory Manager initialization - * DEVELOPERS: Aiken Harris - */ - -#ifndef __XTOSKRNL_MM_INIT_HH -#define __XTOSKRNL_MM_INIT_HH - -#include - - -/* Memory Manager */ -namespace MM -{ - class Init - { - private: - STATIC PLOADER_MEMORY_DESCRIPTOR FreeDescriptor; - STATIC ULONG_PTR HighestPhysicalPage; - STATIC ULONG_PTR LowestPhysicalPage; - STATIC ULONG NumberOfPhysicalPages; - STATIC LOADER_MEMORY_DESCRIPTOR OldFreeDescriptor; - - public: - STATIC XTAPI VOID InitializeMemoryManager(VOID); - STATIC XTAPI VOID InitializePageMapSupport(VOID); - STATIC XTAPI VOID ScanMemoryDescriptors(VOID); - - private: - STATIC XTAPI VOID InitializeArchitecture(VOID); - STATIC XTAPI BOOLEAN VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType); - STATIC XTAPI BOOLEAN VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType); - }; -} - -#endif /* __XTOSKRNL_MM_INIT_HH */ diff --git a/xtoskrnl/includes/mm/kpool.hh b/xtoskrnl/includes/mm/kpool.hh index 2e94881..58afda6 100644 --- a/xtoskrnl/includes/mm/kpool.hh +++ b/xtoskrnl/includes/mm/kpool.hh @@ -21,13 +21,12 @@ namespace MM STATIC UCHAR ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE]; public: - STATIC XTAPI XTSTATUS AllocateKernelStack(IN PVOID *Stack, - IN BOOLEAN LargeStack, - IN UCHAR SystemNode); + STATIC XTAPI XTSTATUS AllocateKernelStack(OUT PVOID *Stack, + IN ULONG StackSize); STATIC XTAPI XTSTATUS AllocateProcessorStructures(IN ULONG CpuNumber, - OUT PVOID *StructuresData); + OUT PVOID *StructuresData); STATIC XTAPI VOID FreeKernelStack(IN PVOID Stack, - IN BOOLEAN LargeStack); + IN ULONG StackSize); STATIC XTAPI VOID FreeProcessorStructures(IN PVOID StructuresData); }; } diff --git a/xtoskrnl/includes/mm/mmgr.hh b/xtoskrnl/includes/mm/mmgr.hh new file mode 100644 index 0000000..770491b --- /dev/null +++ b/xtoskrnl/includes/mm/mmgr.hh @@ -0,0 +1,44 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/mmgr.hh + * DESCRIPTION: Memory Manager + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_MMGR_HH +#define __XTOSKRNL_MM_MMGR_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Manager + { + private: + STATIC MMMEMORY_LAYOUT MemoryLayout; + STATIC PFN_NUMBER NumberOfSystemPtes; + + public: + STATIC XTAPI ULONG_PTR GetInstalledMemorySize(VOID); + STATIC XTAPI PMMMEMORY_LAYOUT GetMemoryLayout(VOID); + STATIC XTAPI PFN_NUMBER GetNumberOfSystemPtes(); + STATIC XTAPI VOID InitializeMemoryLayout(VOID); + STATIC XTAPI VOID InitializeMemoryManager(VOID); + STATIC XTAPI BOOLEAN VerifyMemoryTypeFree(IN LOADER_MEMORY_TYPE MemoryType); + STATIC XTAPI BOOLEAN VerifyMemoryTypeInvisible(IN LOADER_MEMORY_TYPE MemoryType); + + private: + STATIC XTAPI VOID ComputeBootImageSize(OUT PPFN_NUMBER BootImageSize); + STATIC XTAPI VOID ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize); + STATIC XTAPI VOID ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize); + STATIC XTAPI VOID ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize); + STATIC XTAPI VOID ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize); + STATIC XTAPI VOID ComputeSystemPteSize(OUT PPFN_NUMBER PteSize); + STATIC XTAPI VOID DumpMemoryLayout(VOID); + }; +} + +#endif /* __XTOSKRNL_MM_MMGR_HH */ diff --git a/xtoskrnl/includes/mm/paging.hh b/xtoskrnl/includes/mm/paging.hh deleted file mode 100644 index 2f261f3..0000000 --- a/xtoskrnl/includes/mm/paging.hh +++ /dev/null @@ -1,47 +0,0 @@ -/** - * PROJECT: ExectOS - * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/includes/mm/paging.hh - * DESCRIPTION: Low level page management support - * DEVELOPERS: Aiken Harris - */ - -#ifndef __XTOSKRNL_MM_PAGING_HH -#define __XTOSKRNL_MM_PAGING_HH - -#include - - -/* Memory Manager */ -namespace MM -{ - class Paging - { - private: - STATIC PPAGEMAP PmlRoutines; - - public: - STATIC XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer); - STATIC XTAPI VOID FlushTlb(VOID); - STATIC XTAPI PMMPDE GetPdeAddress(PVOID Address); - STATIC XTAPI PMMPPE GetPpeAddress(PVOID Address); - STATIC XTAPI PMMPTE GetPteAddress(PVOID Address); - STATIC XTAPI VOID InitializePageMapSupport(VOID); - STATIC XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer); - STATIC XTAPI VOID SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable); - STATIC XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough); - STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address, - IN ULONG Size); - - private: - STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID); - STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID); - STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID); - }; -} - -#endif /* __XTOSKRNL_MM_PAGING_HH */ diff --git a/xtoskrnl/includes/mm/pfault.hh b/xtoskrnl/includes/mm/pfault.hh new file mode 100644 index 0000000..30e2e49 --- /dev/null +++ b/xtoskrnl/includes/mm/pfault.hh @@ -0,0 +1,25 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/pfault.hh + * DESCRIPTION: Page fault support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_PFAULT_HH +#define __XTOSKRNL_MM_PFAULT_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class PageFault + { + public: + STATIC XTFASTCALL XTSTATUS CheckPdeForPagedPool(IN PVOID VirtualAddress); + }; +} + +#endif /* __XTOSKRNL_MM_PFAULT_HH */ diff --git a/xtoskrnl/includes/mm/pfn.hh b/xtoskrnl/includes/mm/pfn.hh new file mode 100644 index 0000000..5b7a810 --- /dev/null +++ b/xtoskrnl/includes/mm/pfn.hh @@ -0,0 +1,83 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/includes/mm/pfn.hh + * DESCRIPTION: Physical Frame Number (PFN) support + * DEVELOPERS: Aiken Harris + */ + +#ifndef __XTOSKRNL_MM_PFN_HH +#define __XTOSKRNL_MM_PFN_HH + +#include + + +/* Memory Manager */ +namespace MM +{ + class Pfn + { + private: + STATIC PFN_NUMBER AvailablePages; + STATIC MMPFNLIST BadPagesList; + STATIC PLOADER_MEMORY_DESCRIPTOR FreeDescriptor; + STATIC MMPFNLIST FreePagesList; + STATIC ULONG_PTR HighestPhysicalPage; + STATIC PVOID HighestUserAddress; + STATIC ULONG_PTR LowestPhysicalPage; + STATIC MMPFNLIST ModifiedPagesList; + STATIC MMPFNLIST ModifiedReadOnlyPagesList; + STATIC ULONGLONG NumberOfPhysicalPages; + STATIC LOADER_MEMORY_DESCRIPTOR OriginalFreeDescriptor; + STATIC PMMPFNLIST PageLocationList[]; + STATIC MMPFNLIST RomPagesList; + STATIC MMPFNLIST StandbyPagesList; + STATIC MMPFNLIST ZeroedPagesList; + + public: + STATIC XTAPI PFN_NUMBER AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages); + STATIC XTAPI PFN_NUMBER AllocatePhysicalPage(IN ULONG Color); + STATIC XTAPI VOID ComputePfnDatabaseSize(OUT PPFN_NUMBER DatabaseSize); + STATIC XTAPI VOID DecrementReferenceCount(IN PMMPFN Pfn1, + IN PFN_NUMBER PageFrameIndex, + IN BOOLEAN BeginStandbyList = FALSE); + STATIC XTAPI VOID DecrementShareCount(IN PMMPFN Pfn1, + IN PFN_NUMBER PageFrameIndex, + IN BOOLEAN BeginStandbyList = FALSE); + STATIC XTAPI VOID FreePhysicalPage(IN PMMPTE PointerPte); + STATIC XTAPI PFN_NUMBER GetAvailablePages(VOID); + STATIC XTAPI ULONG_PTR GetHighestPhysicalPage(VOID); + STATIC XTAPI ULONGLONG GetNumberOfPhysicalPages(VOID); + STATIC XTAPI PMMPFN GetPfnEntry(IN PFN_NUMBER Pfn); + STATIC XTAPI VOID InitializePfnDatabase(VOID); + STATIC XTAPI VOID LinkPfn(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN BOOLEAN Modified); + STATIC XTAPI VOID LinkPfnWithParent(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN PFN_NUMBER ParentFrame); + STATIC XTAPI VOID ScanMemoryDescriptors(VOID); + + private: + STATIC XTAPI VOID DecrementAvailablePages(VOID); + STATIC XTAPI VOID IncrementAvailablePages(VOID); + STATIC XTAPI VOID InitializePageDirectory(IN PMMPDE StartingPde, + IN PMMPDE EndingPde); + STATIC XTAPI VOID InitializePageTablePfns(VOID); + STATIC XTAPI VOID LinkFreePage(IN PFN_NUMBER PageFrameIndex); + STATIC XTAPI VOID LinkPage(IN PMMPFNLIST ListHead, + IN PFN_NUMBER PageFrameIndex); + STATIC XTAPI VOID LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte); + STATIC XTFASTCALL VOID LinkStandbyPage(IN PFN_NUMBER PageFrameIndex); + STATIC XTAPI VOID ProcessMemoryDescriptor(IN PFN_NUMBER BasePage, + IN PFN_NUMBER PageCount, + IN LOADER_MEMORY_TYPE MemoryType); + STATIC XTAPI VOID ScanPageTable(IN PMMPTE PointerPte, + IN ULONG Level); + STATIC XTAPI PFN_NUMBER UnlinkFreePage(IN PFN_NUMBER PageFrameIndex, + IN ULONG Color); + }; +} + +#endif /* __XTOSKRNL_MM_PFN_HH */ diff --git a/xtoskrnl/ke/amd64/krnlinit.cc b/xtoskrnl/ke/amd64/krnlinit.cc index 985d7d1..3c612e8 100644 --- a/xtoskrnl/ke/amd64/krnlinit.cc +++ b/xtoskrnl/ke/amd64/krnlinit.cc @@ -86,6 +86,10 @@ KE::KernelInit::StartKernel(VOID) /* Save processor state */ Processor::SaveProcessorState(&Prcb->ProcessorState); + /* Initialize spin locks */ + SpinLock::InitializeAllLocks(); + SpinLock::InitializeLockQueues(); + /* Lower to APC runlevel */ RunLevel::LowerRunLevel(APC_LEVEL); @@ -108,6 +112,9 @@ KE::KernelInit::StartKernel(VOID) CurrentThread->WaitRunLevel = DISPATCH_LEVEL; CurrentProcess->ActiveProcessors |= (ULONG_PTR)1 << Prcb->CpuNumber; + /* Initialize Memory Manager */ + MM::Manager::InitializeMemoryManager(); + /* Enter infinite loop */ DebugPrint(L"KernelInit::StartKernel() finished. Entering infinite loop.\n"); Crash::HaltSystem(); diff --git a/xtoskrnl/ke/i686/krnlinit.cc b/xtoskrnl/ke/i686/krnlinit.cc index c2a418a..8ee661a 100644 --- a/xtoskrnl/ke/i686/krnlinit.cc +++ b/xtoskrnl/ke/i686/krnlinit.cc @@ -86,6 +86,10 @@ KE::KernelInit::StartKernel(VOID) /* Save processor state */ Processor::SaveProcessorState(&Prcb->ProcessorState); + /* Initialize spin locks */ + SpinLock::InitializeAllLocks(); + SpinLock::InitializeLockQueues(); + /* Lower to APC runlevel */ RunLevel::LowerRunLevel(APC_LEVEL); @@ -108,6 +112,9 @@ KE::KernelInit::StartKernel(VOID) CurrentThread->WaitRunLevel = DISPATCH_LEVEL; CurrentProcess->ActiveProcessors |= (ULONG_PTR)1 << Prcb->CpuNumber; + /* Initialize Memory Manager */ + MM::Manager::InitializeMemoryManager(); + /* Enter infinite loop */ DebugPrint(L"KernelInit::StartKernel() finished. Entering infinite loop.\n"); Crash::HaltSystem(); diff --git a/xtoskrnl/ke/kthread.cc b/xtoskrnl/ke/kthread.cc index 1e531f4..13755ba 100644 --- a/xtoskrnl/ke/kthread.cc +++ b/xtoskrnl/ke/kthread.cc @@ -152,7 +152,7 @@ KE::KThread::InitializeThread(IN PKPROCESS Process, if(!Stack) { /* Allocate new stack */ - Status = MM::KernelPool::AllocateKernelStack(&Stack, FALSE, 0); + Status = MM::KernelPool::AllocateKernelStack(&Stack, KERNEL_STACK_SIZE); if(Status != STATUS_SUCCESS || !Stack) { /* Stack allocation failed */ diff --git a/xtoskrnl/mm/alloc.cc b/xtoskrnl/mm/alloc.cc new file mode 100644 index 0000000..e60a18c --- /dev/null +++ b/xtoskrnl/mm/alloc.cc @@ -0,0 +1,313 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/alloc.cc + * DESCRIPTION: Memory manager pool allocation + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Allocates pages from the non-paged pool. + * + * @param Pages + * Specifies the number of pages to allocate. + * + * @param Memory + * Supplies a pointer to the allocated pool. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MM::Allocator::AllocateNonPagedPoolPages(IN PFN_COUNT Pages, + OUT PVOID *Memory) +{ + PLIST_ENTRY Entry, LastHead, ListHead; + PMMPTE PointerPte, ValidPte; + PMMFREE_POOL_ENTRY FreePage; + PFN_NUMBER PageFrameNumber; + PVOID BaseAddress; + ULONG Index; + PMMPFN Pfn; + + /* Calculate the free list index based on the requested page count, capped at the maximum list head index */ + Index = MIN(Pages, MM_MAX_FREE_PAGE_LIST_HEADS) - 1; + + /* Set the starting list head and the boundary for the search loop */ + ListHead = &NonPagedPoolFreeList[Index]; + LastHead = &NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS]; + + /* Start a guarded code block */ + { + /* Acquire the Non-Paged pool lock and raise runlevel to DISPATCH_LEVEL */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard NonPagedPoolSpinLock(NonPagedPoolLock); + + /* Iterate through the free lists */ + do + { + /* Iterate through the free entries in the current list */ + Entry = ListHead->Flink; + while(Entry != ListHead) + { + /* Get the free pool entry structure from the list entry */ + FreePage = CONTAIN_RECORD(Entry, MMFREE_POOL_ENTRY, List); + + /* Check if this block is large enough to satisfy the request */ + if(FreePage->Size >= Pages) + { + /* Adjust the size of the free block to account for the allocated pages */ + FreePage->Size -= Pages; + + /* Calculate the base address of the allocated block */ + BaseAddress = (PVOID)((ULONG_PTR)FreePage + (FreePage->Size << MM_PAGE_SHIFT)); + + /* Remove the entry from the free list */ + RTL::LinkedList::RemoveEntryList(&FreePage->List); + + /* Check if there is remaining space in the entry */ + if(FreePage->Size != 0) + { + /* Calculate the new list index for the remaining fragment */ + Index = MIN(FreePage->Size, MM_MAX_FREE_PAGE_LIST_HEADS) - 1; + + /* Insert the entry into the free list */ + RTL::LinkedList::InsertTailList(&NonPagedPoolFreeList[Index], &FreePage->List); + } + + /* Get the Page Table Entry (PTE) for the allocated address */ + PointerPte = MM::Paging::GetPteAddress(BaseAddress); + + /* Get the Page Frame Number (PFN) database entry for the corresponding physical page */ + Pfn = MM::Pfn::GetPfnEntry(MM::Paging::GetPageFrameNumber(PointerPte)); + + /* Denote allocation boundaries */ + Pfn->u3.e1.ReadInProgress = 1; + + /* Check if multiple pages were requested */ + if(Pages != 1) + { + /* Advance to the PTE of the last page in the allocation */ + PointerPte = MM::Paging::AdvancePte(PointerPte, Pages - 1); + + /* Get the PFN entry for the last page */ + Pfn = MM::Pfn::GetPfnEntry(MM::Paging::GetPageFrameNumber(PointerPte)); + } + + /* Denote allocation boundaries */ + Pfn->u3.e1.WriteInProgress = 1; + + /* Set the allocated memory address and return success */ + *Memory = BaseAddress; + return STATUS_SUCCESS; + } + + /* Move to the next entry in the free list */ + Entry = FreePage->List.Flink; + } + } + while(++ListHead < LastHead); + } + + /* No suitable free block found; try to expand the pool by reserving system PTEs */ + PointerPte = MM::Pte::ReserveSystemPtes(Pages, NonPagedPoolExpansion); + if(PointerPte == NULLPTR) + { + /* PTE reservation failed, return insufficient resources */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Acquire the Non-Paged pool lock and raise runlevel to DISPATCH_LEVEL */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard NonPagedPoolSpinLock(NonPagedPoolLock); + + /* Acquire the PFN database lock */ + KE::QueuedSpinLockGuard PfnSpinLock(PfnLock); + + /* Check if there are enough available physical pages to back the allocation */ + if(Pages >= MM::Pfn::GetAvailablePages()) + { + /* Not enough physical pages, release the reserved system PTEs */ + MM::Pte::ReleaseSystemPtes(PointerPte, Pages, NonPagedPoolExpansion); + + /* Return failure due to insufficient resources */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Get a template valid PTE and loop through the allocation to map physical pages */ + ValidPte = MM::Pte::GetValidPte(); + do + { + /* Allocate a physical page */ + PageFrameNumber = MM::Pfn::AllocatePhysicalPage(MM::Colors::GetNextColor()); + + /* Initialize the PFN entry for the allocated physical page */ + Pfn = MM::Pfn::GetPfnEntry(PageFrameNumber); + Pfn->PteAddress = PointerPte; + Pfn->u2.ShareCount = 1; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + Pfn->u4.VerifierAllocation = 0; + + /* Build a valid PTE pointing to the allocated page frame */ + MM::Paging::SetPte(ValidPte, PageFrameNumber, 0); + + /* Write the valid PTE into the system PTE range */ + *(MM::Paging::GetNextPte(PointerPte)) = *ValidPte; + } + while(--Pages > 0); + + /* Dnote allocation boundaries */ + Pfn->u3.e1.WriteInProgress = 1; + + /* Get the PFN entry for the first page of the allocation */ + Pfn = MM::Pfn::GetPfnEntry(MM::Paging::GetPageFrameNumber(PointerPte)); + + /* Denote allocation boundaries */ + Pfn->u3.e1.ReadInProgress = 1; + + /* Convert the PTE address to the virtual address and store in the buffer */ + *Memory = MM::Paging::GetPteVirtualAddress(PointerPte); + + /* Return success */ + return STATUS_SUCCESS; +} + +/** + * Allocates pages from the paged pool. + * + * @param Pages + * Specifies the number of pages to allocate. + * + * @param Memory + * Supplies a pointer to the allocated pool. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MM::Allocator::AllocatePagedPoolPages(IN PFN_COUNT Pages, + OUT PVOID *Memory) +{ + UNIMPLEMENTED; + + /* Return not implemented status code */ + return STATUS_NOT_IMPLEMENTED; +} + +/** + * Allocates pages from the specified pool type. + * + * @param PoolType + * Specifies the type of pool to allocate pages from. + * + * @param Bytes + * Specifies the number of bytes to allocate. + * + * @param Memory + * Supplies a pointer to the allocated pool. + * + * @return This routine returns a status code. + * + * @since XT 1.0 + */ +XTAPI +XTSTATUS +MM::Allocator::AllocatePages(IN MMPOOL_TYPE PoolType, + IN SIZE_T Bytes, + OUT PVOID *Memory) +{ + PFN_COUNT Pages; + + /* Initialize the output parameter */ + *Memory = NULLPTR; + + /* Convert bytes to pages */ + Pages = SIZE_TO_PAGES(Bytes); + + /* Check if there are any pages to allocate */ + if(!Pages) + { + /* Nothing to allocate, return NULLPTR */ + return STATUS_INVALID_PARAMETER; + } + + /* Switch on pool type */ + switch(PoolType & MM_POOL_TYPE_MASK) + { + case NonPagedPool: + /* Allocate non-paged pool */ + return AllocateNonPagedPoolPages(Pages, Memory); + case PagedPool: + /* Allocate paged pool */ + return AllocatePagedPoolPages(Pages, Memory); + } + + /* Invalid pool type specified, return error */ + return STATUS_INVALID_PARAMETER; +} + +/** + * Initializes the non-paged pool for memory allocator. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Allocator::InitializeNonPagedPool(VOID) +{ + PMMMEMORY_LAYOUT MemoryLayout; + PMMFREE_POOL_ENTRY FreeEntry; + ULONG Index; + + /* Retrieve memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Map PTEs for the non-paged pool */ + MapNonPagedPool(); + + /* Iterate over the free page list heads */ + for(Index = 0; Index < MM_MAX_FREE_PAGE_LIST_HEADS; Index++) + { + /* Initialize a free page list head */ + RTL::LinkedList::InitializeListHead(&NonPagedPoolFreeList[Index]); + } + + /* Take the first free page from the pool */ + FreeEntry = (PMMFREE_POOL_ENTRY)MemoryLayout->NonPagedPoolStart; + + /* Take number of pages in the pool */ + Index = (ULONG)(MemoryLayout->NonPagedPoolSize - 1); + if(Index >= MM_MAX_FREE_PAGE_LIST_HEADS) + { + /* Number of pages exceeds the number of free page list heads */ + Index = MM_MAX_FREE_PAGE_LIST_HEADS - 1; + } + + /* Insert the first free page into the free page list and set its size */ + RTL::LinkedList::InsertHeadList(&NonPagedPoolFreeList[Index], &FreeEntry->List); + FreeEntry->Size = MemoryLayout->NonPagedPoolSize; +} + +/** + * Initializes the non-paged pool for memory allocator. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Allocator::InitializePagedPool(VOID) +{ + UNIMPLEMENTED; +} diff --git a/xtoskrnl/mm/amd64/alloc.cc b/xtoskrnl/mm/amd64/alloc.cc new file mode 100644 index 0000000..bc2be3c --- /dev/null +++ b/xtoskrnl/mm/amd64/alloc.cc @@ -0,0 +1,34 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/amd64/alloc.cc + * DESCRIPTION: Memory manager pool allocation + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Maps the PTE for the base of the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Allocator::MapNonPagedPool(VOID) +{ + PMMMEMORY_LAYOUT MemoryLayout; + + /* Retrieve memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Map PPE and PDE for whole non-paged pool */ + MM::Pte::MapPPE(MemoryLayout->NonPagedPoolStart, MemoryLayout->NonPagedExpansionPoolEnd, MM::Pte::GetValidPte()); + MM::Pte::MapPDE(MemoryLayout->NonPagedPoolStart, MemoryLayout->NonPagedExpansionPoolEnd, MM::Pte::GetValidPte()); + + /* Map PTE only for the base of the non-paged pool */ + MM::Pte::MapPTE(MemoryLayout->NonPagedPoolStart, (PCHAR)MemoryLayout->NonPagedPoolEnd - 1, MM::Pte::GetValidPte()); +} diff --git a/xtoskrnl/mm/amd64/init.cc b/xtoskrnl/mm/amd64/init.cc deleted file mode 100644 index 7e7a9d8..0000000 --- a/xtoskrnl/mm/amd64/init.cc +++ /dev/null @@ -1,25 +0,0 @@ -/** - * PROJECT: ExectOS - * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/mm/amd64/init.cc - * DESCRIPTION: Architecture specific Memory Manager initialization routines - * DEVELOPERS: Rafal Kupiec - * Aiken Harris - */ - -#include - - -/** - * Performs architecture specific initialization of the XTOS Memory Manager. - * - * @return This routine does not return any value. - * - * @since XT 1.0 - */ -XTAPI -VOID -MM::Init::InitializeArchitecture(VOID) -{ - UNIMPLEMENTED; -} diff --git a/xtoskrnl/mm/amd64/mmgr.cc b/xtoskrnl/mm/amd64/mmgr.cc new file mode 100644 index 0000000..8219f1b --- /dev/null +++ b/xtoskrnl/mm/amd64/mmgr.cc @@ -0,0 +1,390 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/amd64/mmgr.cc + * DESCRIPTION: Memory Manager + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Calculates the maximum possible size of the non-paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONG_PTR MaximumNonPagedPoolSize; + + /* Start with the 1MiB and add 400KiB for each MiB above 16MiB */ + MaximumNonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 409600); + + /* Check if non-paged pool does not exceed 128GiB */ + if(MaximumNonPagedPoolSize > 137438953472) + { + /* Limit non-paged pool size to 128GiB */ + MaximumNonPagedPoolSize = 137438953472; + } + + /* Return non-paged pool size */ + *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize); +} + +/** + * Calculates the size of the non-paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONG_PTR NonPagedPoolSize; + + /* Start with 1MiB and add 32KiB for each MiB above 16MiB */ + NonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 32768); + + /* Check if non-paged pool does not exceed 128GiB */ + if(NonPagedPoolSize > 137438953472) + { + /* Limit non-paged pool size to 128GiB */ + NonPagedPoolSize = 137438953472; + } + + /* Return non-paged pool size in pages, aligned up to page size boundary */ + *PoolSize = SIZE_TO_PAGES(ROUND_UP(NonPagedPoolSize, MM_PAGE_SIZE)); +} + +/** + * Calculates the size of the paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONGLONG PagedPoolSize, PteCount; + ULONG PtesPerPage; + + /* Start with 4x maximum non-paged pool size and at least 48MiB */ + ComputeMaximumNonPagedPoolSize(&PagedPoolSize); + PagedPoolSize *= 4 * MM_PAGE_SIZE; + + /* Ensure that paged pool size is at least 48MiB */ + if(PagedPoolSize < 50331648) + { + /* Increase paged pool size to at least 48MiB */ + PagedPoolSize = 50331648; + } + + /* Check if paged pool does not overlap non-paged pool */ + if(PagedPoolSize > (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart) + { + /* Limit paged pool size to maximum possible */ + PagedPoolSize = (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart; + } + + /* Check if paged pool does not exceed 128GiB */ + if(PagedPoolSize > 137438953472) + { + /* Limit paged pool size to 128GiB */ + PagedPoolSize = 137438953472; + } + + /* Get the number of PTEs per page and calculate size of paged pool */ + PtesPerPage = MM::Pte::GetPtesPerPage(); + PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage); + + /* Return paged pool size */ + *PoolSize = PteCount * PtesPerPage; +} + +/** + * Calculates the size of the session space. + * + * @param SpaceSize + * A pointer to a variable that will receive the number of pages available by the session space. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize) +{ + PFN_NUMBER SessionSpaceSize; + + /* Session Pool, Session View, Session Image, Session Working Set and System View takes 1120MiB */ + SessionSpaceSize = 1174405120; + + /* Return number of pages used by the session space */ + *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE; +} + +/** + * Calculates the size of the system PTEs. + * + * @param PteSize + * A pointer to a variable that will receive the number of system PTEs. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize) +{ + PFN_NUMBER SystemPteSize; + + /* Check if system has less than 24MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() < 6144) + { + /* Set minimal system PTE size */ + SystemPteSize = 7000; + } + else + { + /* Use standard system PTE size */ + SystemPteSize = 11000; + + /* Check if system has more than 32MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() > 8192) + { + /* Double system PTE size */ + SystemPteSize *= 2; + + /* Check if system has more than 256MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() > 65536) + { + /* Double system PTE size */ + SystemPteSize *= 2; + } + } + } + + /* Return system PTE size */ + *PteSize = SystemPteSize; +} + +/** + * Dumps the kernel's memory layout. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::DumpMemoryLayout(VOID) +{ + /* Dump memory layout */ + DebugPrint(L"System with %zu MiB of installed memory:\n" + L"User Space: %.16P - %.16P\n" + L"Non-Canonical: %.16P - %.16P\n" + L"Reserved System Pool: %.16P - %.16P\n" + L"PTE Space: %.16P - %.16P\n" + L"Hyper Space: %.16P - %.16P\n" + L"Shared System Page: %.16P - %.16P\n" + L"System Working Set: %.16P - %.16P\n" + L"Loader Mappings: %.16P - %.16P\n" + L"Non-Paged System Pool: %.16P - %.16P\n" + L"Paged Pool: %.16P - %.16P\n" + L"Session Space: %.16P - %.16P\n" + L"System Cache: %.16P - %.16P\n" + L"PFN Database: %.16P - %.16P\n" + L"Non-Paged Pool: %.16P - %.16P\n" + L"Non-Paged Expansion Pool: %.16P - %.16P\n" + L"Hardware Pool: %.16P - %.16P\n", + GetInstalledMemorySize(), + MemoryLayout.UserSpaceStart, + MemoryLayout.UserSpaceEnd, + MemoryLayout.NonCanonicalStart, + MemoryLayout.NonCanonicalEnd, + MemoryLayout.ReservedSystemPoolStart, + MemoryLayout.ReservedSystemPoolEnd, + MemoryLayout.PteSpaceStart, + MemoryLayout.PteSpaceEnd, + MemoryLayout.HyperSpaceStart, + MemoryLayout.HyperSpaceEnd, + MemoryLayout.SharedSystemPageStart, + MemoryLayout.SharedSystemPageEnd, + MemoryLayout.SystemWorkingSetStart, + MemoryLayout.SystemWorkingSetEnd, + MemoryLayout.LoaderMappingsStart, + MemoryLayout.LoaderMappingsEnd, + MemoryLayout.NonPagedSystemPoolStart, + MemoryLayout.NonPagedSystemPoolEnd, + MemoryLayout.PagedPoolStart, + MemoryLayout.PagedPoolEnd, + MemoryLayout.SessionSpaceStart, + MemoryLayout.SessionSpaceEnd, + MemoryLayout.SystemCacheStart, + MemoryLayout.SystemCacheEnd, + MemoryLayout.PfnDatabase, + (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE), + MemoryLayout.NonPagedPoolStart, + MemoryLayout.NonPagedPoolEnd, + MemoryLayout.NonPagedExpansionPoolStart, + MemoryLayout.NonPagedExpansionPoolEnd, + MemoryLayout.HardwarePoolStart, + MemoryLayout.HardwarePoolEnd); +} + +/** + * Initializes the kernel's virtual memory layout. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::InitializeMemoryLayout(VOID) +{ + PFN_NUMBER MaximumNonPagedPoolSize; + ULONG_PTR PfnDatabaseEnd; + + /* Check if 5-level paging (LA57) is enabled */ + if(MM::Paging::GetXpaStatus()) + { + /* Set PML5 base address */ + MemoryLayout.SelfMapAddress = (PVOID)MM_P5E_LA57_BASE; + + /* Define memory layout for 5-level paging */ + MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000; + MemoryLayout.UserSpaceEnd = (PVOID)0x00FFFFFFFFFEFFFF; + MemoryLayout.NonCanonicalStart = (PVOID)0x0080000000000000; + MemoryLayout.NonCanonicalEnd = (PVOID)0xFEFFFFFFFFFFFFFF; + MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFF00000000000000; + MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFECFFFFFFFFFFFF; + MemoryLayout.PteSpaceStart = (PVOID)0xFFED000000000000; + MemoryLayout.PteSpaceEnd = (PVOID)0xFFEDFFFFFFFFFFFF; + MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000; + MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF; + MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000; + MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF; + MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000; + MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF; + MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000; + MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF; + MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000; + MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF; + MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000; + MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF; + MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000; + MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000; + MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000; + MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF; + MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000; + MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF; + MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000; + MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF; + } + else + { + /* Set PML4 base address */ + MemoryLayout.SelfMapAddress = (PVOID)MM_PXE_BASE; + + /* Define memory layout for 4-level paging */ + MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000; + MemoryLayout.UserSpaceEnd = (PVOID)0x000007FFFFFEFFFF; + MemoryLayout.NonCanonicalStart = (PVOID)0x0000800000000000; + MemoryLayout.NonCanonicalEnd = (PVOID)0xFFFF7FFFFFFFFFFF; + MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFFFF800000000000; + MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFFFF67FFFFFFFFF; + MemoryLayout.PteSpaceStart = (PVOID)0xFFFFF68000000000; + MemoryLayout.PteSpaceEnd = (PVOID)0xFFFFF6FFFFFFFFFF; + MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000; + MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF; + MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000; + MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF; + MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000; + MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF; + MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000; + MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF; + MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000; + MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF; + MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000; + MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF; + MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000; + MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000; + MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000; + MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF; + MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000; + MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF; + MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000; + MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF; + } + + /* Compute allocation size for the PFN database */ + MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize); + + /* Compute boot image size */ + ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize); + + /* Compute session space size */ + ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize); + + /* Update loader mappings space end address */ + MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONGLONG)MemoryLayout.LoaderMappingsStart + + MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE); + + /* Update session space start address */ + MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd - + MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE); + + /* Compute system PTE size */ + ComputeSystemPteSize(&NumberOfSystemPtes); + + /* Compute non-paged pool size */ + ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize); + ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize); + + /* Compute paged pool size */ + ComputePagedPoolSize(&MemoryLayout.PagedPoolSize); + + /* Insert the PFN database at the beginning of the non-paged pool */ + MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.NonPagedPoolStart; + + /* Compute the PFN database page-aligned end address */ + PfnDatabaseEnd = (ULONGLONG)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE); + PfnDatabaseEnd = ROUND_UP(PfnDatabaseEnd, MM_PAGE_SIZE); + + /* Shrink the non-paged pool to fit the PFN database */ + MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd; + + /* Assign the rest of the non-paged pool to the expansion pool */ + MemoryLayout.NonPagedExpansionPoolStart = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart + + MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE); + MemoryLayout.NonPagedPoolEnd = MemoryLayout.NonPagedExpansionPoolStart; + MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart + + MaximumNonPagedPoolSize * MM_PAGE_SIZE); + MemoryLayout.NonPagedExpansionPoolSize = ((ULONGLONG)MemoryLayout.NonPagedExpansionPoolEnd - + (ULONGLONG)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE; + + /* Update paged pool end address */ + MemoryLayout.PagedPoolEnd = (PVOID)(((ULONGLONG)MemoryLayout.PagedPoolStart + + MemoryLayout.PagedPoolSize * MM_PAGE_SIZE) - 1); +} diff --git a/xtoskrnl/mm/amd64/pagemap.cc b/xtoskrnl/mm/amd64/pagemap.cc index 62fa00b..2d349a3 100644 --- a/xtoskrnl/mm/amd64/pagemap.cc +++ b/xtoskrnl/mm/amd64/pagemap.cc @@ -9,6 +9,28 @@ #include +/** + * Advances a PTE pointer by a given number of entries, considering the actual PTE size. + * + * @param Pte + * The PTE pointer to advance. + * + * @param Count + * The number of PTE entries to advance by. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMap::AdvancePte(IN PMMPTE Pte, + IN LONG Count) +{ + /* Return advanced PTE pointer */ + return (PMMPTE)((ULONG_PTR)Pte + (Count * sizeof(MMPTE))); +} + /** * Clears the contents of a page table entry (PTE). * @@ -21,13 +43,64 @@ */ XTAPI VOID -MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer) +MM::PageMap::ClearPte(IN PMMPTE PtePointer) { - PtePointer->CacheDisable = 0; - PtePointer->PageFrameNumber = 0; - PtePointer->Valid = 0; - PtePointer->Writable = 0; - PtePointer->WriteThrough = 0; + /* Clear PTE */ + PtePointer->Long = 0; +} + +/** + * Gets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to get the next entry from. + * + * @return This routine returns the next entry in the PTE list. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMap::GetNextEntry(IN PMMPTE Pte) +{ + /* Return next entry in PTE list */ + return Pte->List.NextEntry; +} + +/** + * Advances a PTE pointer, considering the actual PTE size. + * + * @param Pte + * The PTE pointer to advance. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMap::GetNextPte(IN PMMPTE Pte) +{ + /* Return advanced PTE pointer */ + return AdvancePte(Pte, 1); +} + +/** + * Checks if a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to check. + * + * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::PageMap::GetOneEntry(IN PMMPTE Pte) +{ + /* Return one entry status */ + return Pte->List.OneEntry; } /** @@ -42,7 +115,7 @@ MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer) */ XTAPI PMMP5E -MM::PageMap::GetP5eAddress(PVOID Address) +MM::PageMap::GetP5eAddress(IN PVOID Address) { ULONGLONG Offset; @@ -51,6 +124,72 @@ MM::PageMap::GetP5eAddress(PVOID Address) return (PMMP5E)((PageMapInfo.P5eBase + Offset) * PageMapInfo.Xpa); } +/** + * Gets the Offset of the P5E (Page Map Level 5 Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding P5E. + * + * @return This routine returns the Offset of the P5E, or NULLPTR if LA57 is not enabled. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetP5eOffset(IN PVOID Address) +{ + /* Return P5E Offset */ + return (((((ULONGLONG)Address) >> MM_P5I_SHIFT) & 0x1FF) * PageMapInfo.Xpa); +} + +/** + * Gets the virtual address that is mapped by a given Page Map Level 5 Entry. + * + * @param P5ePointer + * Specifies the address of the P5E. + * + * @return This routine returns the virtual address mapped by the P5E, or NULLPTR if LA57 is not enabled. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMap::GetP5eVirtualAddress(IN PMMP5E P5ePointer) +{ + return (PVOID)((((LONGLONG)P5ePointer << 52) >> 7) * PageMapInfo.Xpa); +} + +/** + * Gets the page frame number from a corresponding PTE. + * + * @param Pte + * The PTE pointer to get the page frame number from. + * + * @return This routine returns the page frame number. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::PageMap::GetPageFrameNumber(IN PMMPTE Pte) +{ + return Pte->Hardware.PageFrameNumber; +} + +/** + * Gets Page Map Level (PML) for current paging mode. + * + * @return This routine returns the page map level. + * + * @since XT 1.0 + */ +XTAPI +USHORT +MM::PageMap::GetPageMapLevel() +{ + return PageMapInfo.Xpa ? 5 : 4; +} + /** * Gets the address of the PDE (Page Directory Entry), that maps given address. * @@ -63,7 +202,7 @@ MM::PageMap::GetP5eAddress(PVOID Address) */ XTAPI PMMPDE -MM::PageMap::GetPdeAddress(PVOID Address) +MM::PageMap::GetPdeAddress(IN PVOID Address) { ULONGLONG Offset; @@ -73,18 +212,36 @@ MM::PageMap::GetPdeAddress(PVOID Address) } /** - * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address. + * Gets the Offset of the PDE (Page Directory Entry), that maps given address. * * @param Address * Specifies the virtual address for which to retrieve the corresponding PDE. * + * @return This routine returns the Offset of the PDE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPdeOffset(IN PVOID Address) +{ + /* Return PDE Offset */ + return ((((ULONGLONG)Address) >> MM_PDI_SHIFT) & 0x1FF); +} + +/** + * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PPE. + * * @return This routine returns the address of the PPE. * * @since XT 1.0 */ XTAPI PMMPPE -MM::PageMap::GetPpeAddress(PVOID Address) +MM::PageMap::GetPpeAddress(IN PVOID Address) { ULONGLONG Offset; @@ -93,6 +250,42 @@ MM::PageMap::GetPpeAddress(PVOID Address) return (PMMPPE)(PageMapInfo.PpeBase + Offset); } +/** + * Gets the Offset of the PPE (Page Directory Pointer Table Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PPE. + * + * @return This routine returns the Offset of the PPE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPpeOffset(IN PVOID Address) +{ + /* Return PPE Offset */ + return ((((ULONGLONG)Address) >> MM_PPI_SHIFT) & 0x1FF); +} + + /** + * Gets the entire contents of a Page Table Entry (PTE) as a single value. + * + * @param PtePointer + * Pointer to the Page Table Entry (PTE) to read. + * + * @return This routine returns the contents of the PTE as a single value. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMap::GetPte(IN PMMPTE PtePointer) +{ + /* Return PTE value */ + return PtePointer->Long; +} + /** * Gets the address of the PTE (Page Table Entry), that maps given address. * @@ -105,7 +298,7 @@ MM::PageMap::GetPpeAddress(PVOID Address) */ XTAPI PMMPTE -MM::PageMap::GetPteAddress(PVOID Address) +MM::PageMap::GetPteAddress(IN PVOID Address) { ULONGLONG Offset; @@ -114,6 +307,115 @@ MM::PageMap::GetPteAddress(PVOID Address) return (PMMPTE)(PageMapInfo.PteBase + Offset); } +/** + * Calculates the distance between two PTE pointers. + * + * @param EndPte + * Pointer to the ending Page Table Entry. + * + * @param StartPte + * Pointer to the starting Page Table Entry. + * + * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte. + * + * @since XT 1.0 + */ +XTAPI +LONG +MM::PageMap::GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte) +{ + /* Return distance between PTE pointers */ + return ((ULONG_PTR)EndPte - (ULONG_PTR)StartPte) / sizeof(MMPTE); +} + +/** + * Gets the Offset of the PTE (Page Table Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PTE. + * + * @return This routine returns the Offset of the PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteOffset(IN PVOID Address) +{ + /* Return PTE Offset */ + return ((((ULONGLONG)Address) >> MM_PTI_SHIFT) & 0x1FF); +} + +/** + * Gets the size of a PTE. + * + * @return This routine returns the size of a PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteSize(VOID) +{ + /* Return the size of MMPTE */ + return sizeof(MMPTE); +} + +/** + * Gets the software protection value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software protection value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteSoftwareProtection(IN PMMPTE PtePointer) +{ + /* Return PTE software protection value */ + return (ULONG)PtePointer->Software.Protection; +} + +/** + * Gets the software prototype value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software prototype value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteSoftwarePrototype(IN PMMPTE PtePointer) +{ + /* Return PTE software prototype value */ + return (ULONG)PtePointer->Software.Prototype; +} + +/** + * Gets the software transition value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software transition value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteSoftwareTransition(IN PMMPTE PtePointer) +{ + /* Return PTE software transition value */ + return (ULONG)PtePointer->Software.Transition; +} + /** * Gets the address of the PXE (Extended Page Entry), that maps given address. * @@ -126,14 +428,47 @@ MM::PageMap::GetPteAddress(PVOID Address) */ XTAPI PMMPXE -MM::PageMap::GetPxeAddress(PVOID Address) +MM::PageMap::GetPxeAddress(IN PVOID Address) { ULONGLONG Offset; + /* Calculate offset and return PXE address */ Offset = ((((ULONGLONG)Address & (((ULONGLONG)1 << PageMapInfo.VaBits) - 1)) >> MM_PXI_SHIFT) << MM_PTE_SHIFT); return (PMMPXE)(PageMapInfo.PxeBase + Offset); } +/** + * Gets the Offset of the PXE (Extended Page Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PXE. + * + * @return This routine returns the Offset of the PXE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPxeOffset(IN PVOID Address) +{ + /* Return PXE Offset */ + return ((((ULONGLONG)Address) >> MM_PXI_SHIFT) & 0x1FF); +} + +/** + * Gets the status of Extended Paging Address (XPA) mode. + * + * @return This routine returns TRUE if XPA is enabled, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::PageMap::GetXpaStatus() +{ + return PageMapInfo.Xpa; +} + /** * Checks whether the given PML2 page table entry (PTE) is valid. * @@ -146,22 +481,20 @@ MM::PageMap::GetPxeAddress(PVOID Address) */ XTAPI BOOLEAN -MM::PageMap::PteValid(PHARDWARE_PTE PtePointer) +MM::PageMap::PteValid(IN PMMPTE PtePointer) { - return (BOOLEAN)PtePointer->Valid; + /* Check if PTE is valid */ + return (BOOLEAN)PtePointer->Hardware.Valid; } /** - * Sets a PML2 page table entry (PTE) with the specified physical page and access flags. + * Sets the next entry in a PTE list. * - * @param PtePointer - * Pointer to the page table entry (PTE) to set. + * @param Pte + * The PTE pointer to modify. * - * @param PageFrameNumber - * Physical frame number to map. - * - * @param Writable - * Indicates whether the page should be writable. + * @param Value + * The value to set as the next entry. * * @return This routine does not return any value. * @@ -169,13 +502,82 @@ MM::PageMap::PteValid(PHARDWARE_PTE PtePointer) */ XTAPI VOID -MM::PageMap::SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable) +MM::PageMap::SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value) { - PtePointer->PageFrameNumber = PageFrameNumber; - PtePointer->Valid = 1; - PtePointer->Writable = Writable; + /* Set next entry in PTE list */ + Pte->List.NextEntry = Value; +} + +/** + * Sets the flag indicating whether a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set. TRUE if the list has only one entry, FALSE otherwise. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMap::SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value) +{ + /* Set one entry status */ + Pte->List.OneEntry = Value; +} + +/** + * Sets a Page Table Entry (PTE) with the specified physical page and access flags. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param PageFrameNumber + * Physical frame number to map. + * + * @param AttributesMask + * Specifies the attributes mask to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMap::SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask) +{ + /* Set PTE */ + PtePointer->Hardware.PageFrameNumber = PageFrameNumber; + PtePointer->Hardware.Valid = 1; + PtePointer->Long |= AttributesMask; +} + +/** + * Sets a Page Table Entry (PTE) with the specified attributes. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param Attributes + * Specifies the attributes to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMap::SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes) +{ + PtePointer->Long = Attributes; } /** @@ -196,12 +598,138 @@ MM::PageMap::SetPte(PHARDWARE_PTE PtePointer, */ XTAPI VOID -MM::PageMap::SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough) +MM::PageMap::SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough) { - PtePointer->CacheDisable = CacheDisable; - PtePointer->WriteThrough = WriteThrough; + /* Set caching attributes */ + PtePointer->Hardware.CacheDisable = CacheDisable; + PtePointer->Hardware.WriteThrough = WriteThrough; +} + +/** + * Transitions a Page Table Entry (PTE) to invalid state + * + * @param PointerPte + * Pointer to the page table entry (PTE) to transition. + * + * @param Protection + * Specifies the protection attribute to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMap::TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection) +{ + MMPTE TempPte; + + /* Set transition PTE */ + TempPte = *PointerPte; + TempPte.Software.Protection = Protection; + TempPte.Software.Prototype = 0; + TempPte.Software.Transition = 1; + TempPte.Software.Valid = 0; + + /* Write PTE value */ + *PointerPte = TempPte; +} + +/** + * Writes a Page Table Entry (PTE) with the specified value. + * + * @param Pte + * Pointer to the page table entry (PTE) to write. + * + * @param Value + * The value to write to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMap::WritePte(IN PMMPTE Pte, + IN MMPTE Value) +{ + /* Write PTE value */ + Pte->Long = Value.Long; +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Entry (PML4). + * + * @param PdePointer + * Specifies the address of the PDE. + * + * @return This routine returns the virtual address mapped by the PDE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPdeVirtualAddress(IN PMMPDE PdePointer) +{ + /* Return PDE virtual address */ + return (PVOID)(((LONGLONG)PdePointer << 34) >> 16); +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry (PML4). + * + * @param PpePointer + * Specifies the address of the PPE. + * + * @return This routine returns the virtual address mapped by the PPE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPpeVirtualAddress(IN PMMPPE PpePointer) +{ + /* Return PPE virtual address */ + return (PVOID)(((LONGLONG)PpePointer << 43) >> 16); +} + +/** + * Gets the virtual address that is mapped by a given Page Table Entry (PML4). + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the virtual address mapped by the PTE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPteVirtualAddress(IN PMMPTE PtePointer) +{ + /* Return PTE virtual address */ + return (PVOID)(((LONGLONG)PtePointer << 25) >> 16); +} + +/** + * Gets the virtual address that is mapped by a given Extended Page Entry (PML4). + * + * @param PxePointer + * Specifies the address of the PXE. + * + * @return This routine returns the virtual address mapped by the PXE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPxeVirtualAddress(IN PMMPXE PxePointer) +{ + /* Return PXE virtual address */ + return (PVOID)(((LONGLONG)PxePointer << 52) >> 16); } /** @@ -229,6 +757,78 @@ MM::PageMapBasic::InitializePageMapInfo(VOID) PageMapInfo.VaBits = 48; } +/** + * Gets the virtual address that is mapped by a given Page Directory Entry (PML5). + * + * @param PdePointer + * Specifies the address of the PDE. + * + * @return This routine returns the virtual address mapped by the PDE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPdeVirtualAddress(IN PMMPDE PdePointer) +{ + /* Return PDE virtual address */ + return (PVOID)(((LONGLONG)PdePointer << 25) >> 7); +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry (PML5). + * + * @param PpePointer + * Specifies the address of the PPE. + * + * @return This routine returns the virtual address mapped by the PPE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPpeVirtualAddress(IN PMMPPE PpePointer) +{ + /* Return PPE virtual address */ + return (PVOID)(((LONGLONG)PpePointer << 34) >> 7); +} + +/** + * Gets the virtual address that is mapped by a given Page Table Entry (PML5). + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the virtual address mapped by the PTE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPteVirtualAddress(IN PMMPTE PtePointer) +{ + /* Return PTE virtual address */ + return (PVOID)(((LONGLONG)PtePointer << 16) >> 7); +} + +/** + * Gets the virtual address that is mapped by a given Extended Page Entry (PML5). + * + * @param PxePointer + * Specifies the address of the PXE. + * + * @return This routine returns the virtual address mapped by the PXE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPxeVirtualAddress(IN PMMPXE PxePointer) +{ + /* Return PXE virtual address */ + return (PVOID)(((LONGLONG)PxePointer << 43) >> 7); +} + /** * Initializes page map information for XPA paging (PML5). * diff --git a/xtoskrnl/mm/amd64/paging.cc b/xtoskrnl/mm/amd64/paging.cc index 0ab73ef..a04e007 100644 --- a/xtoskrnl/mm/amd64/paging.cc +++ b/xtoskrnl/mm/amd64/paging.cc @@ -25,6 +25,78 @@ MM::Paging::GetExtendedPhysicalAddressingStatus(VOID) return ((AR::CpuFunc::ReadControlRegister(4) & CR4_LA57) != 0) ? TRUE : FALSE; } +/** + * Gets the address of the P5E (Page Map Level 5 Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding P5E. + * + * @return This routine returns the address of the P5E, or NULLPTR if LA57 is not enabled. + * + * @since XT 1.0 + */ +XTAPI +PMMP5E +MM::Paging::GetP5eAddress(IN PVOID Address) +{ + /* Return PDE address */ + return PmlRoutines->GetP5eAddress(Address); +} + +/** + * Gets the virtual address that is mapped by a given Page Map Level 5 Entry. + * + * @param P5ePointer + * Specifies the address of the P5E. + * + * @return This routine returns the virtual address mapped by the P5E, or NULLPTR if LA57 is not enabled. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::Paging::GetP5eVirtualAddress(IN PMMP5E P5ePointer) +{ + /* Return PTE virtual address */ + return PmlRoutines->GetP5eVirtualAddress(P5ePointer); +} + +/** + * Gets the address of the PXE (Extended Page Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PXE. + * + * @return This routine returns the address of the PXE. + * + * @since XT 1.0 + */ +XTAPI +PMMPXE +MM::Paging::GetPxeAddress(IN PVOID Address) +{ + /* Return PXE address */ + return PmlRoutines->GetPxeAddress(Address); +} + +/** + * Gets the virtual address that is mapped by a given Extended Page Entry. + * + * @param PxePointer + * Specifies the address of the PXE. + * + * @return This routine returns the virtual address mapped by the PXE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::Paging::GetPxeVirtualAddress(IN PMMPXE PxePointer) +{ + /* Return PXE virtual address */ + return PmlRoutines->GetPxeVirtualAddress(PxePointer); +} + /** * Fills a section of memory with zeroes like RtlZeroMemory(), but in more efficient way. * diff --git a/xtoskrnl/mm/amd64/pfault.cc b/xtoskrnl/mm/amd64/pfault.cc new file mode 100644 index 0000000..a3d5b9d --- /dev/null +++ b/xtoskrnl/mm/amd64/pfault.cc @@ -0,0 +1,26 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/amd64/pfault.cc + * DESCRIPTION: Page fault support for AMD64 architecture + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Evaluates the PDE for for paged pool and per-session mappings. + * + * @param VirtualAddress + * Specifies the virtual address to verify. + * + * @return This routine returns ACCESS_VIOLATION regardless PML4 or PML5 is used. + */ +XTFASTCALL +XTSTATUS +MM::PageFault::CheckPdeForPagedPool(IN PVOID VirtualAddress) +{ + /* Return access violation */ + return STATUS_ACCESS_VIOLATION; +} diff --git a/xtoskrnl/mm/amd64/pfn.cc b/xtoskrnl/mm/amd64/pfn.cc new file mode 100644 index 0000000..91e8d1d --- /dev/null +++ b/xtoskrnl/mm/amd64/pfn.cc @@ -0,0 +1,342 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/amd64/pfn.cc + * DESCRIPTION: Physical Frame Number for AMD64 support + * DEVELOPERS: Aiken Harris + * Rafal Kupiec + */ + +#include + + +/** + * Allocates and initializes page directory structures for a range of PDEs. + * + * @param StartingPde + * Supplies a pointer to the first PDE in the range to initialize. + * + * @param EndingPde + * Supplies a pointer to the last PDE in the range to initialize. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePageDirectory(IN PMMPDE StartingPde, + IN PMMPDE EndingPde) +{ + PMMPTE ParentPte, ValidPte; + BOOLEAN PteValidated; + + /* Get a template PTE for mapping the PFN database pages */ + ValidPte = MM::Pte::GetValidPte(); + + /* Initialize validation flag */ + PteValidated = FALSE; + + /* Iterate through the range of PDEs to ensure the paging hierarchy is fully mapped */ + while(StartingPde <= EndingPde) + { + /* Check if there is a need to validate upper-level page table entries */ + if(!PteValidated || ((ULONG_PTR)StartingPde & (MM_PAGE_SIZE - 1)) == 0) + { + /* For LA57, ensure PML5 entry exists */ + if(MM::Paging::GetXpaStatus()) + { + /* Get the P5E that maps the PXE page containing this hierarchy */ + ParentPte = MM::Paging::GetPpeAddress(StartingPde); + + /* Check if P5E entry is valid */ + if(!MM::Paging::PteValid(ParentPte)) + { + /* Allocate a new PML4 page and map P5E to it */ + MM::Paging::SetPte(ValidPte, AllocateBootstrapPages(1), 0); + *ParentPte = *ValidPte; + + /* Clear the newly created page */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(ParentPte), MM_PAGE_SIZE); + } + } + + /* Get the PXE that maps the PPE page containing PDE */ + ParentPte = MM::Paging::GetPdeAddress(StartingPde); + + /* Check if PXE entry is valid */ + if(!MM::Paging::PteValid(ParentPte)) + { + /* Allocate a new PPE page and map PXE to it */ + MM::Paging::SetPte(ValidPte, AllocateBootstrapPages(1), 0); + *ParentPte = *ValidPte; + + /* Clear the newly created page */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(ParentPte), MM_PAGE_SIZE); + } + + /* Get the PPE that maps the PDE page containing PTE */ + ParentPte = MM::Paging::GetPteAddress(StartingPde); + + /* Check if PPE entry is valid */ + if(!MM::Paging::PteValid(ParentPte)) + { + /* Allocate a new PDE page and map PPE to it */ + MM::Paging::SetPte(ValidPte, AllocateBootstrapPages(1), 0); + *ParentPte = *ValidPte; + + /* Clear the newly created page */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(ParentPte), MM_PAGE_SIZE); + } + + /* Upper levels for this PDE have been validated */ + PteValidated = TRUE; + } + + /* Ensure the PDE has a PTE page allocated */ + if(!MM::Paging::PteValid(StartingPde)) + { + /* Allocate a new PTE page and map PDE to it */ + MM::Paging::SetPte(ValidPte, AllocateBootstrapPages(1), 0); + *StartingPde = *ValidPte; + } + + /* Move to the next PDE */ + StartingPde = MM::Paging::GetNextPte(StartingPde); + } +} + +/** + * Initializes the PFN database by mapping virtual memory and populating entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePfnDatabase(VOID) +{ + PKERNEL_INITIALIZATION_BLOCK InitializationBlock; + PLIST_ENTRY ListEntry; + PLOADER_MEMORY_DESCRIPTOR Descriptor; + PUCHAR PfnDatabaseEnd; + PMMMEMORY_LAYOUT MemoryLayout; + PMMPTE ValidPte; + + /* Raise runlevel and acquire the PFN lock */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Get the kernel initialization block */ + InitializationBlock = KE::BootInformation::GetInitializationBlock(); + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Get the PFN database size and calculate the end of the PFN database virtual address space */ + PfnDatabaseEnd = (PUCHAR)MemoryLayout->PfnDatabase + (MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE) - 1; + + /* Get a template PTE for mapping the PFN database pages */ + ValidPte = MM::Pte::GetValidPte(); + + /* Map the Page Directory and Page Directory Pointer tables for the PFN database */ + MM::Pte::MapPPE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte); + MM::Pte::MapPDE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte); + MM::Pte::MapPTE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte); + + /* Zero PFN database virtual space */ + RTL::Memory::ZeroMemory(MemoryLayout->PfnDatabase, MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE); + + /* Initialize the color tables */ + MM::Colors::InitializeColorTables(); + + /* Iterate over memory descriptors to map the PFN database and initialize entries */ + ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink; + while(ListEntry != &InitializationBlock->MemoryDescriptorListHead) + { + /* Get the descriptor */ + Descriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_DESCRIPTOR, ListEntry); + + /* Skip invisible memory regions */ + if(MM::Manager::VerifyMemoryTypeInvisible(Descriptor->MemoryType)) + { + /* Move to the next descriptor and continue */ + ListEntry = ListEntry->Flink; + continue; + } + + /* Check if this is the modified free descriptor */ + if(Descriptor == FreeDescriptor) + { + /* Switch to the original descriptor */ + Descriptor = &OriginalFreeDescriptor; + } + + /* Check if the free memory block that was split is being processed */ + if(Descriptor == &OriginalFreeDescriptor) + { + /* Skip loop processing, free memory is initialized separately */ + ListEntry = ListEntry->Flink; + continue; + } + + /* Initialize PFNs for this memory range */ + ProcessMemoryDescriptor(Descriptor->BasePage, Descriptor->PageCount, Descriptor->MemoryType); + + /* Move to the next descriptor */ + ListEntry = ListEntry->Flink; + } + + /* Initialize PFNs for the free memory */ + ProcessMemoryDescriptor(FreeDescriptor->BasePage, FreeDescriptor->PageCount, LoaderFree); + + /* Initialize PFNs for the physical pages backing the PFN database */ + ProcessMemoryDescriptor(OriginalFreeDescriptor.BasePage, + FreeDescriptor->BasePage - OriginalFreeDescriptor.BasePage, + LoaderMemoryData); + + /* Restore original free descriptor */ + *FreeDescriptor = OriginalFreeDescriptor; + + /* Initialize PFNs backing page tables */ + InitializePageTablePfns(); +} + +/** + * Initializes PFN database entries for the system page tables. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePageTablePfns(VOID) +{ + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn; + ULONG RootLevel; + PMMPTE RootPte; + + /* Determine root structure based on paging mode */ + if(MM::Paging::GetXpaStatus()) + { + /* XPA enabled, 5-level paging (LA57) */ + RootLevel = 5; + + /* Retrieve the PFN of the PML5 table and its virtual base address */ + PageFrameIndex = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress((PVOID)MM_P5E_LA57_BASE)); + RootPte = (PMMPTE)MM::Paging::GetP5eAddress(NULLPTR); + } + else + { + /* XPA disabled, 4-level paging */ + RootLevel = 4; + + /* Retrieve the PFN of the PML4 table and its virtual base address */ + PageFrameIndex = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress((PVOID)MM_PXE_BASE)); + RootPte = (PMMPTE)MM::Paging::GetPxeAddress(NULLPTR); + } + + /* Initialize the PFN entry for the root page table itself */ + Pfn = GetPfnEntry(PageFrameIndex); + if(Pfn) + { + /* Initialize the PFN entry */ + Pfn->PteAddress = NULLPTR; + Pfn->u1.WsIndex = 0; + Pfn->u2.ShareCount = 1; + Pfn->u3.e1.CacheAttribute = PfnNonCached; + Pfn->u3.e2.ReferenceCount = 1; + Pfn->u4.PteFrame = 0; + } + + /* Start recursive scan from the top level */ + if(RootPte) + { + /* Scan the root page table */ + ScanPageTable(RootPte, RootLevel); + } +} + +/** + * Recursively scans a page table to initialize PFN database entries for active pages. + * + * @param PointerPte + * Pointer to the base of the page table to scan. + * + * @param Level + * The paging level of the table being scanned. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::ScanPageTable(IN PMMPTE PointerPte, + IN ULONG Level) +{ + PVOID Address; + ULONG Index; + PMMPTE NextLevelPte; + ULONG PtesPerPage; + + /* Get the number of PTEs per page */ + PtesPerPage = MM::Pte::GetPtesPerPage(); + + /* Iterate through all entries in the current page table */ + for(Index = 0; Index < PtesPerPage; Index++) + { + /* Check if the page table entry is present */ + if(MM::Paging::PteValid(PointerPte)) + { + /* Mark the PFN pointed to by this entry as active */ + LinkPfnForPageTable(MM::Paging::GetPageFrameNumber(PointerPte), PointerPte); + + /* Recurse to the next level, if this is not a leaf node (PTE) */ + if(Level > 1) + { + /* Calculate the virtual address mapped by this entry to find the next table */ + switch(Level) + { + case 5: + /* Calculate PXE */ + Address = MM::Paging::GetP5eVirtualAddress((PMMP5E)PointerPte); + NextLevelPte = (PMMPTE)MM::Paging::GetPxeAddress(Address); + break; + case 4: + /* Calculate PPE */ + Address = MM::Paging::GetPxeVirtualAddress((PMMPXE)PointerPte); + NextLevelPte = (PMMPTE)MM::Paging::GetPpeAddress(Address); + break; + case 3: + /* Calculate PDE */ + Address = MM::Paging::GetPpeVirtualAddress((PMMPPE)PointerPte); + NextLevelPte = (PMMPTE)MM::Paging::GetPdeAddress(Address); + break; + case 2: + /* Calculate PTE */ + Address = MM::Paging::GetPdeVirtualAddress((PMMPDE)PointerPte); + NextLevelPte = MM::Paging::GetPteAddress(Address); + break; + default: + /* Nothing to calculate, return NULLPTR */ + NextLevelPte = NULLPTR; + break; + } + + /* Recurse deeper if not at the bottom level (PTE) already */ + if(NextLevelPte) + { + /* Recursively scan the next level page table */ + ScanPageTable(NextLevelPte, Level - 1); + } + } + } + + /* Move to the next entry in the current table */ + PointerPte = MM::Paging::GetNextPte(PointerPte); + } +} diff --git a/xtoskrnl/mm/amd64/pte.cc b/xtoskrnl/mm/amd64/pte.cc new file mode 100644 index 0000000..2ebc9ff --- /dev/null +++ b/xtoskrnl/mm/amd64/pte.cc @@ -0,0 +1,295 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/amd64/pte.cc + * DESCRIPTION: Page Table Entry (PTE) for AMD64 support + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Checks if the virtual address is valid and mapped in the page tables. + * + * @param VirtualAddress + * The virtual address to check. + * + * @return This routine returns TRUE if the address is valid, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Pte::AddressValid(IN PVOID VirtualAddress) +{ + /* Check XPA status */ + if(MM::Paging::GetXpaStatus()) + { + /* Check if the P5E is valid */ + if(!MM::Paging::PteValid(MM::Paging::GetP5eAddress(VirtualAddress))) + { + /* Invalid P5E, return FALSE */ + return FALSE; + } + } + + /* Check if PXE, PPE, PDE and PTE are valid */ + if(!MM::Paging::PteValid(MM::Paging::GetPxeAddress(VirtualAddress)) || + !MM::Paging::PteValid(MM::Paging::GetPpeAddress(VirtualAddress)) || + !MM::Paging::PteValid(MM::Paging::GetPdeAddress(VirtualAddress)) || + !MM::Paging::PteValid(MM::Paging::GetPteAddress(VirtualAddress))) + { + /* Invalid PXE, PPE, PDE or PTE, return FALSE */ + return FALSE; + } + + /* Address is valid, return TRUE */ + return TRUE; +} + +/** + * Retrieves the base virtual address of the system PTEs. + * + * @return This routine returns a pointer to the first PTE in the system PTE space. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Pte::GetSystemPteBaseAddress(VOID) +{ + PMMMEMORY_LAYOUT MemoryLayout; + + /* Retrieve the system's memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Determine the base address for system PTEs based on the paging mode */ + if(MM::Paging::GetXpaStatus()) + { + /* For 5-level paging, system PTEs start at the beginning of system space */ + return MM::Paging::GetPteAddress((PVOID)MemoryLayout->NonPagedSystemPoolStart); + } + else + { + /* For 4-level paging, system PTEs start at the legacy KSEG0_BASE */ + return MM::Paging::GetPteAddress((PVOID)KSEG0_BASE); + } +} + +/** + * Performs the initial setup of the system's page table hierarchy. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::InitializePageTable(VOID) +{ + PMMPTE EndSpacePte, PointerPte; + PMMMEMORY_LAYOUT MemoryLayout; + PVOID MappingRange; + MMPTE TemplatePte; + BOOLEAN Xpa; + + /* Retrieve current paging mode and memory layout */ + Xpa = MM::Paging::GetXpaStatus(); + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Enable the Global Paging (PGE) feature */ + AR::CpuFunc::WriteControlRegister(4, AR::CpuFunc::ReadControlRegister(4) | CR4_PGE); + + /* Check XPA status */ + if(Xpa) + { + /* Get the PML5 user-space range if 5-level paging is active */ + PointerPte = MM::Paging::GetP5eAddress(0); + EndSpacePte = MM::Paging::GetP5eAddress(MemoryLayout->UserSpaceEnd); + } + else + { + /* Otherwise, get the PML4 user-space range for 4-level paging */ + PointerPte = MM::Paging::GetPxeAddress(0); + EndSpacePte = MM::Paging::GetPxeAddress(MemoryLayout->UserSpaceEnd); + } + + /* Clear all top-level entries mapping the user address space */ + while(PointerPte <= EndSpacePte) + { + MM::Paging::ClearPte(PointerPte); + PointerPte = MM::Paging::GetNextPte(PointerPte); + } + + /* Flush the TLB to invalidate all non-global entries */ + AR::CpuFunc::FlushTlb(); + + /* Create a template PTE for mapping kernel pages */ + MM::Paging::ClearPte(&TemplatePte); + MM::Paging::SetPte(&TemplatePte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE); + + /* Check XPA status */ + if(Xpa) + { + /* Map the kernel's PML5 entries if 5-level paging is active */ + MM::Pte::MapP5E(MemoryLayout->HyperSpaceStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte); + } + + /* Map the kernel's PML4 entries */ + MM::Pte::MapPXE(MemoryLayout->HyperSpaceStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte); + + /* Calculate the end address of the hyperspace working set mapping */ + MappingRange = (PVOID)((ULONG_PTR)MemoryLayout->HyperSpaceStart + MM_HYPERSPACE_PAGE_COUNT * MM_PAGE_SIZE); + + /* Map the PDPT entries for paged pool and hyperspace */ + MM::Pte::MapPPE(MemoryLayout->PagedPoolStart, MemoryLayout->PagedPoolEnd, &ValidPte); + MM::Pte::MapPPE(MemoryLayout->HyperSpaceStart, MemoryLayout->HyperSpaceEnd, &ValidPte); + + /* Map the PDEs for the hyperspace working set */ + MM::Pte::MapPDE(MemoryLayout->HyperSpaceStart, MappingRange, &ValidPte); + + /* Set the hyperspace working set's PTE with the total PTE count */ + MM::Paging::SetPte(MM::Paging::GetPteAddress((PVOID)MemoryLayout->HyperSpaceStart), MM_HYPERSPACE_PAGE_COUNT, 0); +} + +/** + * Maps a range of virtual addresses at the P5E (PML5) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplateP5e + * A template P5E to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapP5E(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMP5E TemplateP5e) +{ + PMMP5E EndSpace, PointerP5e; + + /* Get P5E addresses */ + PointerP5e = MM::Paging::GetP5eAddress(StartAddress); + EndSpace = MM::Paging::GetP5eAddress(EndAddress); + + /* Iterate over all P5Es */ + while(PointerP5e <= EndSpace) + { + /* Check if P5E is already mapped */ + if(!MM::Paging::PteValid(PointerP5e)) + { + /* Map P5E */ + MM::Paging::SetPte(TemplateP5e, MM::Pfn::AllocateBootstrapPages(1), 0); + *PointerP5e = *TemplateP5e; + + /* Clear the page table */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerP5e), MM_PAGE_SIZE); + } + + /* Get next table entry */ + PointerP5e = MM::Paging::GetNextPte(PointerP5e); + } +} + +/** + * Maps a range of virtual addresses at the PPE (Page Directory Pointer Entry) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplatePpe + * A template PPE to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapPPE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPPE TemplatePpe) +{ + PMMPPE EndSpace, PointerPpe; + + /* Get PPE addresses */ + PointerPpe = MM::Paging::GetPpeAddress(StartAddress); + EndSpace = MM::Paging::GetPpeAddress(EndAddress); + + /* Iterate over all PPEs */ + while(PointerPpe <= EndSpace) + { + /* Check if PPE is already mapped */ + if(!MM::Paging::PteValid(PointerPpe)) + { + /* Map PPE */ + MM::Paging::SetPte(TemplatePpe, MM::Pfn::AllocateBootstrapPages(1), 0); + *PointerPpe = *TemplatePpe; + + /* Clear the page table */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPpe), MM_PAGE_SIZE); + } + + /* Get next table entry */ + PointerPpe = MM::Paging::GetNextPte(PointerPpe); + } +} + +/** + * Maps a range of virtual addresses at the PXE (PML4) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplatePxe + * A template PXE to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapPXE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPXE TemplatePxe) +{ + PMMPXE EndSpace, PointerPxe; + + /* Get PXE addresses */ + PointerPxe = MM::Paging::GetPxeAddress(StartAddress); + EndSpace = MM::Paging::GetPxeAddress(EndAddress); + + /* Iterate over all PTEs */ + while(PointerPxe <= EndSpace) + { + /* Check if PTE is already mapped */ + if(!MM::Paging::PteValid(PointerPxe)) + { + /* Map PTE */ + MM::Paging::SetPte(TemplatePxe, MM::Pfn::AllocateBootstrapPages(1), 0); + *PointerPxe = *TemplatePxe; + + /* Clear the page table */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPxe), MM_PAGE_SIZE); + } + + /* Get next table entry */ + PointerPxe = MM::Paging::GetNextPte(PointerPxe); + } +} diff --git a/xtoskrnl/mm/colors.cc b/xtoskrnl/mm/colors.cc new file mode 100644 index 0000000..45f2fc9 --- /dev/null +++ b/xtoskrnl/mm/colors.cc @@ -0,0 +1,178 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/colors.cc + * DESCRIPTION: Memory manager page coloring subsystem + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Computes & initializes the system's page coloring. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Colors::ComputePageColoring(VOID) +{ + UNIMPLEMENTED; + + /* Compute L2 paging colors and mask */ + PagingColors = MM_PAGING_COLORS; + PagingColorsMask = PagingColors - 1; +} + +/** + * Retrieves a pointer to the color table for a specific page list and color. + * + * @param PageList + * The page list type (e.g., FreePageList, ZeroedPageList). + * + * @param Color + * Supplies the specific color index. + * + * @return This routine returns a pointer to the corresponding MMCOLOR_TABLES structure. + * + * @since XT 1.0 + */ +XTAPI +PMMCOLOR_TABLES +MM::Colors::GetFreePages(IN MMPAGELISTS PageList, + IN ULONG Color) +{ + /* Return a pointer to the requested color table entry */ + return &FreePages[PageList][Color]; +} + +/** + * Retrieves a pointer to the modified pages list for a specific color. + * + * @param Color + * Supplies the specific color index. + * + * @return This routine returns a pointer to the corresponding MMPFNLIST structure. + * + * @since XT 1.0 + */ +XTAPI +PMMPFNLIST +MM::Colors::GetModifiedPages(IN ULONG Color) +{ + return &ModifiedPages[Color]; +} + +/** + * Retrieves the next available color for page coloring. + * + * @return This routine returns the next color value, ensuring it stays within the valid color range. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Colors::GetNextColor(VOID) +{ + /* Increment the color counter and wrap it around using the mask */ + return ((++PagingColors) & PagingColorsMask); +} + +/** + * Retrieves the total number of page colors configured in the system. + * + * @return This routine returns the number of page colors. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Colors::GetPagingColors(VOID) +{ + /* Return the total number of page colors */ + return PagingColors; +} + +/** + * Retrieves the bitmask used for calculating a page's color. + * + * @return This routine returns the page color mask. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Colors::GetPagingColorsMask(VOID) +{ + /* Return the mask used for page coloring calculations */ + return PagingColorsMask; +} + +/** + * Initializes the data structures for page coloring. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Colors::InitializeColorTables(VOID) +{ + PMMMEMORY_LAYOUT MemoryLayout; + PMMPTE PointerPte, LastPte; + ULONG Color; + PMMPTE ValidPte; + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Set the base address of the color tables to start right after the PFN database */ + FreePages[0] = (PMMCOLOR_TABLES)&((PMMPFN)MemoryLayout->PfnDatabase)[MM::Pfn::GetHighestPhysicalPage() + 1]; + + /* Calculate the virtual address range for both color tables */ + PointerPte = MM::Paging::GetPteAddress(&FreePages[0][0]); + LastPte = MM::Paging::GetPteAddress((PVOID)((ULONG_PTR)FreePages[0] + + (2 * PagingColors * sizeof(MMCOLOR_TABLES)) - 1)); + + /* Get a pointer to a PTE template */ + ValidPte = MM::Pte::GetValidPte(); + + /* Ensure the entire virtual address range for the color tables is mapped */ + while(PointerPte <= LastPte) + { + /* Check if a page in the range is not mapped */ + if(!MM::Paging::PteValid(PointerPte)) + { + /* Use the bootstrap allocator to get a physical page */ + MM::Paging::SetPte(ValidPte, MM::Pfn::AllocateBootstrapPages(1), 0); + *PointerPte = *ValidPte; + + /* Zero out the newly mapped page */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPte), MM_PAGE_SIZE); + } + + /* Move to the next PTE in the range */ + PointerPte = MM::Paging::GetNextPte(PointerPte); + } + + /* Set the pointer for the second list */ + FreePages[1] = &FreePages[0][PagingColors]; + + /* Initialize all entries in both color tables */ + for(Color = 0; Color < PagingColors; Color++) + { + /* Initialize the FreePageList entry for the current color */ + FreePages[FreePageList][Color].Flink = MAXULONG_PTR; + FreePages[FreePageList][Color].Blink = (PVOID)MAXULONG_PTR; + FreePages[FreePageList][Color].Count = 0; + + /* Initialize the ZeroedPageList entry for the current color */ + FreePages[ZeroedPageList][Color].Flink = MAXULONG_PTR; + FreePages[ZeroedPageList][Color].Blink = (PVOID)MAXULONG_PTR; + FreePages[ZeroedPageList][Color].Count = 0; + } +} diff --git a/xtoskrnl/mm/data.cc b/xtoskrnl/mm/data.cc index 7869827..91c32e8 100644 --- a/xtoskrnl/mm/data.cc +++ b/xtoskrnl/mm/data.cc @@ -9,6 +9,21 @@ #include +/* Array of non-paged pool free list heads */ +LIST_ENTRY MM::Allocator::NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS]; + +/* Array of free page lists segregated by cache color */ +PMMCOLOR_TABLES MM::Colors::FreePages[FreePageList + 1]; + +/* Array of modified pages segregated by cache color */ +MMPFNLIST MM::Colors::ModifiedPages[MM_PAGING_COLORS] = {{0, ModifiedPageList, MAXULONG_PTR, MAXULONG_PTR}}; + +/* Number of supported page colors */ +ULONG MM::Colors::PagingColors; + +/* Bitmask used to calculate the cache color index */ +ULONG MM::Colors::PagingColorsMask; + /* Allocation descriptors dedicated for hardware layer */ LOADER_MEMORY_DESCRIPTOR MM::HardwarePool::HardwareAllocationDescriptors[MM_HARDWARE_ALLOCATION_DESCRIPTORS]; @@ -18,23 +33,81 @@ PVOID MM::HardwarePool::HardwareHeapStart = MM_HARDWARE_HEAP_START_ADDRESS; /* Number of used hardware allocation descriptors */ ULONG MM::HardwarePool::UsedHardwareAllocationDescriptors = 0; -/* Biggest free memory descriptor */ -PLOADER_MEMORY_DESCRIPTOR MM::Init::FreeDescriptor; - -/* Highest physical page number */ -ULONG_PTR MM::Init::HighestPhysicalPage; - -/* Lowest physical page number */ -ULONG_PTR MM::Init::LowestPhysicalPage = -1; - -/* Number of physical pages */ -ULONG MM::Init::NumberOfPhysicalPages; - -/* Old biggest free memory descriptor */ -LOADER_MEMORY_DESCRIPTOR MM::Init::OldFreeDescriptor; - /* Processor structures data (THIS IS A TEMPORARY HACK) */ UCHAR MM::KernelPool::ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE] = {{0}}; +/* Global structure describing the virtual memory layout of the system */ +MMMEMORY_LAYOUT MM::Manager::MemoryLayout; + +/* Total number of PTEs reserved for system space mapping */ +PFN_NUMBER MM::Manager::NumberOfSystemPtes; + /* Instance of the page map routines for the current PML level */ MM::PPAGEMAP MM::Paging::PmlRoutines; + +/* Total number of physical pages available for allocation */ +PFN_NUMBER MM::Pfn::AvailablePages; + +/* Head of the list containing physical pages marked as defective */ +MMPFNLIST MM::Pfn::BadPagesList = {0, BadPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* Biggest free memory descriptor */ +PLOADER_MEMORY_DESCRIPTOR MM::Pfn::FreeDescriptor; + +/* List containing free physical pages */ +MMPFNLIST MM::Pfn::FreePagesList = {0, FreePageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* Highest physical page number */ +ULONG_PTR MM::Pfn::HighestPhysicalPage; + +/* Lowest physical page number */ +ULONG_PTR MM::Pfn::LowestPhysicalPage = -1; + +/* List containing modified pages */ +MMPFNLIST MM::Pfn::ModifiedPagesList = {0, ModifiedPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* List containing modified pages mapped as read-only */ +MMPFNLIST MM::Pfn::ModifiedReadOnlyPagesList = {0, ModifiedReadOnlyPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* Number of physical pages */ +ULONGLONG MM::Pfn::NumberOfPhysicalPages; + +/* Old biggest free memory descriptor */ +LOADER_MEMORY_DESCRIPTOR MM::Pfn::OriginalFreeDescriptor; + +/* Array of pointers to PFN lists */ +PMMPFNLIST MM::Pfn::PageLocationList[] = {&ZeroedPagesList, + &FreePagesList, + &StandbyPagesList, + &ModifiedPagesList, + &ModifiedReadOnlyPagesList, + &BadPagesList, + NULLPTR, + NULLPTR}; + +/* List containing pages mapped as Read-Only (ROM) */ +MMPFNLIST MM::Pfn::RomPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* List containing standby pages (clean, can be reclaimed or repurposed) */ +MMPFNLIST MM::Pfn::StandbyPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* List containing free physical pages that have been zeroed out */ +MMPFNLIST MM::Pfn::ZeroedPagesList = {0, ZeroedPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* Array of lists for available System PTEs, separated by pool type */ +MMPTE MM::Pte::FirstSystemFreePte[MaximumPtePoolTypes]; + +/* Virtual base address of the System PTE space */ +PMMPTE MM::Pte::SystemPteBase; + +/* End addresses for the System PTE ranges */ +PMMPTE MM::Pte::SystemPtesEnd[MaximumPtePoolTypes]; + +/* Start addresses for the System PTE ranges */ +PMMPTE MM::Pte::SystemPtesStart[MaximumPtePoolTypes]; + +/* Total count of available System PTEs */ +PFN_COUNT MM::Pte::TotalSystemFreePtes[MaximumPtePoolTypes]; + +/* Template PTE entry containing standard flags for a valid, present kernel page */ +MMPTE MM::Pte::ValidPte; diff --git a/xtoskrnl/mm/hlpool.cc b/xtoskrnl/mm/hlpool.cc index 8728b8f..2504f8b 100644 --- a/xtoskrnl/mm/hlpool.cc +++ b/xtoskrnl/mm/hlpool.cc @@ -170,7 +170,7 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress, { PVOID BaseAddress, ReturnAddress; PFN_NUMBER MappedPages; - PHARDWARE_PTE PtePointer; + PMMPTE PtePointer; /* Initialize variables */ BaseAddress = HardwareHeapStart; @@ -189,7 +189,7 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress, } /* Get PTE pointer and advance to next page */ - PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(ReturnAddress); + PtePointer = MM::Paging::GetPteAddress(ReturnAddress); ReturnAddress = (PVOID)((ULONG_PTR)ReturnAddress + MM_PAGE_SIZE); /* Check if PTE is valid */ @@ -219,10 +219,10 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress, while(MappedPages--) { /* Get PTE pointer */ - PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(BaseAddress); + PtePointer = MM::Paging::GetPteAddress(BaseAddress); /* Fill the PTE */ - MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), TRUE); + MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), MM_PTE_READWRITE); /* Advance to the next address */ PhysicalAddress.QuadPart += MM_PAGE_SIZE; @@ -259,18 +259,18 @@ VOID MM::HardwarePool::MarkHardwareMemoryWriteThrough(IN PVOID VirtualAddress, IN PFN_NUMBER PageCount) { - PHARDWARE_PTE PtePointer; + PMMPTE PtePointer; PFN_NUMBER Page; /* Get PTE address from virtual address */ - PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress); + PtePointer = MM::Paging::GetPteAddress(VirtualAddress); /* Iterate through mapped pages */ for(Page = 0; Page < PageCount; Page++) { /* Mark pages as CD/WT */ MM::Paging::SetPteCaching(PtePointer, TRUE, TRUE); - PtePointer++; + MM::Paging::GetNextEntry(PtePointer); } } @@ -296,13 +296,13 @@ MM::HardwarePool::RemapHardwareMemory(IN PVOID VirtualAddress, IN PHYSICAL_ADDRESS PhysicalAddress, IN BOOLEAN FlushTlb) { - PHARDWARE_PTE PtePointer; + PMMPTE PtePointer; /* Get PTE address from virtual address */ - PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress); + PtePointer = MM::Paging::GetPteAddress(VirtualAddress); /* Remap the PTE */ - MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), TRUE); + MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), MM_PTE_READWRITE); /* Check if TLB needs to be flushed */ if(FlushTlb) @@ -334,7 +334,7 @@ MM::HardwarePool::UnmapHardwareMemory(IN PVOID VirtualAddress, IN PFN_NUMBER PageCount, IN BOOLEAN FlushTlb) { - PHARDWARE_PTE PtePointer; + PMMPTE PtePointer; PFN_NUMBER Page; /* Check if address is valid hardware memory */ @@ -348,7 +348,7 @@ MM::HardwarePool::UnmapHardwareMemory(IN PVOID VirtualAddress, VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress & ~(MM_PAGE_SIZE - 1)); /* Get PTE address from virtual address */ - PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress); + PtePointer = MM::Paging::GetPteAddress(VirtualAddress); /* Iterate through mapped pages */ for(Page = 0; Page < PageCount; Page++) diff --git a/xtoskrnl/mm/i686/alloc.cc b/xtoskrnl/mm/i686/alloc.cc new file mode 100644 index 0000000..79b0abf --- /dev/null +++ b/xtoskrnl/mm/i686/alloc.cc @@ -0,0 +1,31 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/i686/alloc.cc + * DESCRIPTION: Memory manager pool allocation + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Maps the PTE for the base of the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Allocator::MapNonPagedPool(VOID) +{ + PMMMEMORY_LAYOUT MemoryLayout; + + /* Retrieve memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Map PDE and PTE for the base of the non-paged pool */ + MM::Pte::MapPDE(MemoryLayout->NonPagedPoolStart, (PCHAR)MemoryLayout->NonPagedPoolEnd - 1, MM::Pte::GetValidPte()); + MM::Pte::MapPTE(MemoryLayout->NonPagedPoolStart, (PCHAR)MemoryLayout->NonPagedPoolEnd - 1, MM::Pte::GetValidPte()); +} diff --git a/xtoskrnl/mm/i686/init.cc b/xtoskrnl/mm/i686/init.cc deleted file mode 100644 index ab1892d..0000000 --- a/xtoskrnl/mm/i686/init.cc +++ /dev/null @@ -1,25 +0,0 @@ -/** - * PROJECT: ExectOS - * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/mm/i686/init.cc - * DESCRIPTION: Architecture specific Memory Manager initialization routines - * DEVELOPERS: Rafal Kupiec - * Aiken Harris - */ - -#include - - -/** - * Performs architecture specific initialization of the XTOS Memory Manager. - * - * @return This routine does not return any value. - * - * @since XT 1.0 - */ -XTAPI -VOID -MM::Init::InitializeArchitecture(VOID) -{ - UNIMPLEMENTED; -} diff --git a/xtoskrnl/mm/i686/mmgr.cc b/xtoskrnl/mm/i686/mmgr.cc new file mode 100644 index 0000000..1a65d53 --- /dev/null +++ b/xtoskrnl/mm/i686/mmgr.cc @@ -0,0 +1,445 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/i686/mmgr.cc + * DESCRIPTION: Memory Manager + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Calculates the maximum possible size of the non-paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONG_PTR MaximumNonPagedPoolSize; + ULONGLONG PhysicalPages; + + /* Get number of physical pages */ + PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages(); + + /* Start with 1MiB and reserve space for PFN database */ + MaximumNonPagedPoolSize = 1048576; + + /* Check if system has at least 512MiB of physical memory */ + if(PhysicalPages >= 126976) + { + /* Add 200KiB for each MiB above 4MiB */ + MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 204800; + + /* Check if non-paged pool has at least 128MiB */ + if(MaximumNonPagedPoolSize < 134217728) + { + /* Expand non-paged pool size to 128MiB */ + MaximumNonPagedPoolSize = 134217728; + } + } + else + { + /* Add 400KiB for each MiB above 4MiB */ + MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 409600; + } + + /* Check if non-paged pool does not exceed 256MiB */ + if(MaximumNonPagedPoolSize > 268435456) + { + /* Limit non-paged pool size to 256MiB */ + MaximumNonPagedPoolSize = 268435456; + } + + /* Return maximum non-paged pool size in pages */ + *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize); +} + +/** + * Calculates the size of the non-paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the non-paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONG_PTR NonPagedPoolSize; + ULONGLONG PhysicalPages; + + /* Get number of physical pages */ + PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages(); + + /* Verify if system has less than 256MiB of physical memory */ + if(PhysicalPages <= 65536) + { + /* Reduce initial non-paged pool size to 2MiB to save memory */ + NonPagedPoolSize = 2097152; + } + else + { + /* Start with 256KiB and add 32KiB for each MiB above 4MiB */ + NonPagedPoolSize = 262144 + (((PhysicalPages - 1024) / 256) * 32768); + + if(NonPagedPoolSize > 134217728) + { + /* Limit non-paged pool size to 128MiB */ + NonPagedPoolSize = 134217728; + } + } + + /* Return non-paged pool size in pages, aligned down to page size boundary */ + *PoolSize = SIZE_TO_PAGES(ROUND_DOWN(NonPagedPoolSize, MM_PAGE_SIZE)); +} + +/** + * Calculates the size of the paged pool. + * + * @param PoolSize + * A pointer to a variable that will receive the number of pages available for the paged pool. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize) +{ + ULONG_PTR PagedPoolSize, PteCount; + ULONG PtesPerPage; + + /* Start with maximum non-paged pool size */ + ComputeMaximumNonPagedPoolSize(&PagedPoolSize); + PagedPoolSize *= MM_PAGE_SIZE; + + /* Check XPA status */ + if(MM::Paging::GetXpaStatus()) + { + /* Four times the non-paged pool size on PAE-enabled systems */ + PagedPoolSize *= 4; + } + else + { + /* Double the non-paged pool size on PAE-disabled systems */ + PagedPoolSize *= 2; + } + + /* Check if paged pool does not overlap non-paged pool */ + if(PagedPoolSize > (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart) + { + /* Limit paged pool size to maximum possible */ + PagedPoolSize = (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart; + } + + /* Ensure that paged pool size is at least 32MiB */ + if(PagedPoolSize < 33554432) + { + /* Increase paged pool size to at least 32MiB */ + PagedPoolSize = 33554432; + } + + /* Get the number of PTEs per page and calculate size of paged pool */ + PtesPerPage = MM::Pte::GetPtesPerPage(); + PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage); + + /* Return paged pool size */ + *PoolSize = (PFN_NUMBER)(PteCount * PtesPerPage); +} + +/** + * Calculates the size of the session space. + * + * @param SpaceSize + * A pointer to a variable that will receive the number of pages available by the session space. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize) +{ + PFN_NUMBER SessionSpaceSize; + + /* Session Pool, Session View, Session Image, Session Working Set and System View takes 108MiB */ + SessionSpaceSize = 113246208; + + /* Return number of pages used by the session space */ + *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE; +} + +/** + * Calculates the size of the system PTEs. + * + * @param PteSize + * A pointer to a variable that will receive the number of system PTEs. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize) +{ + PFN_NUMBER SystemPteSize; + + /* Check if system has less than 19MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() < 4864) + { + /* Set minimal system PTE size */ + SystemPteSize = 7000; + } + else + { + /* Use standard system PTE size */ + SystemPteSize = 11000; + + /* Check if system has more than 32MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() > 8192) + { + /* Double system PTE size */ + SystemPteSize *= 2; + + /* Check if system has more than 256MiB of physical memory */ + if(MM::Pfn::GetNumberOfPhysicalPages() > 65536) + { + /* Double system PTE size */ + SystemPteSize *= 2; + } + } + } + + /* Return system PTE size */ + *PteSize = SystemPteSize; +} + +/** + * Dumps the kernel's memory layout. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::DumpMemoryLayout(VOID) +{ + /* Dump memory layout */ + DebugPrint(L"System with %zu MiB of installed memory:\n" + L"User Space: %.8P - %.8P\n" + L"Loader Mappings: %.8P - %.8P\n" + L"PFN Database: %.8P - %.8P\n" + L"Non-Paged Pool: %.8P - %.8P\n" + L"Session Space: %.8P - %.8P\n" + L"PTE Space: %.8P - %.8P\n" + L"Hyper Space: %.8P - %.8P\n" + L"System Working Set: %.8P - %.8P\n" + L"System Cache: %.8P - %.8P\n" + L"Paged Pool: %.8P - %.8P\n" + L"Non-Paged System Pool: %.8P - %.8P\n" + L"Non-Paged Expansion Pool: %.8P - %.8P\n" + L"Shared System Page: %.8P - %.8P\n" + L"Hardware Pool: %.8P - %.8P\n", + GetInstalledMemorySize(), + MemoryLayout.UserSpaceStart, + MemoryLayout.UserSpaceEnd, + MemoryLayout.LoaderMappingsStart, + MemoryLayout.LoaderMappingsEnd, + MemoryLayout.PfnDatabase, + (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE), + MemoryLayout.NonPagedPoolStart, + MemoryLayout.NonPagedPoolEnd, + MemoryLayout.SessionSpaceStart, + MemoryLayout.SessionSpaceEnd, + MemoryLayout.PteSpaceStart, + MemoryLayout.PteSpaceEnd, + MemoryLayout.HyperSpaceStart, + MemoryLayout.HyperSpaceEnd, + MemoryLayout.SystemWorkingSetStart, + MemoryLayout.SystemWorkingSetEnd, + MemoryLayout.SystemCacheStart, + MemoryLayout.SystemCacheEnd, + MemoryLayout.PagedPoolStart, + MemoryLayout.PagedPoolEnd, + MemoryLayout.NonPagedSystemPoolStart, + MemoryLayout.NonPagedSystemPoolEnd, + MemoryLayout.NonPagedExpansionPoolStart, + MemoryLayout.NonPagedExpansionPoolEnd, + MemoryLayout.SharedSystemPageStart, + MemoryLayout.SharedSystemPageEnd, + MemoryLayout.HardwarePoolStart, + MemoryLayout.HardwarePoolEnd); +} + +/** + * Initializes the kernel's virtual memory layout. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::InitializeMemoryLayout(VOID) +{ + PFN_NUMBER MaximumNonPagedPoolSize; + ULONG_PTR PfnDatabaseEnd; + + /* Check if 3-level paging (PAE) is enabled */ + if(MM::Paging::GetXpaStatus()) + { + /* Set PML3 base address */ + MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE; + + /* Define memory layout for 3-level paging */ + MemoryLayout.NonCanonicalStart = (PVOID)0x00000000; + MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000; + MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000; + MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000; + MemoryLayout.UserSpaceStart = (PVOID)0x00010000; + MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF; + MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000; + MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000; + MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000; + MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000; + MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000; + MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000; + MemoryLayout.PteSpaceStart = (PVOID)0xC0000000; + MemoryLayout.PteSpaceEnd = (PVOID)0xC07FFFFF; + MemoryLayout.HyperSpaceStart = (PVOID)0xC0800000; + MemoryLayout.HyperSpaceEnd = (PVOID)0xC0BFFFFF; + MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000; + MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF; + MemoryLayout.SystemCacheStart = (PVOID)0xC1000000; + MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF; + MemoryLayout.PagedPoolStart = (PVOID)0xE1000000; + MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000; + MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000; + MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000; + MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000; + MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000; + MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000; + MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000; + MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000; + MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF; + } + else + { + /* Set PML2 base address */ + MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE; + + /* Define memory layout for 2-level paging */ + MemoryLayout.NonCanonicalStart = (PVOID)0x00000000; + MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000; + MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000; + MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000; + MemoryLayout.UserSpaceStart = (PVOID)0x00010000; + MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF; + MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000; + MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000; + MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000; + MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000; + MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000; + MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000; + MemoryLayout.PteSpaceStart = (PVOID)0xC0000000; + MemoryLayout.PteSpaceEnd = (PVOID)0xC03FFFFF; + MemoryLayout.HyperSpaceStart = (PVOID)0xC0400000; + MemoryLayout.HyperSpaceEnd = (PVOID)0xC07FFFFF; + MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000; + MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF; + MemoryLayout.SystemCacheStart = (PVOID)0xC1000000; + MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF; + MemoryLayout.PagedPoolStart = (PVOID)0xE1000000; + MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000; + MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000; + MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000; + MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000; + MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000; + MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000; + MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000; + MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000; + MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF; + } + + /* Compute allocation size for the PFN database */ + MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize); + + /* Compute boot image size */ + ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize); + + /* Compute session space size */ + ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize); + + /* Update loader mappings space end address */ + MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONG_PTR)MemoryLayout.LoaderMappingsStart + + MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE); + + /* Update session space start address */ + MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd - + MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE); + + /* Compute system PTE size */ + ComputeSystemPteSize(&NumberOfSystemPtes); + + /* Compute the initial and maximum non-paged pool sizes */ + ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize); + ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize); + + /* Compute paged pool size */ + ComputePagedPoolSize(&MemoryLayout.PagedPoolSize); + + /* Position the PFN database right after the loader mappings */ + MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.LoaderMappingsEnd; + + /* Compute the PFN database end address */ + PfnDatabaseEnd = (ULONG_PTR)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE); + + /* Position the initial non-paged pool immediately after the PFN database */ + MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd; + + /* Check if the calculated non-paged pool size fits in the KVA */ + if((MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE) > + ((ULONG_PTR)MemoryLayout.SessionSpaceStart - (ULONG_PTR)MemoryLayout.NonPagedPoolStart)) + { + /* Set the final size for the non-paged pool */ + MemoryLayout.NonPagedPoolSize = ((ULONG_PTR)MemoryLayout.NonPagedPoolEnd - + (ULONG_PTR)MemoryLayout.NonPagedPoolStart) / MM_PAGE_SIZE; + } + + /* Set the final non-paged pool end address */ + MemoryLayout.NonPagedPoolEnd = (PVOID)((ULONG_PTR)MemoryLayout.NonPagedPoolStart + + MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE); + + /* Check if non-paged expansion pool overflows */ + if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize * + MM_PAGE_SIZE >= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart) + { + /* Check if non-paged expansion pool fits */ + if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize * + MM_PAGE_SIZE <= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd) + { + /* Set new non-paged expansion pool end address */ + MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + + MaximumNonPagedPoolSize * MM_PAGE_SIZE); + } + } + + /* Compute non-paged expansion pool size */ + MemoryLayout.NonPagedExpansionPoolSize = ((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd - + (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE; +} diff --git a/xtoskrnl/mm/i686/pagemap.cc b/xtoskrnl/mm/i686/pagemap.cc index 3793387..d4db9b7 100644 --- a/xtoskrnl/mm/i686/pagemap.cc +++ b/xtoskrnl/mm/i686/pagemap.cc @@ -10,20 +10,17 @@ /** - * Clears the contents of a page table entry (PTE). + * Gets Page Map Level (PML) for current paging mode. * - * @param PtePointer - * Pointer to the page table entry (PTE) to be cleared. - * - * @return This routine does not return any value. + * @return This routine returns the page map level. * * @since XT 1.0 */ XTAPI -VOID -MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer) +USHORT +MM::PageMap::GetPageMapLevel() { - PtePointer->Long = 0; + return PageMapInfo.Xpa ? 3 : 2; } /** @@ -38,7 +35,7 @@ MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer) */ XTAPI PMMPDE -MM::PageMap::GetPdeAddress(PVOID Address) +MM::PageMap::GetPdeAddress(IN PVOID Address) { ULONG Offset; @@ -47,6 +44,25 @@ MM::PageMap::GetPdeAddress(PVOID Address) return (PMMPDE)(PageMapInfo.PdeBase + Offset); } +/** + * Gets the Offset of the PDE (Page Directory Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PDE. + * + * @return This routine returns the Offset of the PDE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPdeOffset(IN PVOID Address) +{ + /* Return PDE Offset */ + return ((((ULONG_PTR)(Address)) >> PageMapInfo.PdiShift) & (PageMapInfo.Xpa ? 0x1FF : 0x3FF)); +} + + /** * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address. * @@ -59,12 +75,48 @@ MM::PageMap::GetPdeAddress(PVOID Address) */ XTAPI PMMPPE -MM::PageMap::GetPpeAddress(PVOID Address) +MM::PageMap::GetPpeAddress(IN PVOID Address) { /* Return zero */ return (PMMPPE)0; } +/** + * Gets the offset of the PPE (Page Directory Pointer Table Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PPE. + * + * @return This routine returns the offset of the PPE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPpeOffset(IN PVOID Address) +{ + /* Return zero */ + return 0; +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry. + * + * @param PpePointer + * Specifies the virtual address of the PPE. + * + * @return This routine returns the virtual address mapped by the PPE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMap::GetPpeVirtualAddress(IN PMMPPE PpePointer) +{ + /* Return zero */ + return (PVOID)0; +} + /** * Gets the address of the PTE (Page Table Entry), that maps given address. * @@ -77,7 +129,7 @@ MM::PageMap::GetPpeAddress(PVOID Address) */ XTAPI PMMPTE -MM::PageMap::GetPteAddress(PVOID Address) +MM::PageMap::GetPteAddress(IN PVOID Address) { ULONG Offset; @@ -86,6 +138,294 @@ MM::PageMap::GetPteAddress(PVOID Address) return (PMMPTE)(PageMapInfo.PteBase + Offset); } +/** + * Gets the Offset of the PTE (Page Table Entry), that maps given address. + * + * @param Address + * Specifies the virtual address for which to retrieve the corresponding PTE. + * + * @return This routine returns the Offset of the PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMap::GetPteOffset(IN PVOID Address) +{ + /* Return PTE Offset */ + return ((((ULONG_PTR)(Address)) >> MM_PTI_SHIFT) & (PageMapInfo.Xpa ? 0x1FF : 0x3FF)); +} + + +/** + * Gets the status of Extended Paging Address (XPA) mode. + * + * @return This routine returns TRUE if XPA is enabled, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::PageMap::GetXpaStatus() +{ + return PageMapInfo.Xpa; +} + +/** + * Advances a PTE pointer by a given number of entries, considering the actual PTE size for PML2. + * + * @param Pte + * The PTE pointer to advance. + * + * @param Count + * The number of PTE entries to advance by. + * + * @return This routine returns the advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMapBasic::AdvancePte(IN PMMPTE Pte, + IN ULONG Count) +{ + /* Return advanced PTE pointer */ + return (PMMPTE)((ULONG_PTR)Pte + (Count * sizeof(MMPML2_PTE))); +} + +/** + * Clears the contents of a page table entry (PTE). + * + * @param PtePointer + * Pointer to the page table entry (PTE) to be cleared. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::ClearPte(IN PMMPTE PtePointer) +{ + /* Clear PTE */ + PtePointer->Pml2.Long = 0; +} + +/** + * Gets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to get the next entry from. + * + * @return This routine returns the next entry in the PTE list. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMapBasic::GetNextEntry(IN PMMPTE Pte) +{ + /* Return next entry in PTE list, translating the hardware limit (0xFFFFF) to the logical sentinel (MAXULONG) */ + return (Pte->Pml2.List.NextEntry == 0xFFFFF) ? MAXULONG : Pte->Pml2.List.NextEntry; +} + +/** + * Advances a PTE pointer, considering the actual PTE size for PML2. + * + * @param Pte + * The PTE pointer to advance. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMapBasic::GetNextPte(IN PMMPTE Pte) +{ + /* Return advanced PTE pointer */ + return AdvancePte(Pte, 1); +} + +/** + * Checks if a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to check. + * + * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::PageMapBasic::GetOneEntry(IN PMMPTE Pte) +{ + /* Return one entry status */ + return Pte->Pml2.List.OneEntry; +} + +/** + * Gets the page frame number from a corresponding PTE. + * + * @param Pte + * The PTE pointer to get the page frame number from. + * + * @return This routine returns the page frame number. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::PageMapBasic::GetPageFrameNumber(IN PMMPTE Pte) +{ + return Pte->Pml2.Hardware.PageFrameNumber; +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Entry. + * + * @param PdePointer + * Specifies the address of the PDE. + * + * @return This routine returns the virtual address mapped by the PDE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPdeVirtualAddress(IN PMMPDE PdePointer) +{ + /* Return PDE virtual address */ + return ((PVOID)((ULONG)(PdePointer) << 20)); +} + + /** + * Gets the entire contents of a PML2 Page Table Entry (PTE) as a single value. + * + * @param PtePointer + * Pointer to the Page Table Entry (PTE) to read. + * + * @return This routine returns the contents of the PTE as a single value. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMapBasic::GetPte(IN PMMPTE PtePointer) +{ + /* Return PTE value */ + return PtePointer->Pml2.Long; +} +/** + * Calculates the distance between two PTE pointers. + * + * @param EndPte + * Pointer to the ending Page Table Entry. + * + * @param StartPte + * Pointer to the starting Page Table Entry. + * + * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte. + * + * @since XT 1.0 + */ +XTAPI +LONG +MM::PageMapBasic::GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte) +{ + /* Return distance between PTE pointers */ + return ((ULONG_PTR)EndPte - (ULONG_PTR)StartPte) / sizeof(MMPML2_PTE); +} + +/** + * Gets the size of a PTE for basic paging (PML2). + * + * @return This routine returns the size of a PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapBasic::GetPteSize(VOID) +{ + /* Return the size of MMPTE */ + return sizeof(MMPML2_PTE); +} + +/** + * Gets the software protection value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software protection value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapBasic::GetPteSoftwareProtection(IN PMMPTE PtePointer) +{ + /* Return PTE software protection value */ + return (ULONG)PtePointer->Pml2.Software.Protection; +} + +/** + * Gets the software prototype value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software prototype value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapBasic::GetPteSoftwarePrototype(IN PMMPTE PtePointer) +{ + /* Return PTE software prototype value */ + return (ULONG)PtePointer->Pml2.Software.Prototype; +} + +/** + * Gets the software transition value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software transition value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapBasic::GetPteSoftwareTransition(IN PMMPTE PtePointer) +{ + /* Return PTE software transition value */ + return (ULONG)PtePointer->Pml2.Software.Transition; +} + +/** + * Gets the virtual address that is mapped by a given Page Table Entry. + * + * @param PtePointer + * Specifies the virtual address of the PTE. + * + * @return This routine returns the virtual address mapped by the PTE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapBasic::GetPteVirtualAddress(IN PMMPTE PtePointer) +{ + /* Return PTE virtual address */ + return ((PVOID)((ULONG)(PtePointer) << 10)); +} + /** * Initializes page map information for basic paging (PML2). * @@ -121,9 +461,54 @@ MM::PageMapBasic::InitializePageMapInfo(VOID) */ XTAPI BOOLEAN -MM::PageMapBasic::PteValid(PHARDWARE_PTE PtePointer) +MM::PageMapBasic::PteValid(IN PMMPTE PtePointer) { - return (BOOLEAN)PtePointer->Pml2.Valid; + /* Check if PTE is valid */ + return (BOOLEAN)PtePointer->Pml2.Hardware.Valid; +} + +/** + * Sets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set as the next entry. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value) +{ + /* Set next entry in PTE list, translating the logical sentinel (MAXULONG) to the 20-bit hardware limit (0xFFFFF) */ + Pte->Pml2.List.NextEntry = (Value == MAXULONG) ? 0xFFFFF : Value; +} + +/** + * Sets the flag indicating whether a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set. TRUE if the list has only one entry, FALSE otherwise. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value) +{ + /* Set one entry status */ + Pte->Pml2.List.OneEntry = Value; } /** @@ -135,8 +520,8 @@ MM::PageMapBasic::PteValid(PHARDWARE_PTE PtePointer) * @param PageFrameNumber * Physical frame number to map. * - * @param Writable - * Indicates whether the page should be writable. + * @param AttributesMask + * Specifies the attributes mask to apply to the PTE. * * @return This routine does not return any value. * @@ -144,13 +529,35 @@ MM::PageMapBasic::PteValid(PHARDWARE_PTE PtePointer) */ XTAPI VOID -MM::PageMapBasic::SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable) +MM::PageMapBasic::SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask) { - PtePointer->Pml2.PageFrameNumber = PageFrameNumber; - PtePointer->Pml2.Valid = 1; - PtePointer->Pml2.Writable = Writable; + /* Set PTE */ + PtePointer->Pml2.Hardware.PageFrameNumber = PageFrameNumber; + PtePointer->Pml2.Hardware.Valid = 1; + PtePointer->Pml2.Long |= AttributesMask; +} + +/** + * Sets a PML2 page table entry (PTE) with the specified attributes. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param Attributes + * Specifies the attributes to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes) +{ + PtePointer->Pml2.Long = Attributes; } /** @@ -171,12 +578,322 @@ MM::PageMapBasic::SetPte(PHARDWARE_PTE PtePointer, */ XTAPI VOID -MM::PageMapBasic::SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough) +MM::PageMapBasic::SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough) { - PtePointer->Pml2.CacheDisable = CacheDisable; - PtePointer->Pml2.WriteThrough = WriteThrough; + /* Set caching attributes */ + PtePointer->Pml2.Hardware.CacheDisable = CacheDisable; + PtePointer->Pml2.Hardware.WriteThrough = WriteThrough; +} + +/** + * Transitions a Page Table Entry (PTE) to invalid state + * + * @param PointerPte + * Pointer to the page table entry (PTE) to transition. + * + * @param Protection + * Specifies the protection attribute to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection) +{ + MMPTE TempPte; + + /* Set transition PTE */ + TempPte = *PointerPte; + TempPte.Pml2.Software.Protection = Protection; + TempPte.Pml2.Software.Prototype = 0; + TempPte.Pml2.Software.Transition = 1; + TempPte.Pml2.Software.Valid = 0; + + /* Write PTE value */ + *PointerPte = TempPte; +} + +/** + * Writes a PML2 page table entry (PTE) with the specified value. + * + * @param Pte + * Pointer to the page table entry (PTE) to write. + * + * @param Value + * The value to write to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapBasic::WritePte(IN PMMPTE Pte, + IN MMPTE Value) +{ + /* Write PTE value */ + Pte->Pml2.Long = Value.Pml2.Long; +} + +/** + * Advances a PTE pointer by a given number of entries, considering the actual PTE size for PML3. + * + * @param Pte + * The PTE pointer to advance. + * + * @param Count + * The number of PTE entries to advance by. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMapXpa::AdvancePte(IN PMMPTE Pte, + IN ULONG Count) +{ + /* Return advanced PTE pointer */ + return (PMMPTE)((ULONG_PTR)Pte + (Count * sizeof(MMPML3_PTE))); +} + +/** + * Clears the contents of a page table entry (PTE). + * + * @param PtePointer + * Pointer to the page table entry (PTE) to be cleared. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::ClearPte(IN PMMPTE PtePointer) +{ + /* Clear PTE */ + PtePointer->Pml3.Long = 0; +} + +/** + * Gets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to get the next entry from. + * + * @return This routine returns the next entry in the PTE list. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMapXpa::GetNextEntry(IN PMMPTE Pte) +{ + /* Return next entry in PTE list */ + return Pte->Pml3.List.NextEntry; +} + +/** + * Advances a PTE pointer, considering the actual PTE size for PML3. + * + * @param Pte + * The PTE pointer to advance. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::PageMapXpa::GetNextPte(IN PMMPTE Pte) +{ + /* Return advanced PTE pointer */ + return AdvancePte(Pte, 1); +} + +/** + * Checks if a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to check. + * + * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::PageMapXpa::GetOneEntry(IN PMMPTE Pte) +{ + /* Return one entry status */ + return Pte->Pml3.List.OneEntry; +} + +/** + * Gets the page frame number from a corresponding PTE. + * + * @param Pte + * The PTE pointer to get the page frame number from. + * + * @return This routine returns the page frame number. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::PageMapXpa::GetPageFrameNumber(IN PMMPTE Pte) +{ + return Pte->Pml3.Hardware.PageFrameNumber; +} + +/** + * Gets the virtual address that is mapped by a given Page Directory Entry. + * + * @param PdePointer + * Specifies the address of the PDE. + * + * @return This routine returns the virtual address mapped by the PDE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPdeVirtualAddress(IN PMMPDE PdePointer) +{ + /* Return PDE virtual address */ + return ((PVOID)((ULONG)(PdePointer) << 18)); +} + + /** + * Gets the entire contents of a PML3 Page Table Entry (PTE) as a single value. + * + * @param PtePointer + * Pointer to the Page Table Entry (PTE) to read. + * + * @return This routine returns the contents of the PTE as a single value. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::PageMapXpa::GetPte(IN PMMPTE PtePointer) +{ + /* Return PTE value */ + return PtePointer->Pml3.Long; +} + +/** + * Calculates the distance between two PTE pointers. + * + * @param EndPte + * Pointer to the ending Page Table Entry. + * + * @param StartPte + * Pointer to the starting Page Table Entry. + * + * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte. + * + * @since XT 1.0 + */ +XTAPI +LONG +MM::PageMapXpa::GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte) +{ + /* Return distance between PTE pointers */ + return ((ULONG_PTR)EndPte - (ULONG_PTR)StartPte) / sizeof(MMPML3_PTE); +} + +/** + * Gets the size of a PTE for XPA paging (PML3). + * + * @return This routine returns the size of a PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapXpa::GetPteSize(VOID) +{ + /* Return the size of MMPTE */ + return sizeof(MMPML3_PTE); +} + +/** + * Gets the software protection value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software protection value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapXpa::GetPteSoftwareProtection(IN PMMPTE PtePointer) +{ + /* Return PTE software protection value */ + return (ULONG)PtePointer->Pml3.Software.Protection; +} + +/** + * Gets the software prototype value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software prototype value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapXpa::GetPteSoftwarePrototype(IN PMMPTE PtePointer) +{ + /* Return PTE software prototype value */ + return (ULONG)PtePointer->Pml3.Software.Prototype; +} + +/** + * Gets the software transition value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software transition value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::PageMapXpa::GetPteSoftwareTransition(IN PMMPTE PtePointer) +{ + /* Return PTE software transition value */ + return (ULONG)PtePointer->Pml3.Software.Transition; +} + +/** + * Gets the virtual address that is mapped by a given Page Table Entry. + * + * @param PtePointer + * Specifies the virtual address of the PTE. + * + * @return This routine returns the virtual address mapped by the PTE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::PageMapXpa::GetPteVirtualAddress(IN PMMPTE PtePointer) +{ + /* Return PTE virtual address */ + return ((PVOID)((ULONG)(PtePointer) << 9)); } /** @@ -190,14 +907,14 @@ XTAPI VOID MM::PageMapXpa::InitializePageMapInfo(VOID) { - /* Set PML2 page map information */ + /* Set PML3 page map information */ PageMapInfo.Xpa = TRUE; - /* Set PML2 base addresses */ + /* Set PML3 base addresses */ PageMapInfo.PteBase = MM_PTE_BASE; PageMapInfo.PdeBase = MM_PDE_BASE; - /* Set PML2 shift values */ + /* Set PML3 shift values */ PageMapInfo.PdiShift = MM_PDI_SHIFT; PageMapInfo.PteShift = MM_PTE_SHIFT; } @@ -214,9 +931,53 @@ MM::PageMapXpa::InitializePageMapInfo(VOID) */ XTAPI BOOLEAN -MM::PageMapXpa::PteValid(PHARDWARE_PTE PtePointer) +MM::PageMapXpa::PteValid(IN PMMPTE PtePointer) { - return (BOOLEAN)PtePointer->Pml3.Valid; + return (BOOLEAN)PtePointer->Pml3.Hardware.Valid; +} + +/** + * Sets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set as the next entry. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value) +{ + /* Set next entry in PTE list */ + Pte->Pml3.List.NextEntry = Value; +} + +/** + * Sets the flag indicating whether a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set. TRUE if the list has only one entry, FALSE otherwise. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value) +{ + /* Set one entry status */ + Pte->Pml3.List.OneEntry = Value; } /** @@ -228,8 +989,8 @@ MM::PageMapXpa::PteValid(PHARDWARE_PTE PtePointer) * @param PageFrameNumber * Physical frame number to map. * - * @param Writable - * Indicates whether the page should be writable. + * @param AttributesMask + * Specifies the attributes mask to apply to the PTE. * * @return This routine does not return any value. * @@ -237,13 +998,35 @@ MM::PageMapXpa::PteValid(PHARDWARE_PTE PtePointer) */ XTAPI VOID -MM::PageMapXpa::SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable) +MM::PageMapXpa::SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask) { - PtePointer->Pml3.PageFrameNumber = PageFrameNumber; - PtePointer->Pml3.Valid = 1; - PtePointer->Pml3.Writable = Writable; + /* Set PTE */ + PtePointer->Pml3.Hardware.PageFrameNumber = PageFrameNumber; + PtePointer->Pml3.Hardware.Valid = 1; + PtePointer->Pml3.Long |= AttributesMask; +} + +/** + * Sets a PML3 page table entry (PTE) with the specified attributes. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param Attributes + * Specifies the attributes to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes) +{ + PtePointer->Pml3.Long = Attributes; } /** @@ -264,10 +1047,64 @@ MM::PageMapXpa::SetPte(PHARDWARE_PTE PtePointer, */ XTAPI VOID -MM::PageMapXpa::SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough) +MM::PageMapXpa::SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough) { - PtePointer->Pml3.CacheDisable = CacheDisable; - PtePointer->Pml3.WriteThrough = WriteThrough; + /* Set caching attributes */ + PtePointer->Pml3.Hardware.CacheDisable = CacheDisable; + PtePointer->Pml3.Hardware.WriteThrough = WriteThrough; } + +/** + * Transitions a Page Table Entry (PTE) to invalid state + * + * @param PointerPte + * Pointer to the page table entry (PTE) to transition. + * + * @param Protection + * Specifies the protection attribute to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection) +{ + MMPTE TempPte; + + /* Set transition PTE */ + TempPte = *PointerPte; + TempPte.Pml3.Software.Protection = Protection; + TempPte.Pml3.Software.Prototype = 0; + TempPte.Pml3.Software.Transition = 1; + TempPte.Pml3.Software.Valid = 0; + + /* Write PTE value */ + *PointerPte = TempPte; +} + +/** + * Writes a PML3 page table entry (PTE) with the specified value. + * + * @param Pte + * Pointer to the page table entry (PTE) to write. + * + * @param Value + * The value to write to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::PageMapXpa::WritePte(IN PMMPTE Pte, + IN MMPTE Value) +{ + /* Write PTE value */ + Pte->Pml3.Long = Value.Pml3.Long; +} \ No newline at end of file diff --git a/xtoskrnl/mm/i686/pfault.cc b/xtoskrnl/mm/i686/pfault.cc new file mode 100644 index 0000000..3b6a8e6 --- /dev/null +++ b/xtoskrnl/mm/i686/pfault.cc @@ -0,0 +1,36 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/i686/pfault.cc + * DESCRIPTION: Page fault support for i686 architecture + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Evaluates the PDE for for paged pool and per-session mappings. + * + * @param VirtualAddress + * Specifies the virtual address to verify. + * + * @return This routine returns ACCESS_VIOLATION on PML3 or status code on PML2. + */ +XTFASTCALL +XTSTATUS +MM::PageFault::CheckPdeForPagedPool(IN PVOID VirtualAddress) +{ + /* Check if XPA is enabled */ + if(MM::Paging::GetXpaStatus()) + { + /* Access violation for PML3 */ + return STATUS_ACCESS_VIOLATION; + } + + /* Unimplemented path for PML2 */ + UNIMPLEMENTED; + + /* Temporarily, just return access violation */ + return STATUS_ACCESS_VIOLATION; +} diff --git a/xtoskrnl/mm/i686/pfn.cc b/xtoskrnl/mm/i686/pfn.cc new file mode 100644 index 0000000..08815d6 --- /dev/null +++ b/xtoskrnl/mm/i686/pfn.cc @@ -0,0 +1,264 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/i686/pfn.cc + * DESCRIPTION: Physical Frame Number for i686 support + * DEVELOPERS: Aiken Harris + * Rafal Kupiec + */ + +#include + + +/** + * Allocates and initializes page directory structures for a range of PDEs. + * + * @param StartingPde + * Supplies a pointer to the first PDE in the range to initialize. + * + * @param EndingPde + * Supplies a pointer to the last PDE in the range to initialize. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePageDirectory(IN PMMPDE StartingPde, + IN PMMPDE EndingPde) +{ + /* Nothing to do */ +} + +/** + * Initializes the PFN database by mapping virtual memory and populating entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePfnDatabase(VOID) +{ + PLOADER_MEMORY_DESCRIPTOR Descriptor; + PKERNEL_INITIALIZATION_BLOCK InitializationBlock; + PLIST_ENTRY ListEntry; + PMMMEMORY_LAYOUT MemoryLayout; + PUCHAR PfnDatabaseEnd; + PMMPTE ValidPte; + + /* Raise runlevel and acquire PFN lock */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Get the kernel initialization block */ + InitializationBlock = KE::BootInformation::GetInitializationBlock(); + + /* Get memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Get the PFN database size and calculate the end of the PFN database virtual address space */ + PfnDatabaseEnd = (PUCHAR)MemoryLayout->PfnDatabase + (MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE) - 1; + + /* Get a template PTE for mapping the PFN database pages */ + ValidPte = MM::Pte::GetValidPte(); + + /* Map the Page Directory and Page Directory Pointer tables for the PFN database */ + MM::Pte::MapPDE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte); + MM::Pte::MapPTE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte); + + /* Zero PFN database virtual space */ + RTL::Memory::ZeroMemory(MemoryLayout->PfnDatabase, MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE); + + /* Initialize the color tables */ + MM::Colors::InitializeColorTables(); + + /* Iterate over memory descriptors to map the PFN database and initialize entries */ + ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink; + while(ListEntry != &InitializationBlock->MemoryDescriptorListHead) + { + /* Get the descriptor */ + Descriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_DESCRIPTOR, ListEntry); + + /* Skip invisible memory regions */ + if(MM::Manager::VerifyMemoryTypeInvisible(Descriptor->MemoryType)) + { + /* Move to the next descriptor and continue */ + ListEntry = ListEntry->Flink; + continue; + } + + /* Check if this is the modified free descriptor */ + if(Descriptor == FreeDescriptor) + { + /* Switch to the original descriptor */ + Descriptor = &OriginalFreeDescriptor; + } + + /* Check if the free memory block that was split is being processed */ + if(Descriptor == &OriginalFreeDescriptor) + { + /* Skip loop processing, free memory is initialized separately */ + ListEntry = ListEntry->Flink; + continue; + } + + /* Initialize PFNs for this memory range */ + ProcessMemoryDescriptor(Descriptor->BasePage, Descriptor->PageCount, Descriptor->MemoryType); + + /* Move to the next descriptor */ + ListEntry = ListEntry->Flink; + } + + /* Initialize PFNs for the free memory */ + ProcessMemoryDescriptor(FreeDescriptor->BasePage, FreeDescriptor->PageCount, LoaderFree); + + /* Initialize PFNs for the physical pages backing the PFN database */ + ProcessMemoryDescriptor(OriginalFreeDescriptor.BasePage, + FreeDescriptor->BasePage - OriginalFreeDescriptor.BasePage, + LoaderMemoryData); + + /* Restore original free descriptor */ + *FreeDescriptor = OriginalFreeDescriptor; + + /* Initialize PFNs backing page tables */ + InitializePageTablePfns(); +} + +/** + * Initializes PFN database entries for the system page tables. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::InitializePageTablePfns(VOID) +{ + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn; + ULONG RootLevel; + PMMPTE RootPte; + + /* Determine root structure based on paging mode */ + if(MM::Paging::GetXpaStatus()) + { + /* XPA enabled, 3-level paging (PAE) */ + RootLevel = 3; + + /* Retrieve the PFN of the PML3 table and its virtual base address */ + PageFrameIndex = AR::CpuFunc::ReadControlRegister(3) >> MM_PAGE_SHIFT; + RootPte = (PMMPTE)MM::Paging::GetPpeAddress(NULLPTR); + } + else + { + /* XPA disabled, 2-level paging */ + RootLevel = 2; + + /* Retrieve the PFN of the PML2 table and its virtual base address */ + PageFrameIndex = AR::CpuFunc::ReadControlRegister(3) >> MM_PAGE_SHIFT; + RootPte = (PMMPTE)MM::Paging::GetPdeAddress(NULLPTR); + } + + /* Initialize the PFN entry for the root page table itself */ + Pfn = GetPfnEntry(PageFrameIndex); + if(Pfn) + { + /* Initialize the PFN entry */ + Pfn->PteAddress = NULLPTR; + Pfn->u1.WsIndex = 0; + Pfn->u2.ShareCount = 1; + Pfn->u3.e1.CacheAttribute = PfnNonCached; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + Pfn->u4.PteFrame = 0; + } + + /* Start recursive scan from the top level */ + if(RootPte) + { + /* Scan the root page table */ + ScanPageTable(RootPte, RootLevel); + } +} + +/** + * Recursively scans a page table to initialize PFN database entries for active pages. + * + * @param PointerPte + * Pointer to the base of the page table to scan. + * + * @param Level + * The paging level of the table being scanned. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::ScanPageTable(IN PMMPTE PointerPte, + IN ULONG Level) +{ + PVOID Address; + ULONG Index; + PMMPTE NextLevelPte; + ULONG PtesPerPage; + + /* Get the number of PTEs per page */ + PtesPerPage = MM::Pte::GetPtesPerPage(); + + /* Check if PML3 is enabled and current level is PDPT */ + if(Level == 3) + { + /* PAE PDPT has only 4 entries */ + PtesPerPage = 4; + } + + /* Iterate through all entries in the current page table */ + for(Index = 0; Index < PtesPerPage; Index++) + { + /* Check if the page table entry is present */ + if(MM::Paging::PteValid(PointerPte)) + { + /* Mark the PFN pointed to by this entry as active */ + LinkPfnForPageTable(MM::Paging::GetPageFrameNumber(PointerPte), PointerPte); + + /* Recurse to the next level, if this is not a leaf node (PTE) */ + if(Level > 1) + { + /* Calculate the virtual address mapped by this entry to find the next table */ + switch(Level) + { + case 3: + /* Calculate PDE */ + Address = MM::Paging::GetPpeVirtualAddress(PointerPte); + NextLevelPte = (PMMPTE)MM::Paging::GetPdeAddress(Address); + break; + case 2: + /* Calculate PTE */ + Address = MM::Paging::GetPdeVirtualAddress(PointerPte); + NextLevelPte = (PMMPTE)MM::Paging::GetPteAddress(Address); + break; + default: + /* Nothing to calculate, return NULLPTR */ + NextLevelPte = NULLPTR; + break; + } + + /* Recurse deeper if not at the bottom level (PTE) already */ + if(NextLevelPte) + { + /* Recursively scan the next level page table */ + ScanPageTable(NextLevelPte, Level - 1); + } + } + } + + /* Move to the next entry in the current table */ + PointerPte = MM::Paging::GetNextPte(PointerPte); + } +} diff --git a/xtoskrnl/mm/i686/pte.cc b/xtoskrnl/mm/i686/pte.cc new file mode 100644 index 0000000..0ed63bd --- /dev/null +++ b/xtoskrnl/mm/i686/pte.cc @@ -0,0 +1,128 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/i686/pte.cc + * DESCRIPTION: Page Table Entry (PTE) for i686 support + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Checks if the virtual address is valid and mapped in the page tables. + * + * @param VirtualAddress + * The virtual address to check. + * + * @return This routine returns TRUE if the address is valid, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Pte::AddressValid(IN PVOID VirtualAddress) +{ + /* Check if PDE and PTE are valid */ + if(!MM::Paging::PteValid(MM::Paging::GetPdeAddress(VirtualAddress)) || + !MM::Paging::PteValid(MM::Paging::GetPteAddress(VirtualAddress))) + { + /* Invalid PDE or PTE, return FALSE */ + return FALSE; + } + + /* Address is valid, return TRUE */ + return TRUE; +} + +/** + * Retrieves the base virtual address of the system PTEs. + * + * @return This routine returns a pointer to the first PTE in the system PTE space. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Pte::GetSystemPteBaseAddress(VOID) +{ + return MM::Paging::GetPteAddress(NULLPTR); +} + +/** + * Performs the initial setup of the system's page table hierarchy. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::InitializePageTable(VOID) +{ + PMMPTE EndSpacePte, PointerPte; + PMMMEMORY_LAYOUT MemoryLayout; + CPUID_REGISTERS CpuRegisters; + MMPTE TemplatePte; + + /* Retrieve current paging mode and memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Get CPU features */ + CpuRegisters.Leaf = CPUID_GET_STANDARD1_FEATURES; + AR::CpuFunc::CpuId(&CpuRegisters); + + /* Check if Paging Global Extensions (PGE) is supported */ + if(CpuRegisters.Edx & CPUID_FEATURES_EDX_PGE) + { + /* Enable the Global Paging (PGE) feature */ + AR::CpuFunc::WriteControlRegister(4, AR::CpuFunc::ReadControlRegister(4) | CR4_PGE); + } + + /* Get the PD user-space range for both legacy and PAE paging */ + PointerPte = (PMMPTE)MM::Paging::GetPdeAddress(0); + EndSpacePte = (PMMPTE)MM::Paging::GetPdeAddress(MemoryLayout->UserSpaceEnd); + + /* Clear all top-level entries mapping the user address space */ + while(PointerPte <= EndSpacePte) + { + MM::Paging::ClearPte(PointerPte); + PointerPte = MM::Paging::GetNextPte(PointerPte); + } + + /* Flush the TLB to invalidate all non-global entries */ + AR::CpuFunc::FlushTlb(); + + /* Create a template PTE for mapping kernel pages */ + MM::Paging::ClearPte(&TemplatePte); + MM::Paging::SetPte(&TemplatePte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE); + + /* Map the kernel's PD entries */ + MM::Pte::MapPDE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte); +} + +/** + * Maps a range of virtual addresses at the PPE (Page Directory Pointer Entry) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplatePpe + * A template PPE to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapPPE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPPE TemplatePpe) +{ + /* Just a stub on i686 platform */ + return; +} diff --git a/xtoskrnl/mm/init.cc b/xtoskrnl/mm/init.cc deleted file mode 100644 index d7694d8..0000000 --- a/xtoskrnl/mm/init.cc +++ /dev/null @@ -1,150 +0,0 @@ -/** - * PROJECT: ExectOS - * COPYRIGHT: See COPYING.md in the top level directory - * FILE: xtoskrnl/mm/init.cc - * DESCRIPTION: Memory Manager initialization routines - * DEVELOPERS: Rafal Kupiec - */ - -#include - - -/** - * Performs an early initialization of the XTOS Memory Manager. - * - * @return This routine does not return any value. - * - * @since XT 1.0 - */ -XTAPI -VOID -MM::Init::InitializeMemoryManager(VOID) -{ - /* Scan memory descriptors provided by the boot loader */ - ScanMemoryDescriptors(); - - /* Check if there are enough physical pages */ - if(NumberOfPhysicalPages < MM_MINIMUM_PHYSICAL_PAGES) - { - /* Insufficient physical pages, kernel panic */ - DebugPrint(L"Insufficient physical pages! Install additional memory\n"); - KE::Crash::Panic(0); - } - - /* Proceed with architecture specific initialization */ - InitializeArchitecture(); -} - -/** - * Scans memory descriptors provided by the boot loader. - * - * @return This routine does not return any value. - * - * @since XT 1.0 - */ -XTAPI -VOID -MM::Init::ScanMemoryDescriptors(VOID) -{ - PLIST_ENTRY LoaderMemoryDescriptors, MemoryMappings; - PLOADER_MEMORY_DESCRIPTOR MemoryDescriptor; - PFN_NUMBER FreePages; - - /* Initially, set number of free pages to 0 */ - FreePages = 0; - - /* Get a list of memory descriptors provided by the boot loader */ - LoaderMemoryDescriptors = KE::BootInformation::GetMemoryDescriptors(); - - /* Iterate through memory mappings provided by the boot loader */ - MemoryMappings = LoaderMemoryDescriptors->Flink; - while(MemoryMappings != LoaderMemoryDescriptors) - { - /* Get memory descriptor */ - MemoryDescriptor = CONTAIN_RECORD(MemoryMappings, LOADER_MEMORY_DESCRIPTOR, ListEntry); - - /* Check if memory type is invisible or cached */ - if(VerifyMemoryTypeInvisible(MemoryDescriptor->MemoryType) || - (MemoryDescriptor->MemoryType == LoaderHardwareCachedMemory)) - { - /* Skip this mapping */ - MemoryMappings = MemoryMappings->Flink; - continue; - } - - /* Make sure that memory type is not bad */ - if(MemoryDescriptor->MemoryType != LoaderBad) - { - /* Increment number of physical pages */ - NumberOfPhysicalPages += MemoryDescriptor->PageCount; - } - - /* Find lowest physical page */ - if(MemoryDescriptor->BasePage < LowestPhysicalPage) - { - /* Update lowest physical page */ - LowestPhysicalPage = MemoryDescriptor->BasePage; - } - - /* Find highest physical page */ - if(MemoryDescriptor->BasePage + MemoryDescriptor->PageCount > HighestPhysicalPage) - { - /* Update highest physical page */ - HighestPhysicalPage = (MemoryDescriptor->BasePage + MemoryDescriptor->PageCount) - 1; - } - - /* Check if memory type should be considered as free */ - if(VerifyMemoryTypeFree(MemoryDescriptor->MemoryType)) - { - /* Check if this descriptor contains more free pages */ - if(MemoryDescriptor->PageCount >= FreePages) - { - /* Update free descriptor */ - FreePages = MemoryDescriptor->PageCount; - FreeDescriptor = MemoryDescriptor; - } - } - - /* Get next memory descriptor */ - MemoryMappings = MemoryMappings->Flink; - } - - /* Store original free descriptor */ - RTL::Memory::CopyMemory(&OldFreeDescriptor, FreeDescriptor, sizeof(LOADER_MEMORY_DESCRIPTOR)); -} - -/** Checks whether the specified memory type should be considered as free. - * - * @param MemoryType - * Specifies the memory type to verify. - * - * @return This routine returns TRUE if the specified memory type should be considered as free, or FALSE otherwise. - * - * @since XT 1.0 - */ -XTAPI -BOOLEAN -MM::Init::VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType) -{ - return ((MemoryType == LoaderFree) || (MemoryType == LoaderFirmwareTemporary) || - (MemoryType == LoaderLoadedProgram) || (MemoryType == LoaderOsloaderStack)); -} - -/** - * Checks whether the specified memory type should be considered as invisible for the memory manager. - * - * @param MemoryType - * Specifies the memory type to verify. - * - * @return This routine returns TRUE if the specified memory type should be considered as invisible, or FALSE otherwise. - * - * @since XT 1.0 - */ -XTAPI -BOOLEAN -MM::Init::VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType) -{ - return ((MemoryType == LoaderFirmwarePermanent) || - (MemoryType == LoaderSpecialMemory) || - (MemoryType == LoaderBBTMemory)); -} diff --git a/xtoskrnl/mm/kpool.cc b/xtoskrnl/mm/kpool.cc index 190c7da..a8c8835 100644 --- a/xtoskrnl/mm/kpool.cc +++ b/xtoskrnl/mm/kpool.cc @@ -4,6 +4,7 @@ * FILE: xtoskrnl/mm/kpool.cc * DESCRIPTION: Kernel pool memory management * DEVELOPERS: Rafal Kupiec + * Aiken Harris */ #include @@ -27,12 +28,68 @@ */ XTAPI XTSTATUS -MM::KernelPool::AllocateKernelStack(IN PVOID *Stack, - IN BOOLEAN LargeStack, - IN UCHAR SystemNode) +MM::KernelPool::AllocateKernelStack(OUT PVOID *Stack, + IN ULONG StackSize) { - UNIMPLEMENTED; - return STATUS_NOT_IMPLEMENTED; + PFN_COUNT StackPages; + PMMPTE PointerPte, StackPte; + MMPTE TempPte, InvalidPte; + PFN_NUMBER PageFrameIndex; + ULONG Index; + + /* Initialize the output stack pointer to NULLPTR */ + *Stack = NULLPTR; + + /* Convert the requested stack size into a page count */ + StackPages = SIZE_TO_PAGES(StackSize); + + /* Reserve PTEs for the stack pages, plus a guard page */ + StackPte = MM::Pte::ReserveSystemPtes(StackPages + 1, SystemPteSpace); + if(!StackPte) + { + /* Failed to reserve PTEs for the new kernel stack */ + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Set up a template for an invalid PTE */ + MM::Paging::SetPte(&InvalidPte, 0, MM_PTE_GUARDED); + + /* Set up a template for a valid, writable stack PTE */ + MM::Paging::ClearPte(&TempPte); + MM::Paging::SetPte(&TempPte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE); + + /* Acquire the PFN database lock and raise runlevel to DISPATCH_LEVEL */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Start iterating from the base of the reserved PTE block */ + PointerPte = StackPte; + + /* Loop through each page of the stack that needs to be allocated */ + for(Index = 0; Index < StackPages; Index++) + { + /* Advance to the next PTE */ + PointerPte = MM::Paging::GetNextPte(PointerPte); + + /* Allocate a physical page and temporarily mark the PTE as invalid */ + PageFrameIndex = MM::Pfn::AllocatePhysicalPage(MM::Colors::GetNextColor()); + *PointerPte = InvalidPte; + + /* Associate the physical page with its corresponding PTE in the PFN database */ + MM::Pfn::LinkPfn(PageFrameIndex, PointerPte, TRUE); + + /* Make the PTE valid, mapping the virtual address to the physical page */ + MM::Paging::SetPte(&TempPte, PageFrameIndex, 0); + *PointerPte = TempPte; + } + + /* Zero the newly allocated stack memory, skipping the guard page */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(MM::Paging::GetNextPte(StackPte)), + MM_PAGE_SIZE * StackPages); + + /* Return a pointer to the top of the new stack */ + *Stack = MM::Paging::GetPteVirtualAddress(MM::Paging::AdvancePte(StackPte, StackPages + 1)); + return STATUS_SUCCESS; } /** @@ -96,9 +153,40 @@ MM::KernelPool::AllocateProcessorStructures(IN ULONG CpuNumber, XTAPI VOID MM::KernelPool::FreeKernelStack(IN PVOID Stack, - IN BOOLEAN LargeStack) + IN ULONG StackSize) { - UNIMPLEMENTED; + PFN_COUNT StackPages; + PMMPTE PointerPte; + ULONG Index; + + /* Get the PTE for the top of the stack, including the guard page */ + PointerPte = MM::Paging::AdvancePte(MM::Paging::GetPteAddress(Stack), -1); + + /* Convert the stack size into a page count */ + StackPages = SIZE_TO_PAGES(StackSize); + + /* Start a guarded code block */ + { + /* Acquire the PFN database lock */ + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Loop through each page of the stack that needs to be freed */ + for(Index = 0; Index < StackPages; Index++) + { + /* Ensure the PTE is valid */ + if(MM::Paging::PteValid(PointerPte)) + { + /* Free the physical page */ + MM::Pfn::FreePhysicalPage(PointerPte); + } + + /* Advance to the next PTE */ + PointerPte = MM::Paging::AdvancePte(PointerPte, -1); + } + } + + /* Release all system PTEs used by the stack, including the guard page */ + MM::Pte::ReleaseSystemPtes(PointerPte, StackPages + 1, SystemPteSpace); } /** diff --git a/xtoskrnl/mm/mmgr.cc b/xtoskrnl/mm/mmgr.cc new file mode 100644 index 0000000..b8116f2 --- /dev/null +++ b/xtoskrnl/mm/mmgr.cc @@ -0,0 +1,177 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/mmgr.cc + * DESCRIPTION: Memory Manager + * DEVELOPERS: Rafal Kupiec + * Aiken Harris + */ + +#include + + +/** + * Computes the size of the boot image. + * + * @param BootImageSize + * Supplies a pointer to a variable that will receive the size of the boot image in pages. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::ComputeBootImageSize(OUT PPFN_NUMBER BootImageSize) +{ + PKERNEL_INITIALIZATION_BLOCK InitializationBlock; + PFN_NUMBER ImageSize; + ULONG_PTR Alignment; + + /* Get the kernel initialization block */ + InitializationBlock = KE::BootInformation::GetInitializationBlock(); + + /* Calculate the alignment based on the PTE size */ + Alignment = ((MM_PAGE_SIZE / MM::Paging::GetPteSize()) * MM_PAGE_SIZE); + + /* Calculate the size of the boot image */ + ImageSize = InitializationBlock->BootImageSize * MM_PAGE_SIZE; + ImageSize = (ImageSize + Alignment - 1) & ~(Alignment - 1); + + /* Return number of pages used by the boot image */ + *BootImageSize = ImageSize / MM_PAGE_SIZE; +} + +/** + * Retrieves the amount of total available memory in the system. + * + * @return This routine returns the amount of available memory in the system in megabytes. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::Manager::GetInstalledMemorySize(VOID) +{ + /* Return the amount of installed memory */ + return (MM::Pfn::GetNumberOfPhysicalPages() * MM_PAGE_SIZE) / 1048576; +} + +/** + * Retrieves a pointer to the system's virtual memory layout structure. + * + * @return This routine returns a pointer to the memory layout structure. + * + * @since XT 1.0 + */ +XTAPI +PMMMEMORY_LAYOUT +MM::Manager::GetMemoryLayout(VOID) +{ + /* Return a pointer to the global memory layout structure */ + return &MemoryLayout; +} + +/** + * Retrieves the number of system PTEs. + * + * @return This routine returns the number of system PTEs. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Manager::GetNumberOfSystemPtes() +{ + return NumberOfSystemPtes; +} + +/** + * Performs an early initialization of the XTOS Memory Manager. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Manager::InitializeMemoryManager(VOID) +{ + /* Scan memory descriptors provided by the boot loader */ + MM::Pfn::ScanMemoryDescriptors(); + + /* Check if there are enough physical pages */ + if(MM::Pfn::GetNumberOfPhysicalPages() < MM_MINIMUM_PHYSICAL_PAGES) + { + /* Insufficient physical pages, kernel panic */ + DebugPrint(L"Insufficient physical pages! Install additional memory\n"); + KE::Crash::Panic(0); + } + + /* Compute page colors to reduce CPU cache conflicts */ + MM::Colors::ComputePageColoring(); + + /* Initialize and dump memory layout */ + InitializeMemoryLayout(); + DumpMemoryLayout(); + + /* Initialize PTE template */ + MM::Pte::InitializeSystemPte(); + + /* Initialize page table */ + MM::Pte::InitializePageTable(); + + /* Initialize system PTE space */ + MM::Pte::InitializeSystemPteSpace(); + + /* Initialize non-paged pool */ + MM::Allocator::InitializeNonPagedPool(); + + /* Initialize PFN database */ + MM::Pfn::InitializePfnDatabase(); + + /* Initialize paged pool */ + MM::Allocator::InitializePagedPool(); + + /* Flush TLB */ + AR::CpuFunc::FlushTlb(); +} + +/** + * Checks whether the specified memory type should be considered as free. + * + * @param MemoryType + * Specifies the memory type to verify. + * + * @return This routine returns TRUE if the specified memory type should be considered as free, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Manager::VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType) +{ + return ((MemoryType == LoaderFirmwareTemporary) || + (MemoryType == LoaderFree) || + (MemoryType == LoaderLoadedProgram) || + (MemoryType == LoaderOsloaderStack)); +} + +/** + * Checks whether the specified memory type should be considered as invisible for the memory manager. + * + * @param MemoryType + * Specifies the memory type to verify. + * + * @return This routine returns TRUE if the specified memory type should be considered as invisible, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Manager::VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType) +{ + return ((MemoryType == LoaderBBTMemory) || + (MemoryType == LoaderFirmwarePermanent) || + (MemoryType == LoaderSpecialMemory)); +} diff --git a/xtoskrnl/mm/paging.cc b/xtoskrnl/mm/paging.cc index 58833b1..69305a6 100644 --- a/xtoskrnl/mm/paging.cc +++ b/xtoskrnl/mm/paging.cc @@ -9,6 +9,28 @@ #include +/** + * Advances a PTE pointer by a given number of entries, considering the actual PTE size. + * + * @param Pte + * The PTE pointer to advance. + * + * @param Count + * The number of PTE entries to advance by. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Paging::AdvancePte(IN PMMPTE Pte, + IN LONG Count) +{ + /* Return advanced PTE pointer */ + return PmlRoutines->AdvancePte(Pte, Count); +} + /** * Clears the contents of a page table entry (PTE). * @@ -21,8 +43,9 @@ */ XTAPI VOID -MM::Paging::ClearPte(PHARDWARE_PTE PtePointer) +MM::Paging::ClearPte(IN PMMPTE PtePointer) { + /* Clear PTE */ PmlRoutines->ClearPte(PtePointer); } @@ -78,6 +101,77 @@ MM::Paging::FlushTlb(VOID) } } +/** + * Gets the next entry in a PTE list. + * + * @param Pte + * The PTE pointer to get the next entry from. + * + * @return This routine returns the next entry in the PTE list. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::Paging::GetNextEntry(IN PMMPTE Pte) +{ + /* Return next entry in PTE list */ + return PmlRoutines->GetNextEntry(Pte); +} + +/** + * Advances a PTE pointer, considering the actual PTE size. + * + * @param Pte + * The PTE pointer to advance. + * + * @return The advanced PTE pointer. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Paging::GetNextPte(IN PMMPTE Pte) +{ + /* Return advanced PTE pointer */ + return PmlRoutines->GetNextPte(Pte); +} + +/** + * Checks if a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to check. + * + * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Paging::GetOneEntry(IN PMMPTE Pte) +{ + /* Return one entry status */ + return PmlRoutines->GetOneEntry(Pte); +} + +/** + * Gets the page frame number from a corresponding PTE. + * + * @param Pte + * The PTE pointer to get the page frame number from. + * + * @return This routine returns the page frame number. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Paging::GetPageFrameNumber(IN PMMPTE Pte) +{ + return PmlRoutines->GetPageFrameNumber(Pte); +} + /** * Gets the page map routines for basic paging mode (non-XPA). * @@ -90,9 +184,25 @@ MM::PPAGEMAP MM::Paging::GetPageMapBasicRoutines(VOID) { static MM::PageMapBasic PageMapBasicRoutines; + + /* Return non-XPA page map routines */ return &PageMapBasicRoutines; } +/** + * Gets Page Map Level (PML) for current paging mode. + * + * @return This routine returns the page map level. + * + * @since XT 1.0 + */ +XTAPI +USHORT +MM::Paging::GetPageMapLevel() +{ + return PmlRoutines->GetPageMapLevel(); +} + /** * Gets the page map routines for eXtended Physical Addressing (XPA) mode. * @@ -105,6 +215,8 @@ MM::PPAGEMAP MM::Paging::GetPageMapXpaRoutines(VOID) { static MM::PageMapXpa PageMapXpaRoutines; + + /* Return XPA page map routines */ return &PageMapXpaRoutines; } @@ -120,11 +232,30 @@ MM::Paging::GetPageMapXpaRoutines(VOID) */ XTAPI PMMPDE -MM::Paging::GetPdeAddress(PVOID Address) +MM::Paging::GetPdeAddress(IN PVOID Address) { + /* Return PDE address */ return PmlRoutines->GetPdeAddress(Address); } +/** + * Gets the virtual address that is mapped by a given Page Directory Entry. + * + * @param PdePointer + * Specifies the address of the PDE. + * + * @return This routine returns the virtual address mapped by the PDE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::Paging::GetPdeVirtualAddress(IN PMMPDE PdePointer) +{ + /* Return PTE virtual address */ + return PmlRoutines->GetPdeVirtualAddress(PdePointer); +} + /** * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address. * @@ -137,11 +268,48 @@ MM::Paging::GetPdeAddress(PVOID Address) */ XTAPI PMMPPE -MM::Paging::GetPpeAddress(PVOID Address) +MM::Paging::GetPpeAddress(IN PVOID Address) { + /* Return PPE address */ return PmlRoutines->GetPpeAddress(Address); } +/** + * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry. + * + * @param PpePointer + * Specifies the address of the PPE. + * + * @return This routine returns the virtual address mapped by the PPE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::Paging::GetPpeVirtualAddress(IN PMMPPE PpePointer) +{ + /* Return PTE virtual address */ + return PmlRoutines->GetPpeVirtualAddress(PpePointer); +} + + /** + * Gets the entire contents of a Page Table Entry (PTE) as a single value. + * + * @param PtePointer + * Pointer to the Page Table Entry (PTE) to read. + * + * @return This routine returns the contents of the PTE as a single value. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::Paging::GetPte(IN PMMPTE PtePointer) +{ + /* Return PTE value */ + return PmlRoutines->GetPte(PtePointer); +} + /** * Gets the address of the PTE (Page Table Entry), that maps given address. * @@ -154,11 +322,135 @@ MM::Paging::GetPpeAddress(PVOID Address) */ XTAPI PMMPTE -MM::Paging::GetPteAddress(PVOID Address) +MM::Paging::GetPteAddress(IN PVOID Address) { + /* Return PTE address */ return PmlRoutines->GetPteAddress(Address); } +/** + * Calculates the distance between two PTE pointers. + * + * @param EndPte + * Pointer to the ending Page Table Entry. + * + * @param StartPte + * Pointer to the starting Page Table Entry. + * + * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte. + * + * @since XT 1.0 + */ +XTAPI +LONG +MM::Paging::GetPteDistance(PMMPTE EndPte, + PMMPTE StartPte) +{ + /* Return distance between PTE pointers */ + return PmlRoutines->GetPteDistance(EndPte, StartPte); +} + +/** + * Gets the size of a PTE. + * + * @return This routine returns the size of a PTE. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Paging::GetPteSize(VOID) +{ + /* Return the size of MMPTE */ + return PmlRoutines->GetPteSize(); +} + +/** + * Gets the software protection value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software protection value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Paging::GetPteSoftwareProtection(IN PMMPTE PtePointer) +{ + /* Return PTE software protection value */ + return PmlRoutines->GetPteSoftwareProtection(PtePointer); +} + +/** + * Gets the software prototype value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software prototype value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Paging::GetPteSoftwarePrototype(IN PMMPTE PtePointer) +{ + /* Return PTE software prototype value */ + return PmlRoutines->GetPteSoftwarePrototype(PtePointer); +} + +/** + * Gets the software transition value of the corresponding Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the PTE software transition value. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Paging::GetPteSoftwareTransition(IN PMMPTE PtePointer) +{ + /* Return PTE software transition value */ + return PmlRoutines->GetPteSoftwareTransition(PtePointer); +} + +/** + * Gets the virtual address that is mapped by a given Page Table Entry. + * + * @param PtePointer + * Specifies the address of the PTE. + * + * @return This routine returns the virtual address mapped by the PTE. + * + * @since XT 1.0 + */ +XTAPI +PVOID +MM::Paging::GetPteVirtualAddress(IN PMMPTE PtePointer) +{ + /* Return PTE virtual address */ + return PmlRoutines->GetPteVirtualAddress(PtePointer); +} + +/** + * Gets current status of eXtended Physical Addressing (XPA). + * + * @return This routine returns TRUE if PAE or LA57 (XPA) is enabled, or FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Paging::GetXpaStatus() +{ + return PmlRoutines->GetXpaStatus(); +} + /** * Detects if eXtended Physical Addressing (XPA) is enabled and initializes page map support. * @@ -198,22 +490,20 @@ MM::Paging::InitializePageMapSupport(VOID) */ XTAPI BOOLEAN -MM::Paging::PteValid(PHARDWARE_PTE PtePointer) +MM::Paging::PteValid(IN PMMPTE PtePointer) { + /* Check if PTE is valid */ return PmlRoutines->PteValid(PtePointer); } /** - * Sets a PML2 page table entry (PTE) with the specified physical page and access flags. + * Sets the next entry in a PTE list. * - * @param PtePointer - * Pointer to the page table entry (PTE) to set. + * @param Pte + * The PTE pointer to modify. * - * @param PageFrameNumber - * Physical frame number to map. - * - * @param Writable - * Indicates whether the page should be writable. + * @param Value + * The value to set as the next entry. * * @return This routine does not return any value. * @@ -221,11 +511,80 @@ MM::Paging::PteValid(PHARDWARE_PTE PtePointer) */ XTAPI VOID -MM::Paging::SetPte(PHARDWARE_PTE PtePointer, - PFN_NUMBER PageFrameNumber, - BOOLEAN Writable) +MM::Paging::SetNextEntry(IN PMMPTE Pte, + IN ULONG_PTR Value) { - PmlRoutines->SetPte(PtePointer, PageFrameNumber, Writable); + /* Set next entry in PTE list */ + PmlRoutines->SetNextEntry(Pte, Value); +} + +/** + * Sets the flag indicating whether a PTE list contains only one entry. + * + * @param Pte + * The PTE pointer to modify. + * + * @param Value + * The value to set. TRUE if the list has only one entry, FALSE otherwise. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Paging::SetOneEntry(IN PMMPTE Pte, + IN BOOLEAN Value) +{ + /* Set one entry status */ + PmlRoutines->SetOneEntry(Pte, Value); +} + +/** + * Sets a Page Table Entry (PTE) with the specified physical page and access flags. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param PageFrameNumber + * Physical frame number to map. + * + * @param AttributesMask + * Specifies the attributes mask to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Paging::SetPte(IN PMMPTE PtePointer, + IN PFN_NUMBER PageFrameNumber, + IN ULONG_PTR AttributesMask) +{ + /* Set PTE */ + PmlRoutines->SetPte(PtePointer, PageFrameNumber, AttributesMask); +} + +/** + * Sets a Page Table Entry (PTE) with the specified attributes. + * + * @param PtePointer + * Pointer to the page table entry (PTE) to set. + * + * @param Attributes + * Specifies the attributes to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Paging::SetPte(IN PMMPTE PtePointer, + IN ULONG_PTR Attributes) +{ + PmlRoutines->SetPte(PtePointer, Attributes); } /** @@ -246,9 +605,54 @@ MM::Paging::SetPte(PHARDWARE_PTE PtePointer, */ XTAPI VOID -MM::Paging::SetPteCaching(PHARDWARE_PTE PtePointer, - BOOLEAN CacheDisable, - BOOLEAN WriteThrough) +MM::Paging::SetPteCaching(IN PMMPTE PtePointer, + IN BOOLEAN CacheDisable, + IN BOOLEAN WriteThrough) { + /* Set caching attributes */ PmlRoutines->SetPteCaching(PtePointer, CacheDisable, WriteThrough); } + +/** + * Transitions a Page Table Entry (PTE) to invalid state + * + * @param PointerPte + * Pointer to the page table entry (PTE) to transition. + * + * @param Protection + * Specifies the protection attribute to apply to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Paging::TransitionPte(IN PMMPTE PointerPte, + IN ULONG_PTR Protection) +{ + /* Transition PTE */ + PmlRoutines->TransitionPte(PointerPte, Protection); +} + +/** + * Writes a Page Table Entry (PTE) with the specified value. + * + * @param Pte + * Pointer to the page table entry (PTE) to write. + * + * @param Value + * The value to write to the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Paging::WritePte(IN PMMPTE Pte, + IN MMPTE Value) +{ + /* Assign PTE value */ + PmlRoutines->WritePte(Pte, Value); +} diff --git a/xtoskrnl/mm/pfn.cc b/xtoskrnl/mm/pfn.cc new file mode 100644 index 0000000..ad94f0c --- /dev/null +++ b/xtoskrnl/mm/pfn.cc @@ -0,0 +1,1319 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/pfn.cc + * DESCRIPTION: Physical Frame Number (PFN) support + * DEVELOPERS: Aiken Harris + * Rafal Kupiec + */ + +#include + + +/** + * Allocates a block of physical pages for early kernel initialization. + * + * @param NumberOfPages + * The number of physical pages to allocate. + * + * @return This routine returns the base page frame number (PFN) of the allocated block. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Pfn::AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages) +{ + PFN_NUMBER Pfn; + + /* Check if the largest free memory block has enough pages */ + if(NumberOfPages > FreeDescriptor->PageCount) + { + /* Not enough physical memory available, kernel panic */ + DebugPrint(L"Insufficient physical pages! Install additional memory\n"); + KE::Crash::Panic(0); + } + + /* Allocate pages from the beginning of the free descriptor */ + Pfn = FreeDescriptor->BasePage; + FreeDescriptor->BasePage += NumberOfPages; + FreeDescriptor->PageCount -= NumberOfPages; + + /* Return the base page frame number of the allocated block */ + return Pfn; +} + +/** + * Allocates a physical page frame (PFN) from one of the system's free page lists. + * + * @param Color + * The preferred page color, used to optimize CPU cache alignment and reduce cache contention. + * + * @return This routine returns the Page Frame Number (PFN) of the allocated page. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Pfn::AllocatePhysicalPage(IN ULONG Color) +{ + PFN_NUMBER PageNumber; + ULONG PagingColorsMask; + + /* Check if any physical pages are available in the system */ + if(!AvailablePages) + { + /* No physical pages are available in the system, return 0 */ + return 0; + } + + /* Retrieve the bitmask used for calculating a page's color */ + PagingColorsMask = MM::Colors::GetPagingColorsMask(); + + /* Attempt to retrieve a page from the colored free page list */ + PageNumber = MM::Colors::GetFreePages(FreePageList, Color)->Flink; + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the colored free page list, check the colored zero page list */ + PageNumber = MM::Colors::GetFreePages(ZeroedPageList, Color)->Flink; + } + + /* Attempt to retrieve a page from the colored zero list */ + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the colored zero page list, check the global free page list */ + PageNumber = FreePagesList.Flink; + } + + /* Attempt to retrieve a page from the global free page list */ + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the global free page list, check the global zero page list */ + PageNumber = ZeroedPagesList.Flink; + } + + /* Remove the page from its list and return its PFN */ + return UnlinkFreePage(PageNumber, PageNumber & PagingColorsMask); +} + +/** + * Calculates the total number of pages required for the PFN database and its associated color tables. + * + * @param DatabaseSize + * A pointer to a variable that will receive the number of pages required for the PFN database. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::ComputePfnDatabaseSize(OUT PPFN_NUMBER DatabaseSize) +{ + PFN_NUMBER PfnDatabaseSize; + + /* Calculate the total number of pages required for the PFN database */ + PfnDatabaseSize = (HighestPhysicalPage + 1) * sizeof(MMPFN); + PfnDatabaseSize += (MM::Colors::GetPagingColors() * sizeof(MMCOLOR_TABLES) * 2); + PfnDatabaseSize = ROUND_UP(PfnDatabaseSize, MM_PAGE_SIZE); + PfnDatabaseSize >>= MM_PAGE_SHIFT; + + /* Return the PFN database size */ + *DatabaseSize = PfnDatabaseSize; +} + +/** + * Decrements the global count of available pages. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::DecrementAvailablePages(VOID) +{ + /* Decrement the global count of available pages */ + AvailablePages--; +} + +/** + * Decrements the reference count of a PFN entry. + * + * @param PageFrameNumber + * A pointer to the PFN database entry for the physical page. + * + * @param PageFrameIndex + * The page frame number of the physical page. + * + * @param BeginStandbyList + * Determines whether the page should be added to the beginning of the standby list. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::DecrementReferenceCount(IN PMMPFN PageFrameNumber, + IN PFN_NUMBER PageFrameIndex, + IN BOOLEAN BeginStandbyList) +{ + /* Decrement the PFN reference count */ + PageFrameNumber->u3.e2.ReferenceCount--; + + /* If other references exist, no further action is needed */ + if(PageFrameNumber->u3.e2.ReferenceCount) + { + /* No further action can be taken */ + return; + } + + /* If the reference count is zero, the share count should also be zero */ + if(PageFrameNumber->u2.ShareCount) + { + /* This indicates a bug; crash the system */ + KE::Crash::PanicEx(0x4E, + 0x07, + PageFrameIndex, + PageFrameNumber->u2.ShareCount, + 0); + } + + /* Check if the PTE is marked as being ready for removal */ + if(MM::Paging::GetPte(PageFrameNumber->PteAddress) & 0x1) + { + /* Check the page's cache attribute */ + if((PageFrameNumber->u3.e1.CacheAttribute != PfnCached) && + (PageFrameNumber->u3.e1.CacheAttribute != PfnNotMapped)) + { + UNIMPLEMENTED; + } + + /* The page is no longer needed, free it by linking it to the free list */ + LinkFreePage(PageFrameIndex); + + /* No further action is needed */ + return; + } + + /* The page is unreferenced and unmapped, so place it on the appropriate list */ + if(PageFrameNumber->u3.e1.Modified == 1) + { + /* Link dirty page to the modified list */ + LinkPage(&ModifiedPagesList, PageFrameIndex); + } + else + { + /* Check if the page has been marked for removal */ + if(PageFrameNumber->u3.e1.RemovalRequested == 1) + { + /* Update the page location and move it to the bad pages list */ + PageFrameNumber->u3.e1.PageLocation = StandbyPageList; + LinkPage(&BadPagesList, PageFrameIndex); + } + else + { + /* Link clean page to the standby list */ + if(BeginStandbyList) + { + /* Link clean page to the beginning of the standby list */ + LinkStandbyPage(PageFrameIndex); + } + else + { + /* Link clean page to the end of the standby list */ + LinkPage(&StandbyPagesList, PageFrameIndex); + } + } + } +} + +/** + * Decrements the share count of a PFN entry, which represents the number of PTEs mapping the page. + * + * @param PageFrameNumber + * A pointer to the PFN database entry for the physical page. + * + * @param PageFrameIndex + * The page frame number of the physical page. + * + * @param BeginStandbyList + * Determines whether the page should be added to the beginning of the standby list. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::DecrementShareCount(IN PMMPFN PageFrameNumber, + IN PFN_NUMBER PageFrameIndex, + IN BOOLEAN BeginStandbyList) +{ + /* Ensure the page is in a valid state */ + if((PageFrameNumber->u3.e1.PageLocation != ActiveAndValid) && + (PageFrameNumber->u3.e1.PageLocation != StandbyPageList)) + { + /* This indicates a bug; crash the system */ + KE::Crash::PanicEx(0x4E, + 0x99, + PageFrameIndex, + PageFrameNumber->u3.e1.PageLocation, + 0); + } + + /* Decrement the PFN share count */ + PageFrameNumber->u2.ShareCount--; + + /* Check if the share count has dropped to zero */ + if(!PageFrameNumber->u2.ShareCount) + { + /* Check if this is a prototype PTE */ + if(PageFrameNumber->u3.e1.PrototypePte) + { + /* Transition the PTE to invalid state */ + MM::Paging::TransitionPte(PageFrameNumber->PteAddress, + MM::Paging::GetPteSoftwareProtection(&PageFrameNumber->OriginalPte)); + } + + /* Mark the page as being in a transition state */ + PageFrameNumber->u3.e1.PageLocation = TransitionPage; + + /* Check if there are still outstanding references */ + if(PageFrameNumber->u3.e2.ReferenceCount == 1) + { + /* Check if the PTE is marked as being ready for removal */ + if(MM::Paging::GetPte(PageFrameNumber->PteAddress) & 0x1) + { + /* Reset the reference count */ + PageFrameNumber->u3.e2.ReferenceCount = 0; + + /* Check the page's cache attribute */ + if((PageFrameNumber->u3.e1.CacheAttribute != PfnCached) && + (PageFrameNumber->u3.e1.CacheAttribute != PfnNotMapped)) + { + UNIMPLEMENTED; + } + + /* Mark the page as active and valid again */ + PageFrameNumber->u3.e1.PageLocation = ActiveAndValid; + + /* The page is no longer needed, free it by linking it to the free list */ + MM::Pfn::LinkFreePage(PageFrameIndex); + } + else + { + /* The PTE can not be removed yet, decrement the reference count */ + DecrementReferenceCount(PageFrameNumber, PageFrameIndex, BeginStandbyList); + } + } + else + { + /* There are still some outstanding references, decrement the reference count */ + PageFrameNumber->u3.e2.ReferenceCount--; + } + } +} + +/** + * Frees a physical page that is mapped by a given PTE. + * + * @param PointerPte + * A pointer to the Page Table Entry (PTE) that maps the physical page to be freed. + * + * @return This routine does not return a value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::FreePhysicalPage(IN PMMPTE PointerPte) +{ + PFN_COUNT PageFrameNumber, PageTableFrameNumber; + PMMPFN PageFrame, PageTableFrame; + + /* Get the page frame number from the PTE */ + PageFrameNumber = MM::Paging::GetPageFrameNumber(PointerPte); + PageFrame = MM::Pfn::GetPfnEntry(PageFrameNumber); + + /* Get the page table frame number corresponding to the PTE */ + PageTableFrameNumber = PageFrame->u4.PteFrame; + PageTableFrame = MM::Pfn::GetPfnEntry(PageTableFrameNumber); + + /* Decrement the share count of the page table */ + MM::Pfn::DecrementShareCount(PageTableFrame, PageTableFrameNumber, FALSE); + + /* Mark the PTE as being ready for removal */ + MM::Paging::SetPte(PageFrame->PteAddress, MM::Paging::GetPte(PageFrame->PteAddress) | 1); + + /* Decrement the share count of the page */ + MM::Pfn::DecrementShareCount(PageFrame, PageFrameNumber, FALSE); +} + +/** + * Retrieves the number of available physical pages. + * + * @return This routine returns the number of available physical pages. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Pfn::GetAvailablePages(VOID) +{ + /* Return the number of available pages */ + return AvailablePages; +} + +/** + * Retrieves the highest physical page number (PFN) detected in the system. + * + * @return This routine returns the highest physical page number. + * + * @since XT 1.0 + */ +XTAPI +ULONG_PTR +MM::Pfn::GetHighestPhysicalPage(VOID) +{ + /* Return the highest physical page number */ + return HighestPhysicalPage; +} + +/** + * Retrieves the total number of physical pages managed by the system. + * + * @return Returns the total count of physical memory pages. + * + * @since XT 1.0 + */ +XTAPI +ULONGLONG +MM::Pfn::GetNumberOfPhysicalPages(VOID) +{ + /* Return the number of physical pages */ + return NumberOfPhysicalPages; +} + +/** + * Retrieves a pointer to the PFN database entry for a given physical page. + * + * @param Pfn + * The Page Frame Number (PFN) to look up. + * + * @return This routine returns a pointer to the MMPFN structure for the given PFN, or NULLPTR if the PFN is invalid. + * + * @since XT 1.0 + */ +XTAPI +PMMPFN +MM::Pfn::GetPfnEntry(IN PFN_NUMBER Pfn) +{ + PMMMEMORY_LAYOUT MemoryLayout; + + /* Validate that the PFN is within the range of managed physical memory */ + if(Pfn > HighestPhysicalPage) + { + /* The requested page number is outside the bounds, return NULLPTR */ + return NULLPTR; + } + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Calculate the address of the PFN entry by indexing into the PFN database array and return it */ + return &((PMMPFN)MemoryLayout->PfnDatabase)[Pfn]; +} + +/** + * Increments the global count of available pages. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::IncrementAvailablePages(VOID) +{ + /* Increment the global count of available pages */ + AvailablePages++; +} + +/** + * Links a physical page to the appropriate free lists. + * + * @param PageFrameIndex + * The Page Frame Number (PFN) of the page to link. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkFreePage(IN PFN_NUMBER PageFrameIndex) +{ + ULONG Color; + PMMPFNLIST ListHead; + PFN_NUMBER LastPage; + PMMPFN ColoredPfn, PfnEntry; + PMMCOLOR_TABLES ColorTable; + PMMMEMORY_LAYOUT MemoryLayout; + + /* Get the PFN database entry for the page */ + PfnEntry = GetPfnEntry(PageFrameIndex); + + /* Check if the page is part of a ROM image */ + if(PfnEntry->u3.e1.Rom == 1) + { + /* Mark the page as inactive and clear its links */ + PfnEntry->u1.Flink = 0; + PfnEntry->u3.e1.PageLocation = 0; + + /* Do not free ROM pages */ + return; + } + + /* Check if the page is marked for removal */ + if(PfnEntry->u3.e1.RemovalRequested == 1) + { + /* Update cache attribute to not mapped */ + PfnEntry->u3.e1.CacheAttribute = PfnNotMapped; + + /* Insert the page into the bad page list instead */ + LinkPage(&BadPagesList, PageFrameIndex); + + /* Do not add it to the free list */ + return; + } + + /* Insert the page into the global free list */ + ListHead = &FreePagesList; + ListHead->Total++; + + /* Get the current last page on the list */ + LastPage = ListHead->Blink; + + /* Check if the list is not empty */ + if(LastPage != MAXULONG_PTR) + { + /* Link with the previous last page */ + GetPfnEntry(LastPage)->u1.Flink = PageFrameIndex; + } + else + { + /* Put the page as the first entry */ + ListHead->Flink = PageFrameIndex; + } + + /* Set the page as the new tail of the list */ + ListHead->Blink = PageFrameIndex; + PfnEntry->u1.Flink = MAXULONG_PTR; + PfnEntry->u2.Blink = LastPage; + PfnEntry->u3.e1.CacheAttribute = PfnNotMapped; + PfnEntry->u3.e1.PageLocation = FreePageList; + PfnEntry->u4.AweAllocation = 0; + PfnEntry->u4.InPageError = 0; + PfnEntry->u4.Priority = 3; + + /* Insert the page into the colored free list */ + Color = PageFrameIndex & MM::Colors::GetPagingColorsMask(); + ColorTable = MM::Colors::GetFreePages(FreePageList, Color); + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Check if the colored list is empty */ + if(ColorTable->Flink == MAXULONG_PTR) + { + /* Put the page as the first entry */ + ColorTable->Flink = PageFrameIndex; + PfnEntry->u4.PteFrame = MM_PFN_PTE_FRAME; + } + else + { + /* Get the current last page on the colored list */ + ColoredPfn = (PMMPFN)ColorTable->Blink; + + /* Link with the previous last page */ + MM::Paging::SetPte(&ColoredPfn->OriginalPte, PageFrameIndex); + PfnEntry->u4.PteFrame = ColoredPfn - (PMMPFN)MemoryLayout->PfnDatabase; + } + + /* Set the page as the new tail of the colored list */ + ColorTable->Blink = PfnEntry; + ColorTable->Count++; + MM::Paging::SetPte(&PfnEntry->OriginalPte, MAXULONG_PTR); + + /* Increment number of available pages */ + IncrementAvailablePages(); +} + + +/** + * Links a physical page to the appropriate list. + * + * @param ListHead + * Pointer to the list head. + * + * @param PageFrameIndex + * The Page Frame Number (PFN) of the page to link. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkPage(IN PMMPFNLIST ListHead, + IN PFN_NUMBER PageFrameIndex) +{ + PMMCOLOR_TABLES ColorHead; + MMPAGELISTS ListName; + PMMPFN PageFrame; + + /* Get the memory layout */ + PMMMEMORY_LAYOUT MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Get the PFN database entry for the target page */ + PageFrame = &((PMMPFN)MemoryLayout->PfnDatabase)[PageFrameIndex]; + + /* Get the list name */ + ListName = ListHead->ListName; + + /* Handle pages being linked to the modified or standby lists */ + if(ListName == ModifiedPageList || ListName == StandbyPageList) + { + /* Detect an invalid prototype/transition PTE state */ + if(!MM::Paging::GetPteSoftwarePrototype(&PageFrame->OriginalPte) && + MM::Paging::GetPteSoftwareTransition(&PageFrame->OriginalPte)) + { + /* Crash system due to corrupted PFN/PTE state */ + KE::Crash::PanicEx(0x71, 0x8888, 0, 0, 0); + } + } + + /* Check if the page is ROM */ + if(PageFrame->u3.e1.Rom == 1) + { + /* Link the page to the ROM list */ + ListHead = &RomPagesList; + ListHead->Total++; + + /* Append the page to the end of the ROM list */ + if(ListHead->Blink != MAXULONG_PTR) + { + /* Update the previous tail to point to this page */ + (&((PMMPFN)MemoryLayout->PfnDatabase)[ListHead->Blink])->u1.Flink = PageFrameIndex; + } + else + { + /* Initialize the list if it was empty */ + ListHead->Flink = PageFrameIndex; + } + + /* Update list tail and PFN linkage */ + ListHead->Blink = PageFrameIndex; + PageFrame->u1.Flink = MAXULONG_PTR; + PageFrame->u2.Blink = ListHead->Blink; + PageFrame->u3.e1.PageLocation = ListName; + + /* ROM pages require no further processing */ + return; + } + + /* Account for the page being inserted into the target list */ + ListHead->Total++; + + /* Redirect modified pages to the per-color modified list */ + if(ListHead == &ModifiedPagesList) + { + /* Select the modified list matching the page color */ + ListHead = MM::Colors::GetModifiedPages(PageFrame->u3.e1.PageColor); + ListHead->Total++; + } + else if((PageFrame->u3.e1.RemovalRequested == 1) && (ListName <= StandbyPageList)) + { + /* Undo the insertion into the current list */ + ListHead->Total--; + + /* Preserve the standby location for removed pages */ + if(ListName == StandbyPageList) + { + /* Keep the page marked as standby */ + PageFrame->u3.e1.PageLocation = StandbyPageList; + } + + /* Mark the page as no longer cache-mapped */ + PageFrame->u3.e1.CacheAttribute = PfnNotMapped; + + /* Move the page to the bad page list */ + ListHead = &BadPagesList; + ListHead->Total++; + ListName = BadPageList; + } + + /* Insert zeroed pages at the head of the list */ + if(ListName == ZeroedPageList) + { + /* Link the page as the new list head */ + ListHead->Flink = PageFrameIndex; + + /* Initialize PFN forward and backward links */ + PageFrame->u1.Flink = ListHead->Flink; + PageFrame->u2.Blink = MAXULONG_PTR; + + /* Update the previous head if it exists */ + if(ListHead->Flink != MAXULONG_PTR) + { + /* Fix up the backward link of the old head */ + (&((PMMPFN)MemoryLayout->PfnDatabase)[ListHead->Flink])->u2.Blink = PageFrameIndex; + } + else + { + /* Set the tail if the list was empty */ + ListHead->Blink = PageFrameIndex; + } + } + else + { + /* Append the page to the tail of the list */ + if(ListHead->Blink != MAXULONG_PTR) + { + /* Link the current tail to the new page */ + (&((PMMPFN)MemoryLayout->PfnDatabase)[ListHead->Blink])->u1.Flink = PageFrameIndex; + } + else + { + /* Initialize the list if it was empty */ + ListHead->Flink = PageFrameIndex; + } + + /* Update list tail */ + ListHead->Blink = PageFrameIndex; + + /* Terminate PFN forward link and set backward link */ + PageFrame->u1.Flink = MAXULONG_PTR; + PageFrame->u2.Blink = ListHead->Blink; + } + + /* Record the page's current location */ + PageFrame->u3.e1.PageLocation = ListName; + + /* Handle pages that contribute to the available page count */ + if(ListName <= StandbyPageList) + { + /* Increment the system-wide available page counter */ + MM::Pfn::IncrementAvailablePages(); + + /* Select the free list matching the page color */ + ColorHead = MM::Colors::GetFreePages(ZeroedPageList, PageFrameIndex & MM::Colors::GetPagingColorsMask()); + + /* Store the color list linkage in the original PTE */ + MM::Paging::SetPte(&PageFrame->OriginalPte, ColorHead->Flink); + PageFrame->u4.PteFrame = MM_PFN_PTE_FRAME; + + /* Insert the page into the color free list */ + ColorHead->Flink = PageFrameIndex; + + /* Update the previous head or initialize the tail */ + if(ColorHead->Flink != MAXULONG_PTR) + { + /* Fix up the PTE frame of the previous entry */ + (&((PMMPFN)MemoryLayout->PfnDatabase)[ColorHead->Flink])->u4.PteFrame = PageFrameIndex; + } + else + { + /* Set the tail for an empty color list */ + ColorHead->Blink = (PVOID)PageFrame; + } + + /* Increment the color list page count */ + ColorHead->Count++; + } + else if(ListName == ModifiedPageList) + { + /* Modified page insertion logic not implemented yet */ + UNIMPLEMENTED; + } + else if(ListName == ModifiedReadOnlyPageList) + { + /* Modified read-only page handling not implemented yet */ + UNIMPLEMENTED; + } +} + +/** + * Links a PFN entry to its corresponding PTE and ensures the page table that contains the PTE is resident in memory. + * + * @param PageFrameIndex + * Supplies the index into the PFN database for the page being initialized. + * + * @param PointerPte + * Supplies the pointer to the PTE which maps the physical page. + * + * @param Modified + * Supplies a flag indicating if the page's initial state is modified. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkPfn(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN BOOLEAN Modified) +{ + PMMMEMORY_LAYOUT MemoryLayout; + XTSTATUS Status; + PMMPFN Pfn; + PMMPTE Pte; + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Point the PFN to its PTE */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabase)[PageFrameIndex]; + Pfn->PteAddress = PointerPte; + + /* Check if the page is already mapped and in use */ + if(MM::Paging::PteValid(PointerPte)) + { + /* Clear the original PTE information */ + MM::Paging::SetPte(&Pfn->OriginalPte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE); + } + else + { + /* Page is not resident, so save the PTE contents for later use */ + Pfn->OriginalPte = *PointerPte; + } + + /* Initialize the PFN database entry for this page */ + Pfn->u2.ShareCount = 1; + Pfn->u3.e1.Modified = Modified; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + + /* Get the PDE that maps the page table containing this PTE */ + Pte = MM::Paging::GetPteAddress(PointerPte); + if(!MM::Paging::PteValid(Pte)) + { + /* Check if page table is resident */ + Status = MM::PageFault::CheckPdeForPagedPool(PointerPte); + if(Status != STATUS_SUCCESS) + { + /* Could not make the page table resident, crash system */ + KE::Crash::PanicEx(0x1, + (ULONG_PTR)0x61940, + (ULONG_PTR)PointerPte, + MM::Paging::GetPageFrameNumber(PointerPte), + (ULONG_PTR)MM::Paging::GetPteVirtualAddress(PointerPte)); + } + } + + /* Record the page frame of the page table itself */ + PageFrameIndex = MM::Paging::GetPageFrameNumber(Pte); + Pfn->u4.PteFrame = PageFrameIndex; + + /* Pin the page table in memory by incrementing its PFN share count */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabase)[PageFrameIndex]; + Pfn->u2.ShareCount++; +} + +/** + * Initializes the PFN database entry for a physical page that is used as a page table. + * + * @param PageFrameIndex + * The page frame number of the physical page being used as a page table. + * + * @param PointerPte + * A pointer to the higher-level PTE that maps this page table page. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte) +{ + PMMPFN Pfn; + PMMPDE PointerPde; + PVOID EndAddress; + + /* Retrieve the PFN database entry for the physical page of the page table */ + Pfn = GetPfnEntry(PageFrameIndex); + + /* Calculate the end address of the PFN entry to ensure it's mapped */ + EndAddress = (PUCHAR)(Pfn + 1) - 1; + + /* Validate that the PFN entry corresponds to a valid, active physical page */ + if((PageFrameIndex <= HighestPhysicalPage) && (MM::Pte::AddressValid(Pfn)) && + (MM::Pte::AddressValid(EndAddress)) && (Pfn->u3.e1.PageLocation == ActiveAndValid)) + { + /* Initialize the PFN entry for this page table page */ + Pfn->OriginalPte = *PointerPte; + Pfn->PteAddress = PointerPte; + Pfn->u1.WsIndex = 0; + Pfn->u2.ShareCount++; + Pfn->u3.e1.CacheAttribute = PfnNonCached; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress(PointerPte)); + } + + /* Increment the share count of the parent page table that contains the mapping */ + PointerPde = MM::Paging::GetPdeAddress(MM::Paging::GetPteVirtualAddress(PointerPte)); + Pfn = GetPfnEntry(MM::Paging::GetPageFrameNumber(PointerPde)); + Pfn->u2.ShareCount++; +} + +/** + * Links a PFN entry to its corresponding PTE and ensures the page table that contains the PTE is resident in memory. + * + * @param PageFrameIndex + * Supplies the index into the PFN database for the page being initialized. + * + * @param PointerPte + * Supplies the pointer to the PTE which maps the physical page. + * + * @param ParentFrame + * Supplies the page frame number of the page table that contains the PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkPfnWithParent(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN PFN_NUMBER ParentFrame) +{ + PMMMEMORY_LAYOUT MemoryLayout; + PMMPFN Pfn; + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Point the PFN to its PTE */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabase)[PageFrameIndex]; + Pfn->PteAddress = PointerPte; + + /* Initialize the PTE */ + MM::Paging::SetPte(&Pfn->OriginalPte, 0, MM_PTE_GUARDED); + + /* Initialize the PFN entry */ + Pfn->u2.ShareCount++; + Pfn->u3.e1.CacheAttribute = PfnCached; + Pfn->u3.e1.Modified = TRUE; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount++; + Pfn->u4.InPageError = FALSE; + + /* Check if parent PTE frame exists */ + if(ParentFrame) + { + /* Link the page table to its parent */ + Pfn->u4.PteFrame = ParentFrame; + + /* Get the PFN entry for parent PTE and increment share count */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabase)[ParentFrame]; + Pfn->u2.ShareCount++; + } +} + +/** + * Links a page to the beginning of the appropriate standby or ROM list. + * + * @param PageFrameIndex + * The Page Frame Number (PFN) of the page to link. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ + +XTFASTCALL +VOID +MM::Pfn::LinkStandbyPage(IN PFN_NUMBER PageFrameIndex) +{ + PMMPFN AdjacentPageFrame, CurrentPageFrame; + PMMMEMORY_LAYOUT MemoryLayout; + PFN_NUMBER Flink; + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Get the PFN database entry for the target page */ + CurrentPageFrame = &((PMMPFN)MemoryLayout->PfnDatabase)[PageFrameIndex]; + + /* Check if the page is part of a ROM image */ + if(CurrentPageFrame->u3.e1.Rom == 1) + { + /* Increment the total number of ROM pages */ + RomPagesList.Total++; + + /* If the ROM list is not empty, link the new page */ + if(RomPagesList.Blink != (ULONG_PTR)-1) + { + /* Update the old tail to point to the new page */ + AdjacentPageFrame = &((PMMPFN)MemoryLayout->PfnDatabase)[RomPagesList.Blink]; + AdjacentPageFrame->u1.Flink = PageFrameIndex; + } + else + { + /* Otherwise, this page is now the head of the list */ + RomPagesList.Flink = PageFrameIndex; + } + + /* Set the new page as the tail and update its links */ + RomPagesList.Blink = PageFrameIndex; + CurrentPageFrame->u1.Flink = (ULONG_PTR)-1; + CurrentPageFrame->u2.Blink = RomPagesList.Blink; + CurrentPageFrame->u3.e1.PageLocation = StandbyPageList; + + /* ROM pages require no further processing */ + return; + } + + /* Increment the count of pages on the standby list */ + StandbyPagesList.Total++; + + /* Save the old head and set the current page as the new head */ + Flink = StandbyPagesList.Flink; + StandbyPagesList.Flink = PageFrameIndex; + + /* Point the new head to the old one, marking it as the list front */ + CurrentPageFrame->u1.Flink = Flink; + CurrentPageFrame->u2.Blink = MAXULONG_PTR; + + /* If the ROM list is not empty, link the new page */ + if(Flink != MAXULONG_PTR) + { + /* Update the old head to point to the new page */ + AdjacentPageFrame = &((PMMPFN)MemoryLayout->PfnDatabase)[Flink]; + AdjacentPageFrame->u2.Blink = PageFrameIndex; + } + else + { + /* Otherwise, this page is now the tail of the list */ + StandbyPagesList.Blink = PageFrameIndex; + } + + /* Update the page's location to the standby list */ + CurrentPageFrame->u3.e1.PageLocation = StandbyPageList; + + /* Increment number of available pages */ + IncrementAvailablePages(); +} + +/** + * Processes a memory descriptor and initializes the corresponding PFN database entries. + * + * @param BasePage + * The starting physical page number of the memory run. + * + * @param PageCount + * The number of pages in the memory run. + * + * @param MemoryType + * The type of memory as reported by the bootloader (e.g., free, ROM, in-use). + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage, + IN PFN_NUMBER PageCount, + IN LOADER_MEMORY_TYPE MemoryType) +{ + PVOID VirtualRangeStart, VirtualRangeEnd; + PFN_NUMBER PageNumber; + PMMPDE PointerPde; + PMMPFN Pfn; + + /* Check if the memory descriptor describes a free memory region */ + if(MM::Manager::VerifyMemoryTypeFree(MemoryType)) + { + /* Iterate over each page in this free memory run */ + for(PageNumber = 0; PageNumber < PageCount; PageNumber++) + { + /* Get the PFN entry for the current page and ensure it is not referenced */ + Pfn = GetPfnEntry(BasePage + PageNumber); + if(Pfn->u3.e2.ReferenceCount == 0) + { + /* Add the page to the free list to make it available for allocation */ + LinkFreePage(BasePage + PageNumber); + } + } + } + else + { + /* Calculate the virtual address range for this physical memory region */ + VirtualRangeStart = (PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)); + VirtualRangeEnd = (PVOID)(KSEG0_BASE + ((BasePage + PageCount) << MM_PAGE_SHIFT)); + + /* Handle all other (non-free) memory types */ + switch(MemoryType) + { + case LoaderBad: + /* This memory is marked as bad and should not be used */ + for(PageNumber = 0; PageNumber < PageCount; PageNumber++) + { + /* Link the page to the bad pages list */ + LinkPage(&BadPagesList, BasePage + PageNumber); + } + break; + case LoaderXIPRom: + /* Get the page directory entry for the current page */ + PointerPde = MM::Paging::GetPdeAddress(VirtualRangeStart); + + /* Initialize the page directory entries covering this memory range */ + InitializePageDirectory(PointerPde, MM::Paging::GetPdeAddress(VirtualRangeEnd)); + + /* This memory range contains Read-Only Memory (ROM) */ + for(PageNumber = 0; PageNumber < PageCount; PageNumber++) + { + /* Get the PFN entry for the current ROM page */ + Pfn = GetPfnEntry(BasePage + PageNumber); + + /* Ensure that the page is not already in-use */ + if(Pfn->u3.e2.ReferenceCount == 0) + { + /* Initialize the PFN entry to represent a ROM page */ + Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualRangeStart); + Pfn->u1.Flink = 0; + Pfn->u2.ShareCount = 0; + Pfn->u3.e1.CacheAttribute = PfnCached; + Pfn->u3.e1.PageLocation = 0; + Pfn->u3.e1.PrototypePte = 1; + Pfn->u3.e1.Rom = 1; + Pfn->u3.e2.ReferenceCount = 0; + Pfn->u4.InPageError = 0; + Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde); + } + } + break; + default: + /* Get the page directory entry for the current page */ + PointerPde = MM::Paging::GetPdeAddress(VirtualRangeStart); + + /* Initialize the page directory entries covering this memory range */ + InitializePageDirectory(PointerPde, MM::Paging::GetPdeAddress(VirtualRangeEnd)); + + /* All other types are considered in-use (ie, by the kernel, ACPI, etc) */ + for(PageNumber = 0; PageNumber < PageCount; PageNumber++) + { + /* Get the PFN entry for the current in-use page */ + Pfn = GetPfnEntry(BasePage + PageNumber); + + /* Ensure that the page is not already in-use */ + if(Pfn->u3.e2.ReferenceCount == 0) + { + /* Initialize the PFN entry to represent an in-use page and prevent it from being allocated */ + Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualRangeStart); + Pfn->u2.ShareCount++; + Pfn->u3.e1.CacheAttribute = PfnCached; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde); + } + } + break; + } + } +} + +/** + * Scans memory descriptors provided by the boot loader. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::ScanMemoryDescriptors(VOID) +{ + PLIST_ENTRY LoaderMemoryDescriptors, MemoryMappings; + PLOADER_MEMORY_DESCRIPTOR MemoryDescriptor; + PFN_NUMBER FreePages; + + /* Initially, set number of free pages to 0 */ + FreePages = 0; + + /* Get the list head of memory descriptors */ + LoaderMemoryDescriptors = KE::BootInformation::GetMemoryDescriptors(); + + /* Iterate through the memory descriptors */ + MemoryMappings = LoaderMemoryDescriptors->Flink; + while(MemoryMappings != LoaderMemoryDescriptors) + { + /* Get the memory descriptor */ + MemoryDescriptor = CONTAIN_RECORD(MemoryMappings, LOADER_MEMORY_DESCRIPTOR, ListEntry); + + /* Skip invisible or hardware cached memory regions */ + if(MM::Manager::VerifyMemoryTypeInvisible(MemoryDescriptor->MemoryType) || + (MemoryDescriptor->MemoryType == LoaderHardwareCachedMemory)) + { + /* Move to the next descriptor and skip further processing */ + MemoryMappings = MemoryMappings->Flink; + continue; + } + + /* Count the number of physical pages, excluding bad memory */ + if(MemoryDescriptor->MemoryType != LoaderBad) + { + /* Add the pages from this descriptor to the total count */ + NumberOfPhysicalPages += MemoryDescriptor->PageCount; + } + + /* Check if this physical page is the lowest one yet */ + if(MemoryDescriptor->BasePage < LowestPhysicalPage) + { + /* Update the lowest physical page number found so far */ + LowestPhysicalPage = MemoryDescriptor->BasePage; + } + + /* Check if this physical page is the highest one yet */ + if((MemoryDescriptor->BasePage + MemoryDescriptor->PageCount) > HighestPhysicalPage) + { + /* Update the highest physical page number found so far */ + HighestPhysicalPage = (MemoryDescriptor->BasePage + MemoryDescriptor->PageCount) - 1; + } + + /* Identify the largest block of free memory */ + if(MM::Manager::VerifyMemoryTypeFree(MemoryDescriptor->MemoryType)) + { + /* Check if this free memory block is the largest one yet */ + if(MemoryDescriptor->PageCount >= FreePages) + { + /* Update the largest free block size and save the descriptor */ + FreePages = MemoryDescriptor->PageCount; + FreeDescriptor = MemoryDescriptor; + } + } + + /* Get next memory descriptor */ + MemoryMappings = MemoryMappings->Flink; + } + + /* Ensure a free memory descriptor was found */ + if(!FreeDescriptor) + { + /* No free memory available to bootstrap the system */ + KE::Crash::Panic(0); + } + + /* Save a copy of the original free descriptor before it gets modified */ + RTL::Memory::CopyMemory(&OriginalFreeDescriptor, FreeDescriptor, sizeof(LOADER_MEMORY_DESCRIPTOR)); +} + +/** + * Unlinks a physical page from its corresponding list. + * + * @param PageIndex + * The Page Frame Number (PFN) of the page to unlink. + * + * @param Color + * The color of the page, used to find the correct colored list. + * + * @return This routine returns the PFN of the page that was unlinked. + */ +XTAPI +PFN_NUMBER +MM::Pfn::UnlinkFreePage(IN PFN_NUMBER PageFrameIndex, + IN ULONG Color) +{ + PMMPFN Pfn; + PMMPFNLIST PfnList; + MMPAGELISTS PageList; + ULONG NodeColor; + PMMCOLOR_TABLES ColorTable; + PFN_NUMBER NextPage, PrevPage; + + /* Get the PFN database entry for the target page */ + Pfn = GetPfnEntry(PageFrameIndex); + + /* Identify which list the page belongs to (FreePageList or ZeroedPageList) */ + PfnList = PageLocationList[Pfn->u3.e1.PageLocation]; + PageList = PfnList->ListName; + + /* Update the forward link of the previous page */ + if(Pfn->u2.Blink != MAXULONG_PTR) + { + /* The page is not the head of the list; update the previous page's Flink */ + GetPfnEntry(Pfn->u2.Blink)->u1.Flink = Pfn->u1.Flink; + } + else + { + /* This is the first page in the list; update the list head's Flink */ + PfnList->Flink = Pfn->u1.Flink; + } + + /* Update the backward link of the next page */ + if(Pfn->u1.Flink != MAXULONG_PTR) + { + /* The page is not the tail of the list; update the next page's Blink */ + GetPfnEntry(Pfn->u1.Flink)->u2.Blink = Pfn->u2.Blink; + } + else + { + /* This is the last page in the list; update the list head's Blink */ + PfnList->Blink = Pfn->u2.Blink; + } + + /* Get the first page on the color list */ + ColorTable = MM::Colors::GetFreePages(PageList, Color); + NodeColor = Pfn->u3.e1.PageColor; + PrevPage = Pfn->u4.PteFrame; + NextPage = MM::Paging::GetPte(&Pfn->OriginalPte); + + /* Decrement the count of pages for this specific color and total page count for this list */ + ColorTable->Count--; + PfnList->Total--; + + /* Update the forward link of the previous colored page */ + if(PrevPage != MM_PFN_PTE_FRAME) + { + /* This is not the first page; update the previous page's Flink */ + MM::Paging::SetPte(&GetPfnEntry(PrevPage)->OriginalPte, NextPage); + } + else + { + /* This was the first page; update the color table's Flink */ + ColorTable->Flink = NextPage; + } + + /* Update the backward link of the next colored page */ + if (NextPage != MAXULONG_PTR) + { + /* This is not the last page; update the next page's Blink */ + GetPfnEntry(NextPage)->u4.PteFrame = PrevPage; + } + else + { + /* This was the last page; update the color table's Blink */ + ColorTable->Blink = (PVOID)PrevPage; + } + + /* Clear the list pointers and flags, but preserve the color and cache attributes */ + Pfn->u1.Flink = 0; + Pfn->u2.Blink = 0; + Pfn->u3.e1.CacheAttribute = PfnNotMapped; + Pfn->u3.e1.PageColor = NodeColor; + Pfn->u3.e2.ShortFlags = 0; + + /* Decrement the global count of available pages */ + DecrementAvailablePages(); + + /* Return the page that was just unlinked */ + return PageFrameIndex; +} diff --git a/xtoskrnl/mm/pte.cc b/xtoskrnl/mm/pte.cc new file mode 100644 index 0000000..6a47c48 --- /dev/null +++ b/xtoskrnl/mm/pte.cc @@ -0,0 +1,546 @@ +/** + * PROJECT: ExectOS + * COPYRIGHT: See COPYING.md in the top level directory + * FILE: xtoskrnl/mm/pte.cc + * DESCRIPTION: Page Table Entry (PTE) support + * DEVELOPERS: Aiken Harris + */ + +#include + + +/** + * Finds a free cluster of system PTEs that can satisfy a given size. + * + * @param NumberOfPtes + * The number of contiguous PTEs required. + * + * @param SystemPtePoolType + * Specifies the system PTE pool to search within. + * + * @param FoundCluster + * On success, receives a pointer to the first PTE of the found cluster. + * + * @param PreviousClusterNode + * On success, receives a pointer to the list node that precedes the found cluster. + * + * @return This routine returns TRUE if a suitable cluster was found, FALSE otherwise. + * + * @since XT 1.0 + */ +XTAPI +BOOLEAN +MM::Pte::FindFreeCluster(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType, + OUT PMMPTE *FoundCluster, + OUT PMMPTE *PreviousClusterNode) +{ + PMMPTE CurrentCluster; + PMMPTE PreviousNode = &FirstSystemFreePte[SystemPtePoolType]; + ULONG ClusterSize; + + /* Find a free PTE cluster large enough for the request */ + while(MM::Paging::GetNextEntry(PreviousNode) != MAXULONG) + { + /* Retrieve the cluster and its size */ + CurrentCluster = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousNode)); + ClusterSize = GetClusterSize(CurrentCluster); + + /* Check if this cluster is large enough */ + if(NumberOfPtes <= ClusterSize) + { + /* Found a suitable cluster */ + *FoundCluster = CurrentCluster; + *PreviousClusterNode = PreviousNode; + return TRUE; + } + + /* This cluster is too small, check the next one */ + PreviousNode = CurrentCluster; + } + + /* No suitable cluster was found */ + return FALSE; +} + +/** + * Decodes and returns the size of a free PTE cluster. + * + * @param Pte + * Supplies a pointer to the first PTE of the free cluster to inspect. + * + * @return This routine returns the total number of contiguous PTEs in the free cluster. + * + * @since XT 1.0 + */ +XTAPI +ULONG +MM::Pte::GetClusterSize(IN PMMPTE Pte) +{ + /* A special flag in the first PTE indicates a free cluster of size one */ + if(MM::Paging::GetOneEntry(Pte)) + { + /* Flag is set, so the cluster size is 1 by definition */ + return 1; + } + + /* For larger clusters, the size is encoded in the second PTE of the block */ + Pte = MM::Paging::GetNextPte(Pte); + return MM::Paging::GetNextEntry(Pte); +} + +/** + * Calculates the number of Page Table Entries (PTEs) that fit within a single page. + * + * @return This routine returns the number of PTEs per page. + * + * @since XT 1.0 + */ +XTAPI +PFN_COUNT +MM::Pte::GetPtesPerPage(VOID) +{ + /* Calculate and return the number of PTEs per page */ + return MM_PAGE_SIZE / MM::Paging::GetPteSize(); +} + +/** + * Returns a pointer to the valid PTE. + * + * @return This routine returns a pointer to the valid PTE. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Pte::GetValidPte() +{ + /* Return a pointer to the valid PTE */ + return &ValidPte; +} + +/** + * Initializes the system's PTE. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::InitializeSystemPte(VOID) +{ + /* Initialize the PTE template */ + MM::Paging::SetPte(&ValidPte, MM_PTE_VALID | MM_PTE_EXECUTE_READWRITE | MM_PTE_DIRTY | MM_PTE_ACCESSED); +} + +/** + * Formats a range of PTEs into a freelist-based pool for system allocations. + * + * @param StartingPte + * Supplies a pointer to the start of the PTE range to be formatted. + * + * @param NumberOfPtes + * Supplies the total number of PTEs in the contiguous range. + * + * @param PoolType + * The system PTE pool type that this range will be used for. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::InitializeSystemPtePool(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE PoolType) +{ + /* Set the system PTE base address */ + SystemPteBase = GetSystemPteBaseAddress(); + + /* Record the boundaries of this new PTE pool */ + SystemPtesStart[PoolType] = StartingPte; + SystemPtesEnd[PoolType] = MM::Paging::AdvancePte(StartingPte, NumberOfPtes - 1); + + /* Zero the memory for the new PTE pool before use */ + RTL::Memory::ZeroMemory(StartingPte, NumberOfPtes * MM::Paging::GetPteSize()); + + /* Build the free list head to point to the start of the pool */ + MM::Paging::SetNextEntry(StartingPte, MAXULONG); + MM::Paging::ClearPte(&FirstSystemFreePte[PoolType]); + MM::Paging::SetNextEntry(&FirstSystemFreePte[PoolType], MM::Paging::GetPteDistance(StartingPte, SystemPteBase)); + + /* Use the second PTE slot to store the total size of this pool */ + StartingPte = MM::Paging::GetNextPte(StartingPte); + MM::Paging::ClearPte(StartingPte); + MM::Paging::SetNextEntry(StartingPte, NumberOfPtes); + + /* Record the total number of free PTEs in this pool */ + TotalSystemFreePtes[PoolType] = NumberOfPtes; +} + +/** + * Sets up the entire system PTE address space. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::InitializeSystemPteSpace(VOID) +{ + PMMPTE PointerPte; + PMMPTE FirstZeroingPte; + PMMMEMORY_LAYOUT MemoryLayout; + ULONGLONG NonPagedSystemPoolEnd; + + /* Retrieve the system's memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + NonPagedSystemPoolEnd = ((ULONGLONG)MemoryLayout->NonPagedSystemPoolStart + + MM::Manager::GetNumberOfSystemPtes() * MM_PAGE_SIZE); + + /* Map the page table hierarchy for the entire system PTE space */ + MM::Pte::MapPPE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)NonPagedSystemPoolEnd, &ValidPte); + MM::Pte::MapPDE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)NonPagedSystemPoolEnd, &ValidPte); + + /* Format the main block of system PTEs into a free list pool */ + PointerPte = MM::Paging::GetPteAddress(MemoryLayout->NonPagedSystemPoolStart); + + InitializeSystemPtePool(PointerPte, MM::Manager::GetNumberOfSystemPtes(), SystemPteSpace); + + /* Reserve and zero a dedicated block of system PTEs */ + FirstZeroingPte = ReserveSystemPtes(MM_RESERVED_ZERO_PTES + 1, SystemPteSpace); + RTL::Memory::ZeroMemory(FirstZeroingPte, (MM_RESERVED_ZERO_PTES + 1) * MM::Paging::GetPteSize()); + + /* Use the first PTE of this block as a counter for available zeroing PTEs */ + MM::Paging::SetPte(FirstZeroingPte, MM_RESERVED_ZERO_PTES, 0); +} + +/** + * Maps a range of virtual addresses at the PDE (Page Directory Entry) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplatePde + * A template PDE to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapPDE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPDE TemplatePde) +{ + PMMPDE EndSpace, PointerPde; + + /* Get PDE addresses */ + PointerPde = MM::Paging::GetPdeAddress(StartAddress); + EndSpace = MM::Paging::GetPdeAddress(EndAddress); + + /* Iterate over all PDEs */ + while(PointerPde <= EndSpace) + { + /* Check if PDE is already mapped */ + if(!MM::Paging::PteValid(PointerPde)) + { + /* Map PDE */ + MM::Paging::SetPte(TemplatePde, MM::Pfn::AllocateBootstrapPages(1), 0); + MM::Paging::WritePte(PointerPde, *TemplatePde); + + /* Clear the page table */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPde), MM_PAGE_SIZE); + } + + /* Get next table entry */ + PointerPde = MM::Paging::GetNextPte(PointerPde); + } +} + +/** + * Maps a range of virtual addresses at the PTE (Page Table Entry) level. + * + * @param StartAddress + * The beginning of the virtual address range to map. + * + * @param EndAddress + * The end of the virtual address range to map. + * + * @param TemplatePte + * A template PTE to use for creating new entries. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::MapPTE(IN PVOID StartAddress, + IN PVOID EndAddress, + IN PMMPTE TemplatePte) +{ + PMMPTE EndSpace, PointerPte; + + /* Get PTE addresses */ + PointerPte = MM::Paging::GetPteAddress(StartAddress); + EndSpace = MM::Paging::GetPteAddress(EndAddress); + + /* Iterate over all PTEs */ + while(PointerPte <= EndSpace) + { + /* Check if PTE is already mapped */ + if(!MM::Paging::PteValid(PointerPte)) + { + /* Map PTE */ + MM::Paging::SetPte(TemplatePte, MM::Pfn::AllocateBootstrapPages(1), 0); + MM::Paging::WritePte(PointerPte, *TemplatePte); + + /* Clear the page table */ + RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPte), MM_PAGE_SIZE); + } + + /* Get next table entry */ + PointerPte = MM::Paging::GetNextPte(PointerPte); + } +} + + +/** + * Releases a block of system PTEs into a specified pool. + * + * @param StartingPte + * A pointer to the first PTE to release. + * + * @param NumberOfPtes + * The number of contiguous PTEs to release. + * + * @param SystemPtePoolType + * Specifies the system PTE pool to release into. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pte::ReleaseSystemPtes(IN PMMPTE StartingPte, + IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType) +{ + PMMPTE NextPte, PreviousPte, ReleasedPte; + ULONG ClusterSize; + + /* Clear the PTEs before releasing them */ + RtlZeroMemory(StartingPte, NumberOfPtes * MM::Paging::GetPteSize()); + + /* Raise runlevel and acquire lock to protect the PTE pool */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Increment the total number of available PTEs in this pool */ + TotalSystemFreePtes[SystemPtePoolType] += NumberOfPtes; + + /* Start at the head of the free list for this pool */ + PreviousPte = &FirstSystemFreePte[SystemPtePoolType]; + ReleasedPte = NULLPTR; + + /* Iterate through the free list to find adjacent blocks */ + while(MM::Paging::GetNextEntry(PreviousPte) != MAXULONG) + { + /* Get the next free cluster to check its size */ + NextPte = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousPte)); + ClusterSize = GetClusterSize(NextPte); + + /* Check if the released block is adjacent to the current free block */ + if((MM::Paging::AdvancePte(NextPte, ClusterSize) == StartingPte) || + (MM::Paging::AdvancePte(StartingPte, NumberOfPtes) == NextPte)) + { + /* Merge the blocks by adding their sizes */ + NumberOfPtes += ClusterSize; + + /* Check if the current free block is before the released block */ + if(NextPte < StartingPte) + { + /* The new merged block starts at the current free block's address */ + StartingPte = NextPte; + } + + /* Unlink the current free block as it is being merged */ + MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetNextEntry(NextPte)); + + /* Check if the block represents more than one PTE */ + if(!MM::Paging::GetOneEntry(NextPte)) + { + /* Clear block header and move to the size PTE */ + MM::Paging::ClearPte(NextPte); + NextPte = MM::Paging::GetNextPte(NextPte); + } + + /* Clear the merged block */ + MM::Paging::ClearPte(NextPte); + + /* Reset insertion point since block size/address changed due to merge */ + ReleasedPte = NULLPTR; + } + else + { + /* Select the first free block large enough as insertion point */ + if((ReleasedPte == NULLPTR) && (NumberOfPtes <= ClusterSize)) + { + /* Mark this as the insertion point */ + ReleasedPte = PreviousPte; + } + + /* Advance to the next free block */ + PreviousPte = NextPte; + } + } + + /* Check if there is only one PTE to release */ + if(NumberOfPtes == 1) + { + /* Mark it as a single-PTE block */ + MM::Paging::SetOneEntry(StartingPte, 1); + } + else + { + /* Otherwise, mark it as a multi-PTE block */ + MM::Paging::SetOneEntry(StartingPte, 0); + + /* The next PTE stores the size of the block */ + NextPte = MM::Paging::GetNextPte(StartingPte); + MM::Paging::SetNextEntry(NextPte, NumberOfPtes); + } + + /* Check if no suitable insertion point was found */ + if(ReleasedPte == NULLPTR) + { + /* Insert at the end of the list */ + ReleasedPte = PreviousPte; + } + + /* Link the new block into the free list */ + MM::Paging::SetNextEntry(StartingPte, MM::Paging::GetNextEntry(ReleasedPte)); + MM::Paging::SetNextEntry(ReleasedPte, MM::Paging::GetPteDistance(StartingPte, SystemPteBase)); +} + +/** + * Reserves a contiguous block of system PTEs from a specified pool. + * + * @param NumberOfPtes + * The number of contiguous PTEs to reserve. + * + * @param SystemPtePoolType + * Specifies the system PTE pool from which to allocate. + * + * @return This routine returns a pointer to the beginning of the reserved block, + * or NULLPTR if not enough contiguous PTEs are available. + * + * @since XT 1.0 + */ +XTAPI +PMMPTE +MM::Pte::ReserveSystemPtes(IN PFN_COUNT NumberOfPtes, + IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType) +{ + PMMPTE NextPte, PreviousPte, ReservedPte; + ULONG ClusterSize; + + /* Raise runlevel and acquire lock to protect the PTE pool */ + KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL); + KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock); + + /* Find a free PTE cluster large enough for the request */ + if(!FindFreeCluster(NumberOfPtes, SystemPtePoolType, &NextPte, &PreviousPte)) + { + /* Out of system PTEs for this pool, return NULLPTR */ + return NULLPTR; + } + + /* We have the cluster, now get its size for the allocation logic below */ + ClusterSize = GetClusterSize(NextPte); + + /* Unlink the found cluster from the free list for processing */ + MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetNextEntry(NextPte)); + + /* Handle the allocation based on whether the cluster size is an exact match */ + if(ClusterSize == NumberOfPtes) + { + /* Exact match, allocate the entire cluster */ + ReservedPte = NextPte; + + /* Handle metadata cleanup for a single-PTE cluster */ + if(MM::Paging::GetOneEntry(NextPte)) + { + /* Clear the PTE that held the list metadata */ + MM::Paging::ClearPte(NextPte); + NextPte = MM::Paging::GetNextPte(NextPte); + } + + /* Clear the PTE that held the cluster size */ + MM::Paging::ClearPte(NextPte); + } + else + { + /* Cluster is larger than needed, so it will be split */ + ClusterSize -= NumberOfPtes; + ReservedPte = MM::Paging::AdvancePte(NextPte, ClusterSize); + + /* Update metadata for the new, smaller leftover cluster */ + if(ClusterSize == 1) + { + /* The leftover fragment is a single PTE */ + MM::Paging::SetOneEntry(NextPte, 1); + MM::Paging::ClearPte(ReservedPte); + } + else + { + /* The leftover fragment is larger than one PTE */ + NextPte = MM::Paging::GetNextPte(NextPte); + MM::Paging::SetNextEntry(NextPte, ClusterSize); + } + + /* Find the correct sorted position to re-insert the leftover fragment */ + PreviousPte = &FirstSystemFreePte[SystemPtePoolType]; + while(MM::Paging::GetNextEntry(PreviousPte) != MAXULONG) + { + /* Get the next free cluster to check its size */ + NextPte = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousPte)); + + /* Check if the leftover fragment should be inserted here */ + if(ClusterSize <= GetClusterSize(NextPte)) + { + /* Found the correct sorted position */ + break; + } + + /* Advance to the next entry */ + PreviousPte = NextPte; + } + + /* Get a pointer to the start of the leftover fragment */ + NextPte = MM::Paging::AdvancePte(ReservedPte, -ClusterSize); + + /* Insert the leftover fragment back into the free list at its sorted position */ + MM::Paging::SetNextEntry(NextPte, MM::Paging::GetNextEntry(PreviousPte)); + MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetPteDistance(NextPte, SystemPteBase)); + } + + /* Decrement the total number of available PTEs in this pool */ + TotalSystemFreePtes[SystemPtePoolType] -= NumberOfPtes; + + /* Flush the TLB to ensure address translation consistency */ + AR::CpuFunc::FlushTlb(); + + /* Return a pointer to the start of the reserved PTE block */ + return ReservedPte; +}