Remove explicit identity mapping for internal page tables

boot/xtldr/arch/amd64/memory.cc

@@ -44,6 +44,14 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         return Status;
     }
 
+    /* Add new memory mapping for the page map itself */
+    Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Memory mapping failure */
+        return Status;
+    }
+
     /* Assign and zero-fill memory used by page mappings */
     PageMap->PtePointer = (PVOID)(UINT_PTR)Address;
     RTL::Memory::ZeroMemory(PageMap->PtePointer, EFI_PAGE_SIZE);
@@ -57,7 +65,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Map the trampoline code area */
-    Status = MapVirtualMemory(PageMap, (PVOID)MM_TRAMPOLINE_ADDRESS,(PVOID)MM_TRAMPOLINE_ADDRESS,
+    Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS,MM_TRAMPOLINE_ADDRESS,
                               1, LoaderFirmwareTemporary);
     if(Status != STATUS_EFI_SUCCESS)
     {
@@ -74,7 +82,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         ModuleInfo = CONTAIN_RECORD(ModulesListEntry, XTBL_MODULE_INFO, Flink);
 
         /* Map module code */
-        Status = MapVirtualMemory(PageMap, ModuleInfo->ModuleBase, ModuleInfo->ModuleBase,
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)ModuleInfo->ModuleBase, (ULONGLONG)ModuleInfo->ModuleBase,
                                   EFI_SIZE_TO_PAGES(ModuleInfo->ModuleSize), LoaderFirmwareTemporary);
 
         /* Check if mapping succeeded */
@@ -95,7 +103,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
     if(LoaderBase && LoaderSize)
     {
         /* Map boot loader code as well */
-        Status = MapVirtualMemory(PageMap, LoaderBase, LoaderBase,
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)LoaderBase, (ULONGLONG)LoaderBase,
                                   EFI_SIZE_TO_PAGES(LoaderSize), LoaderFirmwareTemporary);
         if(Status != STATUS_EFI_SUCCESS)
         {
@@ -193,7 +201,7 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap,
         }
 
         /* Add new memory mapping */
-        Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)(UINT_PTR)Address, 1, LoaderMemoryData);
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
         if(Status != STATUS_EFI_SUCCESS)
         {
             /* Memory mapping failure */
@@ -239,9 +247,9 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap,
 XTCDECL
 EFI_STATUS
 Memory::MapPage(IN PXTBL_PAGE_MAPPING PageMap,
-                IN ULONG_PTR VirtualAddress,
+                IN ULONGLONG VirtualAddress,
-                IN ULONG_PTR PhysicalAddress,
+                IN ULONGLONG PhysicalAddress,
-                IN ULONG NumberOfPages)
+                IN ULONGLONG NumberOfPages)
 {
     PVOID Pml1, Pml2, Pml3, Pml4, Pml5;
     SIZE_T Pml1Entry, Pml2Entry, Pml3Entry, Pml4Entry, Pml5Entry;

boot/xtldr/arch/i686/memory.cc

@@ -93,8 +93,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Map the trampoline code area */
-    Status = MapVirtualMemory(PageMap, (PVOID)MM_TRAMPOLINE_ADDRESS,(PVOID)MM_TRAMPOLINE_ADDRESS,
+    Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS, MM_TRAMPOLINE_ADDRESS, 1, LoaderFirmwareTemporary);
-                              1, LoaderFirmwareTemporary);
     if(Status != STATUS_EFI_SUCCESS)
     {
         /* Mapping trampoline code failed */
@@ -110,7 +109,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         ModuleInfo = CONTAIN_RECORD(ModulesListEntry, XTBL_MODULE_INFO, Flink);
 
         /* Map module code */
-        Status = MapVirtualMemory(PageMap, ModuleInfo->ModuleBase, ModuleInfo->ModuleBase,
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)ModuleInfo->ModuleBase, (ULONGLONG)ModuleInfo->ModuleBase,
                                   EFI_SIZE_TO_PAGES(ModuleInfo->ModuleSize), LoaderFirmwareTemporary);
 
         /* Check if mapping succeeded */
@@ -131,7 +130,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
     if(LoaderBase && LoaderSize)
     {
         /* Map boot loader code as well */
-        Status = MapVirtualMemory(PageMap, LoaderBase, LoaderBase,
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)LoaderBase, (ULONGLONG)LoaderBase,
                                   EFI_SIZE_TO_PAGES(LoaderSize), LoaderFirmwareTemporary);
         if(Status != STATUS_EFI_SUCCESS)
         {
@@ -157,8 +156,9 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         if(Mapping->VirtualAddress)
         {
             /* Dump memory mapping */
-            Debug::Print(L"   Type=%02lu, PhysicalBase=%.8P, VirtualBase=%.8P, Pages=%llu\n", Mapping->MemoryType,
+            Debug::Print(L"   Type=%02lu, PhysicalBase=0x%.8llX, VirtualBase=0x%.8llX, Pages=%llu\n",
-                         Mapping->PhysicalAddress, Mapping->VirtualAddress, Mapping->NumberOfPages);
+                         Mapping->MemoryType, Mapping->PhysicalAddress,
+                         Mapping->VirtualAddress, Mapping->NumberOfPages);
 
             /* Map memory */
             Status = MapPage(PageMap, (UINT_PTR)Mapping->VirtualAddress,
@@ -252,7 +252,7 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap,
         }
 
         /* Add new memory mapping */
-        Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)(UINT_PTR)Address, 1, LoaderMemoryData);
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
         if(Status != STATUS_EFI_SUCCESS)
         {
             /* Memory mapping failure */
@@ -313,11 +313,11 @@ Memory::GetNextPageTable(IN PXTBL_PAGE_MAPPING PageMap,
 XTCDECL
 EFI_STATUS
 Memory::MapPage(IN PXTBL_PAGE_MAPPING PageMap,
-                IN ULONG_PTR VirtualAddress,
+                IN ULONGLONG VirtualAddress,
-                IN ULONG_PTR PhysicalAddress,
+                IN ULONGLONG PhysicalAddress,
-                IN ULONG NumberOfPages)
+                IN ULONGLONG NumberOfPages)
 {
-    SIZE_T PageFrameNumber;
+    ULONGLONG PageFrameNumber;
     PVOID Pml1, Pml2, Pml3;
     SIZE_T Pml1Entry, Pml2Entry, Pml3Entry;
     PHARDWARE_LEGACY_PTE LegacyPmlTable;

boot/xtldr/includes/xtldr.hh

@@ -180,12 +180,12 @@ class Memory
                                                IN OUT PVOID *MemoryMapAddress,
                                                IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine);
         STATIC XTCDECL EFI_STATUS MapPage(IN PXTBL_PAGE_MAPPING PageMap,
-                                          IN ULONG_PTR VirtualAddress,
+                                          IN ULONGLONG VirtualAddress,
-                                          IN ULONG_PTR PhysicalAddress,
+                                          IN ULONGLONG PhysicalAddress,
-                                          IN ULONG NumberOfPages);
+                                          IN ULONGLONG NumberOfPages);
         STATIC XTCDECL EFI_STATUS MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
-                                                   IN PVOID VirtualAddress,
+                                                   IN ULONGLONG VirtualAddress,
-                                                   IN PVOID PhysicalAddress,
+                                                   IN ULONGLONG PhysicalAddress,
                                                    IN ULONGLONG NumberOfPages,
                                                    IN LOADER_MEMORY_TYPE MemoryType);
         STATIC XTCDECL PVOID PhysicalAddressToVirtual(IN PVOID PhysicalAddress,

boot/xtldr/memory.cc

@@ -330,16 +330,16 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
                      IN OUT PVOID *MemoryMapAddress,
                      IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine)
 {
+    ULONGLONG MaxAddress, VirtualAddress;
     PEFI_MEMORY_DESCRIPTOR Descriptor;
     LOADER_MEMORY_TYPE MemoryType;
     PEFI_MEMORY_MAP MemoryMap;
     SIZE_T DescriptorCount;
-    PUCHAR VirtualAddress;
     EFI_STATUS Status;
     SIZE_T Index;
 
     /* Set virtual address as specified in argument */
-    VirtualAddress = (PUCHAR)*MemoryMapAddress;
+    VirtualAddress = (ULONGLONG)*MemoryMapAddress;
 
     /* Check if custom memory type routine is specified */
     if(GetMemoryTypeRoutine == NULLPTR)
@@ -367,8 +367,37 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     /* Iterate through all descriptors from the memory map */
     for(Index = 0; Index < DescriptorCount; Index++)
     {
-        /* Make sure descriptor does not start beyond lowest physical page */
+        /* Check page map level */
-        if(Descriptor->PhysicalStart <= MAXUINT_PTR)
+        if(PageMap->PageMapLevel == 2)
+        {
+            /* Limit physical address to 4GB in legacy mode */
+            MaxAddress = 0xFFFFFFFF;
+        }
+        else if(PageMap->PageMapLevel == 3)
+        {
+            /* Limit physical address to 64GB in PAE mode */
+            MaxAddress = 0xFFFFFFFFFULL;
+        }
+
+        /* Check page map level */
+        if(PageMap->PageMapLevel == 2 || PageMap->PageMapLevel == 3)
+        {
+            /* Check if physical address starts beyond limit */
+            if(Descriptor->PhysicalStart >= MaxAddress)
+            {
+                /* Go to the next descriptor */
+                Descriptor = (PEFI_MEMORY_DESCRIPTOR)((PUCHAR)Descriptor + MemoryMap->DescriptorSize);
+                continue;
+            }
+
+            /* Check if memory descriptor exceeds the lowest physical page */
+            if(Descriptor->PhysicalStart + (Descriptor->NumberOfPages << EFI_PAGE_SHIFT) > MaxAddress)
+            {
+                /* Truncate memory descriptor to the lowest supported physical page */
+                Descriptor->NumberOfPages = (MaxAddress - Descriptor->PhysicalStart) >> EFI_PAGE_SHIFT;
+            }
+        }
+
         {
             /* Skip EFI reserved memory */
             if(Descriptor->Type == EfiReservedMemoryType)
@@ -378,25 +407,6 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
                 continue;
             }
 
-            /* Check if preparing page map level 2 (non-PAE i686) */
-            if(PageMap->PageMapLevel == 2)
-            {
-                /* Check if physical address starts beyond 4GB */
-                if(Descriptor->PhysicalStart > 0xFFFFFFFF)
-                {
-                    /* Go to the next descriptor */
-                    Descriptor = (PEFI_MEMORY_DESCRIPTOR)((PUCHAR)Descriptor + MemoryMap->DescriptorSize);
-                    continue;
-                }
-
-                /* Check if memory descriptor exceeds the lowest physical page */
-                if(Descriptor->PhysicalStart + (Descriptor->NumberOfPages << EFI_PAGE_SHIFT) > MAXULONG)
-                {
-                    /* Truncate memory descriptor to the 4GB */
-                    Descriptor->NumberOfPages = (((ULONGLONG)MAXULONG + 1) - Descriptor->PhysicalStart) >> EFI_PAGE_SHIFT;
-                }
-            }
-
             /* Convert EFI memory type into XTLDR memory type */
             MemoryType = GetMemoryTypeRoutine((EFI_MEMORY_TYPE)Descriptor->Type);
 
@@ -404,22 +414,22 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             if(MemoryType == LoaderFirmwareTemporary)
             {
                 /* Map EFI firmware code */
-                Status = MapVirtualMemory(PageMap, (PVOID)Descriptor->PhysicalStart,
+                Status = MapVirtualMemory(PageMap, Descriptor->PhysicalStart,
-                                          (PVOID)Descriptor->PhysicalStart, Descriptor->NumberOfPages, MemoryType);
+                                          Descriptor->PhysicalStart, Descriptor->NumberOfPages, MemoryType);
             }
             else if(MemoryType != LoaderFree)
             {
                 /* Add any non-free memory mapping */
-                Status = MapVirtualMemory(PageMap, VirtualAddress, (PVOID)Descriptor->PhysicalStart,
+                Status = MapVirtualMemory(PageMap, VirtualAddress, Descriptor->PhysicalStart,
                                           Descriptor->NumberOfPages, MemoryType);
 
-                /* Calculate next valid virtual address */
+                /* Update virtual address */
-                VirtualAddress += Descriptor->NumberOfPages * EFI_PAGE_SIZE;
+                VirtualAddress = VirtualAddress + (Descriptor->NumberOfPages * MM_PAGE_SIZE);
             }
             else
             {
                 /* Map all other memory as loader free */
-                Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)Descriptor->PhysicalStart,
+                Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Descriptor->PhysicalStart,
                                           Descriptor->NumberOfPages, LoaderFree);
             }
 
@@ -436,7 +446,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Always map first page */
-    Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)0, 1, LoaderFirmwarePermanent);
+    Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, 0, 1, LoaderFirmwarePermanent);
     if(Status != STATUS_EFI_SUCCESS)
     {
         /* Mapping failed */
@@ -444,7 +454,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Map BIOS ROM and VRAM */
-    Status = MapVirtualMemory(PageMap, NULLPTR, (PVOID)0xA0000, 0x60, LoaderFirmwarePermanent);
+    Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, 0xA0000, 0x60, LoaderFirmwarePermanent);
     if(Status != STATUS_EFI_SUCCESS)
     {
         /* Mapping failed */
@@ -452,7 +462,7 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Store next valid virtual address and return success */
-    *MemoryMapAddress = VirtualAddress;
+    *MemoryMapAddress = (PVOID)VirtualAddress;
     return STATUS_EFI_SUCCESS;
 }
 
@@ -481,13 +491,13 @@ Memory::MapEfiMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
 XTCDECL
 EFI_STATUS
 Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
-                         IN PVOID VirtualAddress,
+                         IN ULONGLONG VirtualAddress,
-                         IN PVOID PhysicalAddress,
+                         IN ULONGLONG PhysicalAddress,
                          IN ULONGLONG NumberOfPages,
                          IN LOADER_MEMORY_TYPE MemoryType)
 {
     PXTBL_MEMORY_MAPPING Mapping1, Mapping2, Mapping3;
-    PVOID PhysicalAddressEnd, PhysicalAddress2End;
+    ULONGLONG PhysicalAddressEnd, PhysicalAddress2End;
     PLIST_ENTRY ListEntry, MappingListEntry;
     SIZE_T NumberOfMappedPages;
     EFI_STATUS Status;
@@ -507,7 +517,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     Mapping1->MemoryType = MemoryType;
 
     /* Calculate the end of the physical address */
-    PhysicalAddressEnd = (PVOID)((ULONG_PTR)PhysicalAddress + (NumberOfPages * EFI_PAGE_SIZE) - 1);
+    PhysicalAddressEnd = PhysicalAddress + (NumberOfPages * EFI_PAGE_SIZE) - 1;
 
     /* Iterate through all the mappings already set to insert new mapping at the correct place */
     ListEntry = PageMap->MemoryMap.Flink;
@@ -515,7 +525,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
     {
         /* Take a mapping from the list and calculate its end of physical address */
         Mapping2 = CONTAIN_RECORD(ListEntry, XTBL_MEMORY_MAPPING, ListEntry);
-        PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1);
+        PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1;
 
         /* Check if new mapping is a subset of an existing mapping */
         if(Mapping1->PhysicalAddress >= Mapping2->PhysicalAddress && PhysicalAddressEnd <= PhysicalAddress2End)
@@ -539,7 +549,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             }
 
             /* Calculate number of pages for this mapping */
-            NumberOfMappedPages = ((PUCHAR)PhysicalAddress2End - (PUCHAR)PhysicalAddressEnd) / EFI_PAGE_SIZE;
+            NumberOfMappedPages = (PhysicalAddress2End - PhysicalAddressEnd) / EFI_PAGE_SIZE;
             if(NumberOfMappedPages > 0)
             {
                 /* Pages associated to the mapping, allocate memory for it */
@@ -551,8 +561,8 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
                 }
 
                 /* Set mapping fields and insert it on the top */
-                Mapping3->PhysicalAddress = (PUCHAR)PhysicalAddressEnd + 1;
+                Mapping3->PhysicalAddress = PhysicalAddressEnd + 1;
-                Mapping3->VirtualAddress = NULLPTR;
+                Mapping3->VirtualAddress = (ULONGLONG)NULLPTR;
                 Mapping3->NumberOfPages = NumberOfMappedPages;
                 Mapping3->MemoryType = Mapping2->MemoryType;
                 RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry);
@@ -561,7 +571,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             /* Calculate number of pages and the end of the physical address */
             Mapping2->NumberOfPages = ((PUCHAR)PhysicalAddressEnd + 1 -
                                        (PUCHAR)Mapping2->PhysicalAddress) / EFI_PAGE_SIZE;
-            PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1);
+            PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1;
         }
 
         /* Check if they overlap */
@@ -588,7 +598,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
 
                 /* Set mapping fields and insert it on the top */
                 Mapping3->PhysicalAddress = Mapping1->PhysicalAddress;
-                Mapping3->VirtualAddress = NULLPTR;
+                Mapping3->VirtualAddress = (ULONGLONG)NULLPTR;
                 Mapping3->NumberOfPages = NumberOfMappedPages;
                 Mapping3->MemoryType = Mapping2->MemoryType;
                 RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry);
@@ -597,7 +607,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             /* Calculate number of pages and the end of the physical address */
             Mapping2->NumberOfPages = ((PUCHAR)Mapping1->PhysicalAddress -
                                        (PUCHAR)Mapping2->PhysicalAddress) / EFI_PAGE_SIZE;
-            PhysicalAddress2End = (PVOID)((ULONG_PTR)Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1);
+            PhysicalAddress2End = Mapping2->PhysicalAddress + (Mapping2->NumberOfPages * EFI_PAGE_SIZE) - 1;
         }
 
         /* Check if mapping is really needed */

boot/xtldr/modules/pecoff/pecoff.cc

@@ -729,11 +729,11 @@ PeCoff::RelocateLoadedImage(IN PPECOFF_IMAGE_CONTEXT Image)
     }
     else
     {
-        /* Check if loaded 32-bit PE32 image should be relocated */
         /* Set relocation data directory and image base address */
         DataDirectory = &Image->PeHeader->OptionalHeader32.DataDirectory[PECOFF_IMAGE_DIRECTORY_ENTRY_BASERELOC];
         ImageBase = Image->PeHeader->OptionalHeader32.ImageBase;
 
+        /* Check if loaded 32-bit PE32 image should be relocated */
         if(Image->PeHeader->OptionalHeader32.NumberOfRvaAndSizes <= PECOFF_IMAGE_DIRECTORY_ENTRY_BASERELOC ||
         DataDirectory->VirtualAddress == 0 || DataDirectory->Size < sizeof(PECOFF_IMAGE_BASE_RELOCATION))
         {

boot/xtldr/modules/xtos_o/i686/memory.cc

@@ -59,9 +59,22 @@ EFI_STATUS
 Xtos::EnablePaging(IN PXTBL_PAGE_MAPPING PageMap)
 {
     EFI_STATUS Status;
+    ULONG_PTR SelfMapAddress;
+
+    /* Initialize self map address */
+    if(PageMap->PageMapLevel == 3)
+    {
+        /* For PML3 (PAE) use PTE base address */
+        SelfMapAddress = MM_PTE_BASE;
+    }
+    else
+    {
+        /* For PML2 (PAE disabled) use legacy PDE base address */
+        SelfMapAddress = MM_PDE_LEGACY_BASE;
+    }
 
     /* Build page map */
-    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, MM_PTE_BASE);
+    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, SelfMapAddress);
     if(Status != STATUS_EFI_SUCCESS)
     {
         /* Failed to build page map */

boot/xtldr/modules/xtos_o/xtos.cc

@@ -206,7 +206,7 @@ Xtos::GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap,
         return Status;
     }
 
-    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)Address, Pages, LoaderMemoryData);
+    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderMemoryData);
     if(Status != STATUS_EFI_SUCCESS)
     {
         XtLdrProtocol->Memory.FreePages(Address, Pages);
@@ -234,6 +234,9 @@ Xtos::GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap,
 
     XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, MemoryDescriptorList, PhysicalBase, *VirtualAddress);
 
+    /* Calculate next valid virtual address */
+    *VirtualAddress = (PUINT8)*VirtualAddress + (Pages * EFI_PAGE_SIZE);
+
     return STATUS_EFI_SUCCESS;
 }
 
@@ -266,7 +269,7 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
     {
         return Status;
     }
-    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)Address, Pages, LoaderFirmwarePermanent);
+    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderFirmwarePermanent);
     if(Status != STATUS_EFI_SUCCESS)
     {
         XtLdrProtocol->Memory.FreePages(Address, Pages);
@@ -336,8 +339,8 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
     FrameBufferResource->Header.VirtualAddress = *VirtualAddress;
 
     /* Map frame buffer memory */
-    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, FrameBufferResource->Header.VirtualAddress,
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)FrameBufferResource->Header.VirtualAddress,
-                                           FrameBufferResource->Header.PhysicalAddress,
+                                           (ULONGLONG)FrameBufferResource->Header.PhysicalAddress,
                                            FrameBufferPages, LoaderFirmwarePermanent);
 
     /* Close FrameBuffer protocol */
@@ -389,7 +392,7 @@ Xtos::InitializeApicBase(IN PXTBL_PAGE_MAPPING PageMap)
     }
 
     /* Map APIC base address */
-    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (PVOID)APIC_BASE, ApicBaseAddress, 1, LoaderFirmwarePermanent);
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, APIC_BASE, (ULONGLONG)ApicBaseAddress, 1, LoaderFirmwarePermanent);
     return STATUS_EFI_SUCCESS;
 }
 
@@ -456,7 +459,7 @@ Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap,
                   ParametersSize);
 
     /* Map kernel initialization block */
-    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, *VirtualAddress, (PVOID)LoaderBlock,
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, (ULONGLONG)LoaderBlock,
                                            BlockPages, LoaderSystemBlock);
 
     /* Calculate next valid virtual address */
@@ -469,6 +472,9 @@ Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap,
     XtLdrProtocol->LinkedList.InitializeHead(&LoaderBlock->MemoryDescriptorListHead);
     GetMemoryDescriptorList(PageMap, VirtualAddress, &LoaderBlock->MemoryDescriptorListHead);
 
+    /* Set boot image size */
+    LoaderBlock->BootImageSize = (PFN_NUMBER)(((ULONGLONG)*VirtualAddress - KSEG0_BASE) / EFI_PAGE_SIZE);
+
     /* Return success */
     return STATUS_EFI_SUCCESS;
 }
@@ -607,7 +613,7 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir,
     PXTBL_FRAMEBUFFER_PROTOCOL FrameBufProtocol;
     PPECOFF_IMAGE_CONTEXT ImageContext = NULLPTR;
     PEFI_LOADED_IMAGE_PROTOCOL ImageProtocol;
-    PVOID VirtualAddress, VirtualMemoryArea;
+    PVOID VirtualAddress;
     PXT_ENTRY_POINT KernelEntryPoint;
     EFI_HANDLE ProtocolHandle;
     EFI_STATUS Status;
@@ -629,13 +635,12 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir,
     XtLdrProtocol->Protocol.Close(&ProtocolHandle, &FrameBufGuid);
 
     /* Set base virtual memory area for the kernel mappings */
-    VirtualMemoryArea = (PVOID)KSEG0_BASE;
+    VirtualAddress = (PVOID)(KSEG0_BASE);
-    VirtualAddress = (PVOID)(KSEG0_BASE + KSEG0_KERNEL_BASE);
 
     /* Initialize virtual memory mappings */
     XtLdrProtocol->Memory.InitializePageMap(&PageMap, DeterminePagingLevel(Parameters->Parameters), Size4K);
 
-    Status = XtLdrProtocol->Memory.MapEfiMemory(&PageMap, &VirtualMemoryArea, NULLPTR);
+    Status = XtLdrProtocol->Memory.MapEfiMemory(&PageMap, &VirtualAddress, NULLPTR);
     if(Status != STATUS_EFI_SUCCESS)
     {
         return Status;
@@ -650,8 +655,8 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir,
     }
 
     /* Add kernel image memory mapping */
-    Status = XtLdrProtocol->Memory.MapVirtualMemory(&PageMap, ImageContext->VirtualAddress,
+    Status = XtLdrProtocol->Memory.MapVirtualMemory(&PageMap, (ULONGLONG)ImageContext->VirtualAddress,
-                                                    ImageContext->PhysicalAddress, ImageContext->ImagePages,
+                                                    (ULONGLONG)ImageContext->PhysicalAddress, ImageContext->ImagePages,
                                                     LoaderSystemCode);
     if(Status != STATUS_EFI_SUCCESS)
     {

sdk/xtdk/amd64/ketypes.h

@@ -108,9 +108,6 @@
 /* Static Kernel-Mode address start */
 #define KSEG0_BASE                        0xFFFFF80000000000
 
-/* XTOS Kernel address base */
-#define KSEG0_KERNEL_BASE                 0x0000000800000000
-
 /* XTOS Kernel stack size */
 #define KERNEL_STACK_SIZE                 0x8000
 

sdk/xtdk/amd64/mmtypes.h

@@ -25,11 +25,11 @@
 #define MM_PXE_BASE                                0xFFFFF6FB7DBED000ULL
 
 /* Page directory and page base addresses for 5-level paging */
-#define MM_PTE_LA57_BASE                           0xFFFF000000000000ULL
+#define MM_PTE_LA57_BASE                           0xFFED000000000000ULL
-#define MM_PDE_LA57_BASE                           0xFFFF010000000000ULL
+#define MM_PDE_LA57_BASE                           0xFFEDF68000000000ULL
-#define MM_PPE_LA57_BASE                           0xFFFF010800000000ULL
+#define MM_PPE_LA57_BASE                           0xFFEDF6FB40000000ULL
-#define MM_PXE_LA57_BASE                           0xFFFF010840000000ULL
+#define MM_PXE_LA57_BASE                           0xFFEDF6FB7DA00000ULL
-#define MM_P5E_LA57_BASE                           0xFFFF010840200000ULL
+#define MM_P5E_LA57_BASE                           0xFFEDF6FB7DBED000ULL
 
 /* PTE shift values */
 #define MM_PTE_SHIFT                               3
@@ -39,15 +39,55 @@
 #define MM_PXI_SHIFT                               39
 #define MM_P5I_SHIFT                               48
 
-/* Number of PTEs per page */
+/* PTE state flags */
-#define MM_PTE_PER_PAGE                            512
+#define MM_PTE_VALID                               0x0000000000000001ULL
-#define MM_PDE_PER_PAGE                            512
+#define MM_PTE_ACCESSED                            0x0000000000000020ULL
-#define MM_PPE_PER_PAGE                            512
+#define MM_PTE_DIRTY                               0x0000000000000040ULL
-#define MM_PXE_PER_PAGE                            512
+
+/* PTE scope flags */
+#define MM_PTE_LARGE_PAGE                          0x0000000000000080ULL
+#define MM_PTE_GLOBAL                              0x0000000000000100ULL
+
+/* PTE access flags */
+#define MM_PTE_NOACCESS                            0x0000000000000000ULL
+#define MM_PTE_READONLY                            0x0000000000000000ULL
+#define MM_PTE_EXECUTE                             0x0000000000000000ULL
+#define MM_PTE_EXECUTE_READ                        0x0000000000000000ULL
+#define MM_PTE_READWRITE                           0x8000000000000002ULL
+#define MM_PTE_WRITECOPY                           0x8000000000000200ULL
+#define MM_PTE_EXECUTE_READWRITE                   0x0000000000000002ULL
+#define MM_PTE_EXECUTE_WRITECOPY                   0x0000000000000200ULL
+
+/* PTE protection flags */
+#define MM_PTE_NOEXECUTE                           0x8000000000000000ULL
+#define MM_PTE_PROTECT                             0x8000000000000612ULL
+
+/* PTE cache flags */
+#define MM_PTE_CACHE_ENABLE                        0x0000000000000000ULL
+#define MM_PTE_CACHE_DISABLE                       0x0000000000000010ULL
+#define MM_PTE_CACHE_WRITECOMBINED                 0x0000000000000010ULL
+#define MM_PTE_CACHE_WRITETHROUGH                  0x0000000000000008ULL
+
+/* PTE software flags */
+#define MM_PTE_COPY_ON_WRITE                       0x0000000000000200ULL
+#define MM_PTE_PROTOTYPE                           0x0000000000000400ULL
+#define MM_PTE_TRANSITION                          0x0000000000000800ULL
+
+/* PTE frame bits */
+#define MM_PTE_FRAME_BITS                          57
+
+/* PTE protection bits */
+#define MM_PTE_PROTECTION_BITS                     5
+
+/* Base address of the system page table */
+#define MM_SYSTEM_PTE_BASE                         KSEG0_BASE
 
 /* Minimum number of physical pages needed by the system */
 #define MM_MINIMUM_PHYSICAL_PAGES                  2048
 
+/* Number of system PTEs */
+#define MM_DEFAULT_NUMBER_SYSTEM_PTES              22000
+
 /* Default number of secondary colors */
 #define MM_DEFAULT_SECONDARY_COLORS                64
 
@@ -63,6 +103,9 @@
 /* Maximum physical address used by HAL allocations */
 #define MM_MAXIMUM_PHYSICAL_ADDRESS                0x00000000FFFFFFFFULL
 
+/* Highest system address */
+#define MM_HIGHEST_SYSTEM_ADDRESS                  0xFFFFFFFFFFFFFFFFULL
+
 /* Trampoline code address */
 #define MM_TRAMPOLINE_ADDRESS                      0x80000
 
@@ -252,6 +295,7 @@ typedef struct _MMPFN
             USHORT ReferenceCount;
         } e2;
     } u3;
+    ULONG UsedPageTableEntries;
     union
     {
         MMPTE OriginalPte;
@@ -262,12 +306,11 @@ typedef struct _MMPFN
         ULONG_PTR EntireFrame;
         struct
         {
-            ULONG_PTR PteFrame:58;
+            ULONG_PTR PteFrame:57;
             ULONG_PTR InPageError:1;
             ULONG_PTR VerifierAllocation:1;
             ULONG_PTR AweAllocation:1;
-            ULONG_PTR LockCharged:1;
+            ULONG_PTR Priority:3;
-            ULONG_PTR KernelStack:1;
             ULONG_PTR MustBeCached:1;
         };
     } u4;

sdk/xtdk/bltypes.h

@@ -101,8 +101,8 @@ typedef VOID (XTCDECL *PBL_LLIST_INSERT_HEAD)(IN OUT PLIST_ENTRY ListHead, IN PL
 typedef VOID (XTCDECL *PBL_LLIST_INSERT_TAIL)(IN OUT PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry);
 typedef VOID (XTCDECL *PBL_LLIST_REMOVE_ENTRY)(IN PLIST_ENTRY Entry);
 typedef EFI_STATUS (XTCDECL *PBL_MAP_EFI_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN OUT PVOID *MemoryMapAddress, IN PBL_GET_MEMTYPE_ROUTINE GetMemoryTypeRoutine);
-typedef EFI_STATUS (XTCDECL *PBL_MAP_PAGE)(IN PXTBL_PAGE_MAPPING PageMap, IN ULONG_PTR VirtualAddress, IN ULONG_PTR PhysicalAddress, IN ULONG NumberOfPages);
+typedef EFI_STATUS (XTCDECL *PBL_MAP_PAGE)(IN PXTBL_PAGE_MAPPING PageMap, IN ULONGLONG VirtualAddress, IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages);
-typedef EFI_STATUS (XTCDECL *PBL_MAP_VIRTUAL_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN PVOID VirtualAddress, IN PVOID PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType);
+typedef EFI_STATUS (XTCDECL *PBL_MAP_VIRTUAL_MEMORY)(IN OUT PXTBL_PAGE_MAPPING PageMap, IN ULONGLONG VirtualAddress, IN ULONGLONG PhysicalAddress, IN ULONGLONG NumberOfPages, IN LOADER_MEMORY_TYPE MemoryType);
 typedef VOID (XTAPI *PBL_MOVE_MEMORY)(IN OUT PVOID Destination, IN PCVOID Source, IN SIZE_T Length);
 typedef EFI_STATUS (XTCDECL *PBL_OPEN_VOLUME)(IN PEFI_DEVICE_PATH_PROTOCOL DevicePath, OUT PEFI_HANDLE DiskHandle, OUT PEFI_FILE_HANDLE *FsHandle);
 typedef EFI_STATUS (XTCDECL *PBL_OPEN_PROTOCOL)(OUT PEFI_HANDLE Handle, OUT PVOID *ProtocolHandler, IN PEFI_GUID ProtocolGuid);
@@ -232,8 +232,8 @@ typedef struct _XTBL_KNOWN_BOOT_PROTOCOL
 typedef struct _XTBL_MEMORY_MAPPING
 {
     LIST_ENTRY ListEntry;
-    PVOID VirtualAddress;
+    ULONGLONG VirtualAddress;
-    PVOID PhysicalAddress;
+    ULONGLONG PhysicalAddress;
     ULONGLONG NumberOfPages;
     LOADER_MEMORY_TYPE MemoryType;
 } XTBL_MEMORY_MAPPING, *PXTBL_MEMORY_MAPPING;

sdk/xtdk/i686/ketypes.h

@@ -128,9 +128,6 @@
 /* Static Kernel-Mode address start */
 #define KSEG0_BASE                        0x80000000
 
-/* XTOS Kernel address base */
-#define KSEG0_KERNEL_BASE                 0x01800000
-
 /* XTOS Kernel stack size */
 #define KERNEL_STACK_SIZE                 0x4000
 

sdk/xtdk/i686/mmtypes.h

@@ -35,9 +35,57 @@
 #define MM_PTE_LEGACY_SHIFT                        2
 #define MM_PDI_LEGACY_SHIFT                        22
 
+/* PTE state flags */
+#define MM_PTE_VALID                               0x00000001
+#define MM_PTE_ACCESSED                            0x00000020
+#define MM_PTE_DIRTY                               0x00000040
+
+/* PTE scope flags */
+#define MM_PTE_LARGE_PAGE                          0x00000080
+#define MM_PTE_GLOBAL                              0x00000100
+
+/* PTE access flags */
+#define MM_PTE_NOACCESS                            0x00000000
+#define MM_PTE_READONLY                            0x00000000
+#define MM_PTE_EXECUTE                             0x00000000
+#define MM_PTE_EXECUTE_READ                        0x00000000
+#define MM_PTE_READWRITE                           0x00000002
+#define MM_PTE_WRITECOPY                           0x00000200
+#define MM_PTE_EXECUTE_READWRITE                   0x00000002
+#define MM_PTE_EXECUTE_WRITECOPY                   0x00000200
+
+/* PTE protection flags */
+#define MM_PTE_NOEXECUTE                           0x00000000
+#define MM_PTE_PROTECT                             0x00000612
+
+/* PTE cache flags */
+#define MM_PTE_CACHE_ENABLE                        0x00000000
+#define MM_PTE_CACHE_DISABLE                       0x00000010
+#define MM_PTE_CACHE_WRITECOMBINED                 0x00000010
+#define MM_PTE_CACHE_WRITETHROUGH                  0x00000008
+
+/* PTE software flags */
+#define MM_PTE_COPY_ON_WRITE                       0x00000200
+#define MM_PTE_PROTOTYPE                           0x00000400
+#define MM_PTE_TRANSITION                          0x00000800
+
+/* PTE frame bits */
+#define MM_PTE_FRAME_BITS                          25
+
+/* PTE protection bits */
+#define MM_PTE_PROTECTION_BITS                     5
+
+/* Base address of the system page table */
+#define MM_SYSTEM_PTE_BASE                         NULLPTR
+
 /* Minimum number of physical pages needed by the system */
 #define MM_MINIMUM_PHYSICAL_PAGES                  1100
 
+/* Number of system PTEs */
+#define MM_MINIMUM_NUMBER_SYSTEM_PTES              7000
+#define MM_DEFAULT_NUMBER_SYSTEM_PTES              11000
+#define MM_MAXIMUM_NUMBER_SYSTEM_PTES              22000
+
 /* Default number of secondary colors */
 #define MM_DEFAULT_SECONDARY_COLORS                64
 
@@ -53,6 +101,9 @@
 /* Maximum physical address used by HAL allocations */
 #define MM_MAXIMUM_PHYSICAL_ADDRESS                0xFFFFFFFF
 
+/* Highest system address */
+#define MM_HIGHEST_SYSTEM_ADDRESS                  0xFFFFFFFF
+
 /* Trampoline code address */
 #define MM_TRAMPOLINE_ADDRESS                      0x80000
 
@@ -106,7 +157,6 @@ typedef struct _HARDWARE_MODERN_PTE
 /* Generic Page Table entry union to abstract PML2 and PML3 formats */
 typedef union _HARDWARE_PTE
 {
-    ULONGLONG Long;
     HARDWARE_LEGACY_PTE Pml2;
     HARDWARE_MODERN_PTE Pml3;
 } HARDWARE_PTE, *PHARDWARE_PTE;
@@ -201,12 +251,12 @@ typedef struct _MMPML2_PTE_TRANSITION
 typedef union _MMPML2_PTE
 {
     ULONG Long;
-    HARDWARE_PTE Flush;
+    HARDWARE_LEGACY_PTE Flush;
-    MMPML2_PTE_HARDWARE Hard;
+    MMPML2_PTE_HARDWARE Hardware;
-    MMPML2_PTE_PROTOTYPE Proto;
+    MMPML2_PTE_PROTOTYPE Prototype;
-    MMPML2_PTE_SOFTWARE Soft;
+    MMPML2_PTE_SOFTWARE Software;
-    MMPML2_PTE_TRANSITION Trans;
+    MMPML2_PTE_TRANSITION Transition;
-    MMPML2_PTE_SUBSECTION Subsect;
+    MMPML2_PTE_SUBSECTION Subsection;
     MMPML2_PTE_LIST List;
 } MMPML2_PTE, *PMMPML2_PTE;
 
@@ -296,7 +346,7 @@ typedef struct _MMPML3_PTE_TRANSITION
 typedef union _MMPML3_PTE
 {
     ULONGLONG Long;
-    HARDWARE_PTE Flush;
+    HARDWARE_MODERN_PTE Flush;
     MMPML3_PTE_HARDWARE Hardware;
     MMPML3_PTE_PROTOTYPE Prototype;
     MMPML3_PTE_SOFTWARE Software;
@@ -308,7 +358,6 @@ typedef union _MMPML3_PTE
 /* Generic Page Table Entry union to abstract PML2 and PML3 formats */
 typedef union _MMPTE
 {
-    ULONGLONG Long;
     MMPML2_PTE Pml2;
     MMPML3_PTE Pml3;
 } MMPTE, *PMMPTE;
@@ -339,6 +388,7 @@ typedef struct _MMPFN
             USHORT ReferenceCount;
         } e2;
     } u3;
+    ULONG UsedPageTableEntries;
     union
     {
         MMPTE OriginalPte;
@@ -349,12 +399,11 @@ typedef struct _MMPFN
         ULONG_PTR EntireFrame;
         struct
         {
-            ULONG_PTR PteFrame:26;
+            ULONG_PTR PteFrame:25;
             ULONG_PTR InPageError:1;
             ULONG_PTR VerifierAllocation:1;
             ULONG_PTR AweAllocation:1;
-            ULONG_PTR LockCharged:1;
+            ULONG_PTR Priority:3;
-            ULONG_PTR KernelStack:1;
             ULONG_PTR MustBeCached:1;
         };
     } u4;

sdk/xtdk/mmtypes.h

@@ -4,6 +4,7 @@
  * FILE:            sdk/xtdk/mmtypes.h
  * DESCRIPTION:     Memory management data structures
  * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
+ *                  Aiken Harris <harraiken91@gmail.com>
  */
 
 #ifndef __XTDK_MMTYPES_H
@@ -13,6 +14,48 @@
 #include ARCH_HEADER(xtstruct.h)
 
 
+/* Number of hyper space pages */
+#define MM_HYPERSPACE_PAGE_COUNT                   255
+
+/* Number of paging colors */
+#define MM_PAGING_COLORS                           64
+
+/* PTE frame mask definition */
+#define MM_PFN_PTE_FRAME                           (((ULONG_PTR)1 << MM_PTE_FRAME_BITS) - 1)
+
+/* Number of reserved zeroed PTEs */
+#define MM_RESERVED_ZERO_PTES                      32
+
+/* Memory manager page lists */
+typedef enum _MMPAGELISTS
+{
+   ZeroedPageList = 0,
+   FreePageList = 1,
+   StandbyPageList = 2,
+   ModifiedPageList = 3,
+   ModifiedReadOnlyPageList = 4,
+   BadPageList = 5,
+   ActiveAndValid = 6,
+   TransitionPage = 7
+} MMPAGELISTS, *PMMPAGELISTS;
+
+/* Page cache attributes */
+typedef enum _MMPFN_CACHE_ATTRIBUTE
+{
+    PfnNonCached,
+    PfnCached,
+    PfnWriteCombined,
+    PfnNotMapped
+} MMPFN_CACHE_ATTRIBUTE, *PMMPFN_CACHE_ATTRIBUTE;
+
+/* Page table pool types */
+typedef enum _MMSYSTEM_PTE_POOL_TYPE
+{
+    SystemPteSpace,
+    NonPagedPoolExpansion,
+    MaximumPtePoolTypes
+} MMSYSTEM_PTE_POOL_TYPE, *PMMSYSTEM_PTE_POOL_TYPE;
+
 /* Page map routines structure definition */
 typedef CONST STRUCT _CMMPAGEMAP_ROUTINES
 {
@@ -30,6 +73,50 @@ typedef struct _MMCOLOR_TABLES
     ULONG_PTR Count;
 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 
+/* Memory layout structure definition */
+typedef struct _MMMEMORY_LAYOUT
+{
+    PMMPFN PfnDatabase;
+    PFN_NUMBER PfnDatabaseSize;
+    PVOID SelfMapAddress;
+    PVOID HardwarePoolStart;
+    PVOID HardwarePoolEnd;
+    PVOID HyperSpaceStart;
+    PVOID HyperSpaceEnd;
+    PVOID LoaderMappingsStart;
+    PVOID LoaderMappingsEnd;
+    PFN_NUMBER LoaderMappingsSize;
+    PVOID NonCanonicalStart;
+    PVOID NonCanonicalEnd;
+    PVOID NonPagedPoolStart;
+    PVOID NonPagedPoolEnd;
+    PFN_NUMBER NonPagedPoolSize;
+    PVOID NonPagedExpansionPoolStart;
+    PVOID NonPagedExpansionPoolEnd;
+    PFN_NUMBER NonPagedExpansionPoolSize;
+    PVOID NonPagedSystemPoolStart;
+    PVOID NonPagedSystemPoolEnd;
+    PFN_NUMBER NonPagedSystemPoolSize;
+    PVOID PagedPoolStart;
+    PVOID PagedPoolEnd;
+    PFN_NUMBER PagedPoolSize;
+    PVOID ReservedSystemPoolStart;
+    PVOID ReservedSystemPoolEnd;
+    PVOID SessionSpaceStart;
+    PVOID SessionSpaceEnd;
+    PFN_NUMBER SessionSpaceSize;
+    PVOID SharedSystemPageStart;
+    PVOID SharedSystemPageEnd;
+    PVOID SystemCacheStart;
+    PVOID SystemCacheEnd;
+    PVOID SystemWorkingSetStart;
+    PVOID SystemWorkingSetEnd;
+    PVOID UserSpaceStart;
+    PVOID UserSpaceEnd;
+    PVOID PteSpaceStart;
+    PVOID PteSpaceEnd;
+} MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT;
+
 /* Page Frame Entry structure definition */
 typedef struct _MMPFNENTRY
 {
@@ -45,4 +132,13 @@ typedef struct _MMPFNENTRY
     USHORT ParityError:1;
 } MMPFNENTRY, *PMMPFNENTRY;
 
+/* Page Frame List structure definition */
+typedef struct _MMPFNLIST
+{
+    PFN_NUMBER Total;
+    MMPAGELISTS ListName;
+    PFN_NUMBER Flink;
+    PFN_NUMBER Blink;
+} MMPFNLIST, *PMMPFNLIST;
+
 #endif /* __XTDK_MMTYPES_H */

sdk/xtdk/xtbase.h

@@ -29,6 +29,9 @@ typedef UCHAR KRUNLEVEL, *PKRUNLEVEL;
 /* Spin locks synchronization mechanism */
 typedef ULONG_PTR KSPIN_LOCK, *PKSPIN_LOCK;
 
+/* Page Frame Number count */
+typedef ULONG PFN_COUNT;
+
 /* Page Frame Number */
 typedef ULONG_PTR PFN_NUMBER, *PPFN_NUMBER;
 

sdk/xtdk/xtfw.h

@@ -107,6 +107,7 @@ typedef struct _KERNEL_INITIALIZATION_BLOCK
     ULONG BlockVersion;
     ULONG ProtocolVersion;
     PWCHAR KernelParameters;
+    PFN_NUMBER BootImageSize;
     LIST_ENTRY LoadOrderListHead;
     LIST_ENTRY MemoryDescriptorListHead;
     LIST_ENTRY BootDriverListHead;

sdk/xtdk/xtstruct.h

@@ -48,6 +48,9 @@ typedef enum _KTHREAD_STATE KTHREAD_STATE, *PKTHREAD_STATE;
 typedef enum _KTIMER_TYPE KTIMER_TYPE, *PKTIMER_TYPE;
 typedef enum _KUBSAN_DATA_TYPE KUBSAN_DATA_TYPE, *PKUBSAN_DATA_TYPE;
 typedef enum _LOADER_MEMORY_TYPE LOADER_MEMORY_TYPE, *PLOADER_MEMORY_TYPE;
+typedef enum _MMPAGELISTS MMPAGELISTS, *PMMPAGELISTS;
+typedef enum _MMPFN_CACHE_ATTRIBUTE MMPFN_CACHE_ATTRIBUTE, *PMMPFN_CACHE_ATTRIBUTE;
+typedef enum _MMSYSTEM_PTE_POOL_TYPE MMSYSTEM_PTE_POOL_TYPE, *PMMSYSTEM_PTE_POOL_TYPE;
 typedef enum _MODE MODE, *PMODE;
 typedef enum _RTL_VARIABLE_TYPE RTL_VARIABLE_TYPE, *PRTL_VARIABLE_TYPE;
 typedef enum _SYSTEM_FIRMWARE_TYPE SYSTEM_FIRMWARE_TYPE, *PSYSTEM_FIRMWARE_TYPE;
@@ -274,7 +277,9 @@ typedef struct _LOADER_INFORMATION_BLOCK LOADER_INFORMATION_BLOCK, *PLOADER_INFO
 typedef struct _LOADER_MEMORY_DESCRIPTOR LOADER_MEMORY_DESCRIPTOR, *PLOADER_MEMORY_DESCRIPTOR;
 typedef struct _M128 M128, *PM128;
 typedef struct _MMCOLOR_TABLES MMCOLOR_TABLES, *PMMCOLOR_TABLES;
+typedef struct _MMMEMORY_LAYOUT MMMEMORY_LAYOUT, *PMMMEMORY_LAYOUT;
 typedef struct _MMPFNENTRY MMPFNENTRY, *PMMPFNENTRY;
+typedef struct _MMPFNLIST MMPFNLIST, *PMMPFNLIST;
 typedef struct _PCAT_FIRMWARE_INFORMATION PCAT_FIRMWARE_INFORMATION, *PPCAT_FIRMWARE_INFORMATION;
 typedef struct _PCI_BRIDGE_CONTROL_REGISTER PCI_BRIDGE_CONTROL_REGISTER, *PPCI_BRIDGE_CONTROL_REGISTER;
 typedef struct _PCI_COMMON_CONFIG PCI_COMMON_CONFIG, *PPCI_COMMON_CONFIG;

xtoskrnl/CMakeLists.txt

@@ -51,14 +51,22 @@ list(APPEND XTOSKRNL_SOURCE
     ${XTOSKRNL_SOURCE_DIR}/ke/spinlock.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/sysres.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/timer.cc
-    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/init.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/alloc.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/mmgr.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pagemap.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/paging.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfault.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfn.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pte.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/alloc.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/colors.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/data.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/hlpool.cc
-    ${XTOSKRNL_SOURCE_DIR}/mm/init.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/kpool.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/mmgr.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/paging.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/pfn.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/pte.cc
     ${XTOSKRNL_SOURCE_DIR}/po/idle.cc
     ${XTOSKRNL_SOURCE_DIR}/rtl/${ARCH}/dispatch.cc
     ${XTOSKRNL_SOURCE_DIR}/rtl/${ARCH}/exsup.cc
@@ -82,11 +90,12 @@ add_library(libxtos ${XTOSKRNL_SOURCE})
 
 # Link kernel executable
 add_executable(xtoskrnl
-               ${XTOSKRNL_SOURCE}
                ${CMAKE_CURRENT_BINARY_DIR}/xtoskrnl.def)
 
 # Add linker libraries
-target_link_libraries(xtoskrnl)
+target_link_libraries(xtoskrnl
+                      PRIVATE
+                      libxtos)
 
 # Set proper binary name and install target
 set_target_properties(xtoskrnl PROPERTIES SUFFIX .exe)

xtoskrnl/README.md

@@ -4,6 +4,18 @@ within the XTOS kernel space. It is responsible for various core services, such
 management, and process scheduling. The kernel contains the scheduler (sometimes referred to as the Dispatcher), the
 cache, object, and memory managers, the security manager, and other executive components described below.
 
+
+## Kernel Parameters
+Kernel parameters are XTOS boot-time options used to ensure proper initialization and handling of hardware peripherals.
+These parameters can be configured either temporarily by editing the boot entry in the bootloader’s selection menu, or
+permanently by modifying the XTLDR configuration file.
+
+The following is a consolidated list of available kernel parameters:
+ * **NOXPA**: Disables PAE or LA57 support, depending on the CPU architecture. This parameter is handled by the
+   bootloader, which configures paging and selects the appropriate Page Map Level (PML) before transferring control to
+   the kernel.
+
+## Source Code
 The source code of the kernel is organized into subsystem-specific directories. Each directory name also defines the
 corresponding C++ namespace in which the subsystem's classes and routines reside. These subsystems include:
 

xtoskrnl/includes/mm.hh

@@ -12,10 +12,15 @@
 #include <xtos.hh>
 
 #include XTOS_ARCH_HEADER(mm, pagemap.hh)
+#include XTOS_ARCH_HEADER(mm, paging.hh)
+#include XTOS_ARCH_HEADER(mm, pte.hh)
 
+#include <mm/alloc.hh>
+#include <mm/colors.hh>
 #include <mm/hlpool.hh>
-#include <mm/init.hh>
 #include <mm/kpool.hh>
-#include <mm/paging.hh>
+#include <mm/mmgr.hh>
+#include <mm/pfault.hh>
+#include <mm/pfn.hh>
 
 #endif /* __XTOSKRNL_MM_HH */

xtoskrnl/includes/mm/alloc.hh

@@ -0,0 +1,25 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/alloc.hh
+ * DESCRIPTION:     Memory manager pool allocation
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_ALLOC_HH
+#define __XTOSKRNL_MM_ALLOC_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Allocator
+    {
+        public:
+            STATIC XTAPI VOID InitializeNonPagedPool(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_ALLOC_HH */

xtoskrnl/includes/mm/amd64/pagemap.hh

@@ -1,13 +1,13 @@
 /**
  * PROJECT:         ExectOS
  * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/includes/mm/pagemap.hh
+ * FILE:            xtoskrnl/includes/mm/amd64/pagemap.hh
  * DESCRIPTION:     Low-level support for page map manipulation
  * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
  */
 
-#ifndef __XTOSKRNL_MM_PAGEMAP_HH
+#ifndef __XTOSKRNL_MM_AMD64_PAGEMAP_HH
-#define __XTOSKRNL_MM_PAGEMAP_HH
+#define __XTOSKRNL_MM_AMD64_PAGEMAP_HH
 
 #include <xtos.hh>
 
@@ -21,33 +21,75 @@ namespace MM
             MMPAGEMAP_INFO PageMapInfo;
 
         public:
-            XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer);
+            XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
-            XTAPI PMMP5E GetP5eAddress(PVOID Address);
+                                    IN LONG Count);
-            XTAPI PMMPDE GetPdeAddress(PVOID Address);
+            XTAPI VOID ClearPte(IN PMMPTE PtePointer);
-            XTAPI PMMPPE GetPpeAddress(PVOID Address);
+            XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
-            XTAPI PMMPTE GetPteAddress(PVOID Address);
+            XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
-            XTAPI PMMPXE GetPxeAddress(PVOID Address);
+            XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
-            virtual XTAPI VOID InitializePageMapInfo(VOID) = 0;
+            XTAPI PMMP5E GetP5eAddress(IN PVOID Address);
-            XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer);
+            XTAPI ULONG GetP5eOffset(IN PVOID Address);
-            XTAPI VOID SetPte(PHARDWARE_PTE PtePointer,
+            XTAPI PVOID GetP5eVirtualAddress(IN PMMP5E P5ePointer);
-                              PFN_NUMBER PageFrameNumber,
+            XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
-                              BOOLEAN Writable);
+            XTAPI PMMPDE GetPdeAddress(IN PVOID Address);
-            XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer,
+            XTAPI ULONG GetPdeOffset(IN PVOID Address);
-                                     BOOLEAN CacheDisable,
+            VIRTUAL XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer) = 0;
-                                     BOOLEAN WriteThrough);
+            XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
+            XTAPI ULONG GetPpeOffset(IN PVOID Address);
+            VIRTUAL XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer) = 0;
+            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI PMMPTE GetPteAddress(IN PVOID Address);
+            XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                      PMMPTE StartPte);
+            XTAPI ULONG GetPteOffset(IN PVOID Address);
+            XTAPI ULONG GetPteSize(VOID);
+            XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer);
+            VIRTUAL XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer) = 0;
+            XTAPI PMMPXE GetPxeAddress(IN PVOID Address);
+            XTAPI ULONG GetPxeOffset(IN PVOID Address);
+            VIRTUAL XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer) = 0;
+            XTAPI BOOLEAN GetXpaStatus();
+            VIRTUAL XTAPI VOID InitializePageMapInfo(VOID) = 0;
+            XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer);
+            XTAPI VOID SetNextEntry(IN PMMPTE Pte,
+                                    IN ULONG_PTR Value);
+            XTAPI VOID SetOneEntry(IN PMMPTE Pte,
+                                   IN BOOLEAN Value);
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                              IN PFN_NUMBER PageFrameNumber,
+                              IN ULONG_PTR AttributesMask);
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                              IN ULONG_PTR Attributes);
+            XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                     IN BOOLEAN CacheDisable,
+                                     IN BOOLEAN WriteThrough);
+            XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                     IN ULONG_PTR Protection);
+            XTAPI VOID WritePte(IN PMMPTE Pte,
+                                IN MMPTE Value);
     } PAGEMAP, *PPAGEMAP;
 
     class PageMapBasic final : public PageMap
     {
         public:
+            XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
+            XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
+            XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer);
             XTAPI VOID InitializePageMapInfo(VOID);
     };
 
     class PageMapXpa final : public PageMap
     {
         public:
+            XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
+            XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
+            XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer);
             XTAPI VOID InitializePageMapInfo(VOID);
     };
 }
 
-#endif /* __XTOSKRNL_MM_PAGEMAP_HH */
+#endif /* __XTOSKRNL_MM_AMD64_PAGEMAP_HH */

xtoskrnl/includes/mm/amd64/paging.hh

@@ -0,0 +1,78 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/amd64/paging.hh
+ * DESCRIPTION:     Low level page management support for AMD64
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_AMD64_PAGING_HH
+#define __XTOSKRNL_MM_AMD64_PAGING_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Paging
+    {
+        private:
+            STATIC PPAGEMAP PmlRoutines;
+
+        public:
+            STATIC XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
+                                           IN LONG Count);
+            STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID FlushTlb(VOID);
+            STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
+            STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
+            STATIC XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
+            STATIC XTAPI PMMP5E GetP5eAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetP5eVirtualAddress(IN PMMP5E P5ePointer);
+            STATIC XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
+            STATIC XTAPI PMMPDE GetPdeAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
+            STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address);
+            STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                             PMMPTE StartPte);
+            STATIC XTAPI ULONG GetPteSize(VOID);
+            STATIC XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer);
+            STATIC XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
+            STATIC XTAPI PMMPXE GetPxeAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetPxeVirtualAddress(IN PMMPXE PxePointer);
+            STATIC XTAPI BOOLEAN GetXpaStatus(VOID);
+            STATIC XTAPI VOID InitializePageMapSupport(VOID);
+            STATIC XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID SetNextEntry(IN PMMPTE Pte,
+                                           IN ULONG_PTR Value);
+            STATIC XTAPI VOID SetOneEntry(IN PMMPTE Pte,
+                                          IN BOOLEAN Value);
+            STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                     IN PFN_NUMBER PageFrameNumber,
+                                     IN ULONG_PTR AttributesMask);
+            STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                     IN ULONG_PTR Attributes);
+            STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                            IN BOOLEAN CacheDisable,
+                                            IN BOOLEAN WriteThrough);
+            STATIC XTAPI VOID WritePte(IN PMMPTE Pte,
+                                       IN MMPTE Value);
+            STATIC XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                            IN ULONG_PTR Protection);
+            STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address,
+                                             IN ULONG Size);
+
+        private:
+            STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID);
+            STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID);
+            STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_AMD64_PAGING_HH */

xtoskrnl/includes/mm/amd64/pte.hh

@@ -0,0 +1,69 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/amd64/pte.hh
+ * DESCRIPTION:     Page Table Entry (PTE) for AMD64 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_AMD64_PTE_HH
+#define __XTOSKRNL_MM_AMD64_PTE_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Pte
+    {
+        private:
+            STATIC MMPTE FirstSystemFreePte[MaximumPtePoolTypes];
+            STATIC PMMPTE SystemPteBase;
+            STATIC PMMPTE SystemPtesEnd[MaximumPtePoolTypes];
+            STATIC PMMPTE SystemPtesStart[MaximumPtePoolTypes];
+            STATIC PFN_COUNT TotalSystemFreePtes[MaximumPtePoolTypes];
+            STATIC MMPTE ValidPte;
+
+        public:
+            STATIC XTAPI BOOLEAN AddressValid(IN PVOID VirtualAddress);
+            STATIC XTAPI PFN_COUNT GetPtesPerPage(VOID);
+            STATIC XTAPI PMMPTE GetSystemPteBaseAddress(VOID);
+            STATIC XTAPI PMMPTE GetValidPte(VOID);
+            STATIC XTAPI VOID InitializePageTable(VOID);
+            STATIC XTAPI VOID InitializeSystemPte(VOID);
+            STATIC XTAPI VOID InitializeSystemPteSpace(VOID);
+            STATIC XTAPI VOID MapP5E(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMP5E TemplateP5e);
+            STATIC XTAPI VOID MapPDE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPDE TemplatePde);
+            STATIC XTAPI VOID MapPPE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPPE TemplatePpe);
+            STATIC XTAPI VOID MapPTE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPTE TemplatePte);
+            STATIC XTAPI VOID MapPXE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPXE TemplatePxe);
+            STATIC XTAPI VOID ReleaseSystemPtes(IN PMMPTE StartingPte,
+                                                IN PFN_COUNT NumberOfPtes,
+                                                IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType);
+            STATIC XTAPI PMMPTE ReserveSystemPtes(IN PFN_COUNT NumberOfPtes,
+                                                  IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType);
+
+        private:
+            STATIC XTAPI BOOLEAN FindFreeCluster(IN PFN_COUNT NumberOfPtes,
+                                                 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+                                                 OUT PMMPTE *FoundCluster,
+                                                 OUT PMMPTE *PreviousClusterNode);
+            STATIC XTAPI ULONG GetClusterSize(IN PMMPTE Pte);
+            STATIC XTAPI VOID InitializeSystemPtePool(IN PMMPTE StartingPte,
+                                                      IN PFN_COUNT NumberOfPtes,
+                                                      IN MMSYSTEM_PTE_POOL_TYPE PoolType);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_AMD64_PTE_HH */

xtoskrnl/includes/mm/colors.hh

@@ -0,0 +1,38 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/colors.hh
+ * DESCRIPTION:     Memory manager page coloring subsystem
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_COLORS_HH
+#define __XTOSKRNL_MM_COLORS_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Colors
+    {
+        private:
+            STATIC PMMCOLOR_TABLES FreePages[FreePageList + 1];
+            STATIC MMPFNLIST ModifiedPages[MM_PAGING_COLORS];
+            STATIC ULONG PagingColors;
+            STATIC ULONG PagingColorsMask;
+
+        public:
+            STATIC XTAPI VOID ComputePageColoring(VOID);
+            STATIC XTAPI PMMCOLOR_TABLES GetFreePages(IN MMPAGELISTS PageList,
+                                                      IN ULONG Color);
+            STATIC XTAPI PMMPFNLIST GetModifiedPages(IN ULONG Color);
+            STATIC XTAPI ULONG GetNextColor(VOID);
+            STATIC XTAPI ULONG GetPagingColors(VOID);
+            STATIC XTAPI ULONG GetPagingColorsMask(VOID);
+            STATIC XTAPI VOID InitializeColorTables(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_COLORS_HH */

xtoskrnl/includes/mm/i686/pagemap.hh

@@ -1,13 +1,13 @@
 /**
  * PROJECT:         ExectOS
  * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/includes/mm/pagemap.hh
+ * FILE:            xtoskrnl/includes/mm/i686/pagemap.hh
  * DESCRIPTION:     Low-level support for page map manipulation
  * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
  */
 
-#ifndef __XTOSKRNL_MM_PAGEMAP_HH
+#ifndef __XTOSKRNL_MM_I686_PAGEMAP_HH
-#define __XTOSKRNL_MM_PAGEMAP_HH
+#define __XTOSKRNL_MM_I686_PAGEMAP_HH
 
 #include <xtos.hh>
 
@@ -21,45 +21,128 @@ namespace MM
             MMPAGEMAP_INFO PageMapInfo;
 
         public:
-            XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer);
+            VIRTUAL XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
-            XTAPI PMMPDE GetPdeAddress(PVOID Address);
+                                            IN ULONG Count) = 0;
-            XTAPI PMMPPE GetPpeAddress(PVOID Address);
+            VIRTUAL XTAPI VOID ClearPte(IN PMMPTE PtePointer) = 0;
-            XTAPI PMMPTE GetPteAddress(PVOID Address);
+            VIRTUAL XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte) = 0;
-            virtual XTAPI VOID InitializePageMapInfo(VOID) = 0;
+            VIRTUAL XTAPI PMMPTE GetNextPte(IN PMMPTE Pte) = 0;
-            virtual XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer) = 0;
+            VIRTUAL XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte) = 0;
-            virtual XTAPI VOID SetPte(PHARDWARE_PTE PtePointer,
+            VIRTUAL XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte) = 0;
-                                      PFN_NUMBER PageFrameNumber,
+            XTAPI PMMPDE GetPdeAddress(IN PVOID Address);
-                                      BOOLEAN Writable) = 0;
+            XTAPI ULONG GetPdeOffset(IN PVOID Address);
-            virtual XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer,
+            VIRTUAL XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer) = 0;
-                                             BOOLEAN CacheDisable,
+            XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
-                                             BOOLEAN WriteThrough) = 0;
+            XTAPI ULONG GetPpeOffset(IN PVOID Address);
+            XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
+            VIRTUAL XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer) = 0;
+            XTAPI PMMPTE GetPteAddress(IN PVOID Address);
+            XTAPI ULONG GetPteOffset(IN PVOID Address);
+            VIRTUAL XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                              PMMPTE StartPte) = 0;
+            VIRTUAL XTAPI ULONG GetPteSize(VOID) = 0;
+            VIRTUAL XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer) = 0;
+            VIRTUAL XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer) = 0;
+            VIRTUAL XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer) = 0;
+            VIRTUAL XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer) = 0;
+            XTAPI BOOLEAN GetXpaStatus();
+            VIRTUAL XTAPI VOID InitializePageMapInfo(VOID) = 0;
+            VIRTUAL XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer) = 0;
+            VIRTUAL XTAPI VOID SetNextEntry(IN PMMPTE Pte,
+                                            IN ULONG_PTR Value) = 0;
+            VIRTUAL XTAPI VOID SetOneEntry(IN PMMPTE Pte,
+                                           IN BOOLEAN Value) = 0;
+            VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                      IN PFN_NUMBER PageFrameNumber,
+                                      IN ULONG_PTR AttributesMask) = 0;
+            VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                      IN ULONG_PTR Attributes) = 0;
+            VIRTUAL XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                             IN BOOLEAN CacheDisable,
+                                             IN BOOLEAN WriteThrough) = 0;
+            VIRTUAL XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                             IN ULONG_PTR Protection) = 0;
+            VIRTUAL XTAPI VOID WritePte(IN PMMPTE Pte,
+                                        IN MMPTE Value) = 0;
+
     } PAGEMAP, *PPAGEMAP;
 
     class PageMapBasic final : public PageMap
     {
         public:
+            XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
+                                    IN ULONG Count);
+            XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
+            XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
+            XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
+            XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
+            XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                      PMMPTE StartPte);
+            XTAPI ULONG GetPteSize(VOID);
+            XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer);
+            XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
             XTAPI VOID InitializePageMapInfo(VOID);
-            XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer);
+            XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer);
-            XTAPI VOID SetPte(PHARDWARE_PTE PtePointer,
+            XTAPI VOID SetNextEntry(IN PMMPTE Pte,
-                              PFN_NUMBER PageFrameNumber,
+                                    IN ULONG_PTR Value);
-                              BOOLEAN Writable);
+            XTAPI VOID SetOneEntry(IN PMMPTE Pte,
-            XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer,
+                                   IN BOOLEAN Value);
-                                     BOOLEAN CacheDisable,
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                                     BOOLEAN WriteThrough);
+                              IN PFN_NUMBER PageFrameNumber,
+                              IN ULONG_PTR AttributesMask);
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                              IN ULONG_PTR Attributes);
+            XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                     IN BOOLEAN CacheDisable,
+                                     IN BOOLEAN WriteThrough);
+            XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                     IN ULONG_PTR Protection);
+            XTAPI VOID WritePte(IN PMMPTE Pte,
+                                IN MMPTE Value);
     };
 
     class PageMapXpa final : public PageMap
     {
         public:
+            XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
+                                    IN ULONG Count);
+            XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
+            XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
+            XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
+            XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
+            XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                      PMMPTE StartPte);
+            XTAPI ULONG GetPteSize(VOID);
+            XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer);
+            XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer);
+            XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
             XTAPI VOID InitializePageMapInfo(VOID);
-            XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer);
+            XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer);
-            XTAPI VOID SetPte(PHARDWARE_PTE PtePointer,
+            XTAPI VOID SetNextEntry(IN PMMPTE Pte,
-                              PFN_NUMBER PageFrameNumber,
+                                    IN ULONG_PTR Value);
-                              BOOLEAN Writable);
+            XTAPI VOID SetOneEntry(IN PMMPTE Pte,
-            XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer,
+                                   IN BOOLEAN Value);
-                                     BOOLEAN CacheDisable,
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                                     BOOLEAN WriteThrough);
+                              IN PFN_NUMBER PageFrameNumber,
+                              IN ULONG_PTR AttributesMask);
+            XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                              IN ULONG_PTR Attributes);
+            XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                     IN BOOLEAN CacheDisable,
+                                     IN BOOLEAN WriteThrough);
+            XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                     IN ULONG_PTR Protection);
+            XTAPI VOID WritePte(IN PMMPTE Pte,
+                                IN MMPTE Value);
     };
 }
 
-#endif /* __XTOSKRNL_MM_PAGEMAP_HH */
+#endif /* __XTOSKRNL_MM_I686_PAGEMAP_HH */

xtoskrnl/includes/mm/i686/paging.hh

@@ -0,0 +1,74 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/i686/paging.hh
+ * DESCRIPTION:     Low level page management support for i686
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_I686_PAGING_HH
+#define __XTOSKRNL_MM_I686_PAGING_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Paging
+    {
+        private:
+            STATIC PPAGEMAP PmlRoutines;
+
+        public:
+            STATIC XTAPI PMMPTE AdvancePte(IN PMMPTE Pte,
+                                           IN LONG Count);
+            STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID FlushTlb(VOID);
+            STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
+            STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
+            STATIC XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
+            STATIC XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
+            STATIC XTAPI PMMPDE GetPdeAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
+            STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
+            STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
+            STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address);
+            STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte,
+                                             PMMPTE StartPte);
+            STATIC XTAPI ULONG GetPteSize(VOID);
+            STATIC XTAPI ULONG GetPteSoftwareProtection(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONG GetPteSoftwarePrototype(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONG GetPteSoftwareTransition(IN PMMPTE PtePointer);
+            STATIC XTAPI PVOID GetPteVirtualAddress(IN PMMPTE PtePointer);
+            STATIC XTAPI BOOLEAN GetXpaStatus(VOID);
+            STATIC XTAPI VOID InitializePageMapSupport(VOID);
+            STATIC XTAPI BOOLEAN PteValid(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID SetNextEntry(IN PMMPTE Pte,
+                                           IN ULONG_PTR Value);
+            STATIC XTAPI VOID SetOneEntry(IN PMMPTE Pte,
+                                          IN BOOLEAN Value);
+            STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                     IN PFN_NUMBER PageFrameNumber,
+                                     IN ULONG_PTR AttributesMask);
+            STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
+                                     IN ULONG_PTR Attributes);
+            STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
+                                            IN BOOLEAN CacheDisable,
+                                            IN BOOLEAN WriteThrough);
+            STATIC XTAPI VOID WritePte(IN PMMPTE Pte,
+                                       IN MMPTE Value);
+            STATIC XTAPI VOID TransitionPte(IN PMMPTE PointerPte,
+                                            IN ULONG_PTR Protection);
+            STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address,
+                                             IN ULONG Size);
+
+        private:
+            STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID);
+            STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID);
+            STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_I686_PAGING_HH */

xtoskrnl/includes/mm/i686/pte.hh

@@ -0,0 +1,63 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/i686/pte.hh
+ * DESCRIPTION:     Page Table Entry (PTE) for i686 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_I686_PTE_HH
+#define __XTOSKRNL_MM_I686_PTE_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Pte
+    {
+        private:
+            STATIC MMPTE FirstSystemFreePte[MaximumPtePoolTypes];
+            STATIC PMMPTE SystemPteBase;
+            STATIC PMMPTE SystemPtesEnd[MaximumPtePoolTypes];
+            STATIC PMMPTE SystemPtesStart[MaximumPtePoolTypes];
+            STATIC PFN_COUNT TotalSystemFreePtes[MaximumPtePoolTypes];
+            STATIC MMPTE ValidPte;
+
+        public:
+            STATIC XTAPI BOOLEAN AddressValid(IN PVOID VirtualAddress);
+            STATIC XTAPI PFN_COUNT GetPtesPerPage(VOID);
+            STATIC XTAPI PMMPTE GetSystemPteBaseAddress(VOID);
+            STATIC XTAPI PMMPTE GetValidPte(VOID);
+            STATIC XTAPI VOID InitializePageTable(VOID);
+            STATIC XTAPI VOID InitializeSystemPte(VOID);
+            STATIC XTAPI VOID InitializeSystemPteSpace(VOID);
+            STATIC XTAPI VOID MapPDE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPDE TemplatePde);
+            STATIC XTAPI VOID MapPPE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPPE TemplatePpe);
+            STATIC XTAPI VOID MapPTE(IN PVOID StartAddress,
+                                     IN PVOID EndAddress,
+                                     IN PMMPTE TemplatePte);
+            STATIC XTAPI VOID ReleaseSystemPtes(IN PMMPTE StartingPte,
+                                                IN PFN_COUNT NumberOfPtes,
+                                                IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType);
+            STATIC XTAPI PMMPTE ReserveSystemPtes(IN PFN_COUNT NumberOfPtes,
+                                                  IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType);
+
+        private:
+            STATIC XTAPI BOOLEAN FindFreeCluster(IN PFN_COUNT NumberOfPtes,
+                                                 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+                                                 OUT PMMPTE *FoundCluster,
+                                                 OUT PMMPTE *PreviousClusterNode);
+            STATIC XTAPI ULONG GetClusterSize(IN PMMPTE Pte);
+            STATIC XTAPI VOID InitializeSystemPtePool(IN PMMPTE StartingPte,
+                                                      IN PFN_COUNT NumberOfPtes,
+                                                      IN MMSYSTEM_PTE_POOL_TYPE PoolType);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_I686_PTE_HH */

xtoskrnl/includes/mm/init.hh

@@ -1,39 +0,0 @@
-/**
- * PROJECT:         ExectOS
- * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/includes/mm/init.hh
- * DESCRIPTION:     Memory Manager initialization
- * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
- */
-
-#ifndef __XTOSKRNL_MM_INIT_HH
-#define __XTOSKRNL_MM_INIT_HH
-
-#include <xtos.hh>
-
-
-/* Memory Manager */
-namespace MM
-{
-    class Init
-    {
-        private:
-            STATIC PLOADER_MEMORY_DESCRIPTOR FreeDescriptor;
-            STATIC ULONG_PTR HighestPhysicalPage;
-            STATIC ULONG_PTR LowestPhysicalPage;
-            STATIC ULONG NumberOfPhysicalPages;
-            STATIC LOADER_MEMORY_DESCRIPTOR OldFreeDescriptor;
-
-        public:
-            STATIC XTAPI VOID InitializeMemoryManager(VOID);
-            STATIC XTAPI VOID InitializePageMapSupport(VOID);
-            STATIC XTAPI VOID ScanMemoryDescriptors(VOID);
-
-        private:
-            STATIC XTAPI VOID InitializeArchitecture(VOID);
-            STATIC XTAPI BOOLEAN VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType);
-            STATIC XTAPI BOOLEAN VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType);
-       };
-}
-
-#endif /* __XTOSKRNL_MM_INIT_HH */

xtoskrnl/includes/mm/kpool.hh

@@ -21,13 +21,12 @@ namespace MM
             STATIC UCHAR ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE];
 
         public:
-            STATIC XTAPI XTSTATUS AllocateKernelStack(IN PVOID *Stack,
+            STATIC XTAPI XTSTATUS AllocateKernelStack(OUT PVOID *Stack,
-                                                     IN BOOLEAN LargeStack,
+                                                      IN ULONG StackSize);
-                                                     IN UCHAR SystemNode);
             STATIC XTAPI XTSTATUS AllocateProcessorStructures(IN ULONG CpuNumber,
                                                               OUT PVOID *StructuresData);
             STATIC XTAPI VOID FreeKernelStack(IN PVOID Stack,
-                                             IN BOOLEAN LargeStack);
+                                              IN ULONG StackSize);
             STATIC XTAPI VOID FreeProcessorStructures(IN PVOID StructuresData);
     };
 }

xtoskrnl/includes/mm/mmgr.hh

@@ -0,0 +1,44 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/mmgr.hh
+ * DESCRIPTION:     Memory Manager
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_MMGR_HH
+#define __XTOSKRNL_MM_MMGR_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Manager
+    {
+        private:
+            STATIC MMMEMORY_LAYOUT MemoryLayout;
+            STATIC PFN_NUMBER NumberOfSystemPtes;
+
+        public:
+            STATIC XTAPI ULONG_PTR GetInstalledMemorySize(VOID);
+            STATIC XTAPI PMMMEMORY_LAYOUT GetMemoryLayout(VOID);
+            STATIC XTAPI PFN_NUMBER GetNumberOfSystemPtes();
+            STATIC XTAPI VOID InitializeMemoryLayout(VOID);
+            STATIC XTAPI VOID InitializeMemoryManager(VOID);
+            STATIC XTAPI BOOLEAN VerifyMemoryTypeFree(IN LOADER_MEMORY_TYPE MemoryType);
+            STATIC XTAPI BOOLEAN VerifyMemoryTypeInvisible(IN LOADER_MEMORY_TYPE MemoryType);
+
+        private:
+            STATIC XTAPI VOID ComputeBootImageSize(OUT PPFN_NUMBER BootImageSize);
+            STATIC XTAPI VOID ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize);
+            STATIC XTAPI VOID ComputeSystemPteSize(OUT PPFN_NUMBER PteSize);
+            STATIC XTAPI VOID DumpMemoryLayout(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_MMGR_HH */

xtoskrnl/includes/mm/paging.hh

@@ -1,47 +0,0 @@
-/**
- * PROJECT:         ExectOS
- * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/includes/mm/paging.hh
- * DESCRIPTION:     Low level page management support
- * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
- */
-
-#ifndef __XTOSKRNL_MM_PAGING_HH
-#define __XTOSKRNL_MM_PAGING_HH
-
-#include <xtos.hh>
-
-
-/* Memory Manager */
-namespace MM
-{
-    class Paging
-    {
-        private:
-            STATIC PPAGEMAP PmlRoutines;
-
-        public:
-            STATIC XTAPI VOID ClearPte(PHARDWARE_PTE PtePointer);
-            STATIC XTAPI VOID FlushTlb(VOID);
-            STATIC XTAPI PMMPDE GetPdeAddress(PVOID Address);
-            STATIC XTAPI PMMPPE GetPpeAddress(PVOID Address);
-            STATIC XTAPI PMMPTE GetPteAddress(PVOID Address);
-            STATIC XTAPI VOID InitializePageMapSupport(VOID);
-            STATIC XTAPI BOOLEAN PteValid(PHARDWARE_PTE PtePointer);
-            STATIC XTAPI VOID SetPte(PHARDWARE_PTE PtePointer,
-                                     PFN_NUMBER PageFrameNumber,
-                                     BOOLEAN Writable);
-            STATIC XTAPI VOID SetPteCaching(PHARDWARE_PTE PtePointer,
-                                            BOOLEAN CacheDisable,
-                                            BOOLEAN WriteThrough);
-            STATIC XTFASTCALL VOID ZeroPages(IN PVOID Address,
-                                             IN ULONG Size);
-
-        private:
-            STATIC XTAPI BOOLEAN GetExtendedPhysicalAddressingStatus(VOID);
-            STATIC XTAPI PPAGEMAP GetPageMapBasicRoutines(VOID);
-            STATIC XTAPI PPAGEMAP GetPageMapXpaRoutines(VOID);
-    };
-}
-
-#endif /* __XTOSKRNL_MM_PAGING_HH */

xtoskrnl/includes/mm/pfault.hh

@@ -0,0 +1,25 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/pfault.hh
+ * DESCRIPTION:     Page fault support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_PFAULT_HH
+#define __XTOSKRNL_MM_PFAULT_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class PageFault
+    {
+        public:
+            STATIC XTFASTCALL XTSTATUS CheckPdeForPagedPool(IN PVOID VirtualAddress);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_PFAULT_HH */

xtoskrnl/includes/mm/pfn.hh

@@ -0,0 +1,77 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/pfn.hh
+ * DESCRIPTION:     Physical Frame Number (PFN) support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_PFN_HH
+#define __XTOSKRNL_MM_PFN_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Pfn
+    {
+        private:
+            STATIC PFN_NUMBER AvailablePages;
+            STATIC MMPFNLIST BadPagesList;
+            STATIC PLOADER_MEMORY_DESCRIPTOR FreeDescriptor;
+            STATIC MMPFNLIST FreePagesList;
+            STATIC ULONG_PTR HighestPhysicalPage;
+            STATIC PVOID HighestUserAddress;
+            STATIC ULONG_PTR LowestPhysicalPage;
+            STATIC MMPFNLIST ModifiedPagesList;
+            STATIC MMPFNLIST ModifiedReadOnlyPagesList;
+            STATIC ULONGLONG NumberOfPhysicalPages;
+            STATIC LOADER_MEMORY_DESCRIPTOR OriginalFreeDescriptor;
+            STATIC PMMPFNLIST PageLocationList[];
+            STATIC MMPFNLIST RomPagesList;
+            STATIC MMPFNLIST StandbyPagesList;
+            STATIC MMPFNLIST ZeroedPagesList;
+
+        public:
+            STATIC XTAPI PFN_NUMBER AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages);
+            STATIC XTAPI PFN_NUMBER AllocatePhysicalPage(IN ULONG Color);
+            STATIC XTAPI VOID ComputePfnDatabaseSize(OUT PPFN_NUMBER DatabaseSize);
+            STATIC XTAPI VOID DecrementReferenceCount(IN PMMPFN Pfn1,
+                                                      IN PFN_NUMBER PageFrameIndex,
+                                                      IN BOOLEAN BeginStandbyList = FALSE);
+            STATIC XTAPI VOID DecrementShareCount(IN PMMPFN Pfn1,
+                                                  IN PFN_NUMBER PageFrameIndex,
+                                                  IN BOOLEAN BeginStandbyList = FALSE);
+            STATIC XTAPI VOID FreePhysicalPage(IN PMMPTE PointerPte);
+            STATIC XTAPI ULONG_PTR GetHighestPhysicalPage(VOID);
+            STATIC XTAPI ULONGLONG GetNumberOfPhysicalPages(VOID);
+            STATIC XTAPI PMMPFN GetPfnEntry(IN PFN_NUMBER Pfn);
+            STATIC XTAPI VOID InitializePfnDatabase(VOID);
+            STATIC VOID XTAPI LinkPfnToPte(IN PFN_NUMBER PageFrameIndex,
+                                           IN PMMPTE PointerPte,
+                                           IN BOOLEAN Modified);
+            STATIC XTAPI VOID ScanMemoryDescriptors(VOID);
+
+        private:
+            STATIC XTAPI VOID DecrementAvailablePages(VOID);
+            STATIC XTAPI VOID IncrementAvailablePages(VOID);
+            STATIC XTAPI VOID InitializePageTablePfns(VOID);
+            STATIC XTAPI VOID LinkFreePage(IN PFN_NUMBER PageFrameIndex);
+            STATIC XTAPI VOID LinkPage(IN PMMPFNLIST ListHead,
+                                       IN PFN_NUMBER PageFrameIndex);
+            STATIC XTAPI VOID LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex,
+                                                  IN PMMPTE PointerPte);
+            STATIC XTFASTCALL VOID LinkStandbyPage(IN PFN_NUMBER PageFrameIndex);
+            STATIC XTAPI VOID ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
+                                                      IN PFN_NUMBER PageCount,
+                                                      IN LOADER_MEMORY_TYPE MemoryType);
+            STATIC XTAPI VOID ScanPageTable(IN PMMPTE PointerPte,
+                                            IN ULONG Level);
+            STATIC XTAPI PFN_NUMBER UnlinkFreePage(IN PFN_NUMBER PageFrameIndex,
+                                                   IN ULONG Color);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_PFN_HH */

xtoskrnl/ke/amd64/krnlinit.cc

@@ -86,6 +86,10 @@ KE::KernelInit::StartKernel(VOID)
     /* Save processor state */
     Processor::SaveProcessorState(&Prcb->ProcessorState);
 
+    /* Initialize spin locks */
+    SpinLock::InitializeAllLocks();
+    SpinLock::InitializeLockQueues();
+
     /* Lower to APC runlevel */
     RunLevel::LowerRunLevel(APC_LEVEL);
 
@@ -108,6 +112,9 @@ KE::KernelInit::StartKernel(VOID)
     CurrentThread->WaitRunLevel = DISPATCH_LEVEL;
     CurrentProcess->ActiveProcessors |= (ULONG_PTR)1 << Prcb->CpuNumber;
 
+    /* Initialize Memory Manager */
+    MM::Manager::InitializeMemoryManager();
+
     /* Enter infinite loop */
     DebugPrint(L"KernelInit::StartKernel() finished. Entering infinite loop.\n");
     Crash::HaltSystem();

xtoskrnl/ke/i686/krnlinit.cc

@@ -86,6 +86,10 @@ KE::KernelInit::StartKernel(VOID)
     /* Save processor state */
     Processor::SaveProcessorState(&Prcb->ProcessorState);
 
+    /* Initialize spin locks */
+    SpinLock::InitializeAllLocks();
+    SpinLock::InitializeLockQueues();
+
     /* Lower to APC runlevel */
     RunLevel::LowerRunLevel(APC_LEVEL);
 
@@ -108,6 +112,9 @@ KE::KernelInit::StartKernel(VOID)
     CurrentThread->WaitRunLevel = DISPATCH_LEVEL;
     CurrentProcess->ActiveProcessors |= (ULONG_PTR)1 << Prcb->CpuNumber;
 
+    /* Initialize Memory Manager */
+    MM::Manager::InitializeMemoryManager();
+
     /* Enter infinite loop */
     DebugPrint(L"KernelInit::StartKernel() finished. Entering infinite loop.\n");
     Crash::HaltSystem();

xtoskrnl/ke/kthread.cc

@@ -152,7 +152,7 @@ KE::KThread::InitializeThread(IN PKPROCESS Process,
     if(!Stack)
     {
         /* Allocate new stack */
-        Status = MM::KernelPool::AllocateKernelStack(&Stack, FALSE, 0);
+        Status = MM::KernelPool::AllocateKernelStack(&Stack, KERNEL_STACK_SIZE);
         if(Status != STATUS_SUCCESS || !Stack)
         {
             /* Stack allocation failed */

xtoskrnl/mm/alloc.cc

@@ -0,0 +1,11 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/alloc.cc
+ * DESCRIPTION:     Memory manager pool allocation
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+

xtoskrnl/mm/amd64/alloc.cc

@@ -0,0 +1,29 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/amd64/alloc.cc
+ * DESCRIPTION:     Memory manager pool allocation
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+XTAPI
+VOID
+MM::Allocator::InitializeNonPagedPool(VOID)
+{
+    PMMMEMORY_LAYOUT MemoryLayout;
+
+    UNIMPLEMENTED;
+
+    /* Retrieve memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Map PPE and PDE for whole non-paged pool */
+    MM::Pte::MapPPE(MemoryLayout->NonPagedPoolStart, MemoryLayout->NonPagedExpansionPoolEnd, MM::Pte::GetValidPte());
+    MM::Pte::MapPDE(MemoryLayout->NonPagedPoolStart, MemoryLayout->NonPagedExpansionPoolEnd, MM::Pte::GetValidPte());
+
+    /* Map PTE only for the base of the non-paged pool */
+    MM::Pte::MapPTE(MemoryLayout->NonPagedPoolStart, (PCHAR)MemoryLayout->NonPagedPoolEnd - 1, MM::Pte::GetValidPte());
+}

xtoskrnl/mm/amd64/init.cc

@@ -1,25 +0,0 @@
-/**
- * PROJECT:         ExectOS
- * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/mm/amd64/init.cc
- * DESCRIPTION:     Architecture specific Memory Manager initialization routines
- * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
- *                  Aiken Harris <harraiken91@gmail.com>
- */
-
-#include <xtos.hh>
-
-
-/**
- * Performs architecture specific initialization of the XTOS Memory Manager.
- *
- * @return This routine does not return any value.
- *
- * @since XT 1.0
- */
-XTAPI
-VOID
-MM::Init::InitializeArchitecture(VOID)
-{
-    UNIMPLEMENTED;
-}

xtoskrnl/mm/amd64/mmgr.cc

@@ -0,0 +1,390 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/amd64/mmgr.cc
+ * DESCRIPTION:     Memory Manager
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Calculates the maximum possible size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR MaximumNonPagedPoolSize;
+
+    /* Start with the 1MiB and add 400KiB for each MiB above 16MiB */
+    MaximumNonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 409600);
+
+    /* Check if non-paged pool does not exceed 128GiB */
+    if(MaximumNonPagedPoolSize > 137438953472)
+    {
+        /* Limit non-paged pool size to 128GiB */
+        MaximumNonPagedPoolSize = 137438953472;
+    }
+
+    /* Return non-paged pool size */
+    *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize);
+}
+
+/**
+ * Calculates the size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR NonPagedPoolSize;
+
+    /* Start with 1MiB and add 32KiB for each MiB above 16MiB */
+    NonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 32768);
+
+    /* Check if non-paged pool does not exceed 128GiB */
+    if(NonPagedPoolSize > 137438953472)
+    {
+        /* Limit non-paged pool size to 128GiB */
+        NonPagedPoolSize = 137438953472;
+    }
+
+    /* Return non-paged pool size in pages, aligned up to page size boundary */
+    *PoolSize = SIZE_TO_PAGES(ROUND_UP(NonPagedPoolSize, MM_PAGE_SIZE));
+}
+
+/**
+ * Calculates the size of the paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONGLONG PagedPoolSize, PteCount;
+    ULONG PtesPerPage;
+
+    /* Start with 4x maximum non-paged pool size and at least 48MiB */
+    ComputeMaximumNonPagedPoolSize(&PagedPoolSize);
+    PagedPoolSize *= 4 * MM_PAGE_SIZE;
+
+    /* Ensure that paged pool size is at least 48MiB */
+    if(PagedPoolSize < 50331648)
+    {
+        /* Increase paged pool size to at least 48MiB */
+        PagedPoolSize = 50331648;
+    }
+
+    /* Check if paged pool does not overlap non-paged pool */
+    if(PagedPoolSize > (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart)
+    {
+        /* Limit paged pool size to maximum possible */
+        PagedPoolSize = (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart;
+    }
+
+    /* Check if paged pool does not exceed 128GiB */
+    if(PagedPoolSize > 137438953472)
+    {
+        /* Limit paged pool size to 128GiB */
+        PagedPoolSize = 137438953472;
+    }
+
+    /* Get the number of PTEs per page and calculate size of paged pool */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+    PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage);
+
+    /* Return paged pool size */
+    *PoolSize = PteCount * PtesPerPage;
+}
+
+/**
+ * Calculates the size of the session space.
+ *
+ * @param SpaceSize
+ *        A pointer to a variable that will receive the number of pages available by the session space.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize)
+{
+    PFN_NUMBER SessionSpaceSize;
+
+    /* Session Pool, Session View, Session Image, Session Working Set and System View takes 1120MiB */
+    SessionSpaceSize = 1174405120;
+
+    /* Return number of pages used by the session space */
+    *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE;
+}
+
+/**
+ * Calculates the size of the system PTEs.
+ *
+ * @param PteSize
+ *        A pointer to a variable that will receive the number of system PTEs.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize)
+{
+    PFN_NUMBER SystemPteSize;
+
+    /* Check if system has less than 24MiB of physical memory */
+    if(MM::Pfn::GetNumberOfPhysicalPages() < 6144)
+    {
+        /* Set minimal system PTE size */
+        SystemPteSize = 7000;
+    }
+    else
+    {
+        /* Use standard system PTE size */
+        SystemPteSize = 11000;
+
+        /* Check if system has more than 32MiB of physical memory */
+        if(MM::Pfn::GetNumberOfPhysicalPages() > 8192)
+        {
+            /* Double system PTE size */
+            SystemPteSize *= 2;
+
+            /* Check if system has more than 256MiB of physical memory */
+            if(MM::Pfn::GetNumberOfPhysicalPages() > 65536)
+            {
+                /* Double system PTE size */
+                SystemPteSize *= 2;
+            }
+        }
+    }
+
+    /* Return system PTE size */
+    *PteSize = SystemPteSize;
+}
+
+/**
+ * Dumps the kernel's memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::DumpMemoryLayout(VOID)
+{
+    /* Dump memory layout */
+    DebugPrint(L"System with %zu MiB of installed memory:\n"
+               L"User Space:               %.16P - %.16P\n"
+               L"Non-Canonical:            %.16P - %.16P\n"
+               L"Reserved System Pool:     %.16P - %.16P\n"
+               L"PTE Space:                %.16P - %.16P\n"
+               L"Hyper Space:              %.16P - %.16P\n"
+               L"Shared System Page:       %.16P - %.16P\n"
+               L"System Working Set:       %.16P - %.16P\n"
+               L"Loader Mappings:          %.16P - %.16P\n"
+               L"Non-Paged System Pool:    %.16P - %.16P\n"
+               L"Paged Pool:               %.16P - %.16P\n"
+               L"Session Space:            %.16P - %.16P\n"
+               L"System Cache:             %.16P - %.16P\n"
+               L"PFN Database:             %.16P - %.16P\n"
+               L"Non-Paged Pool:           %.16P - %.16P\n"
+               L"Non-Paged Expansion Pool: %.16P - %.16P\n"
+               L"Hardware Pool:            %.16P - %.16P\n",
+               GetInstalledMemorySize(),
+               MemoryLayout.UserSpaceStart,
+               MemoryLayout.UserSpaceEnd,
+               MemoryLayout.NonCanonicalStart,
+               MemoryLayout.NonCanonicalEnd,
+               MemoryLayout.ReservedSystemPoolStart,
+               MemoryLayout.ReservedSystemPoolEnd,
+               MemoryLayout.PteSpaceStart,
+               MemoryLayout.PteSpaceEnd,
+               MemoryLayout.HyperSpaceStart,
+               MemoryLayout.HyperSpaceEnd,
+               MemoryLayout.SharedSystemPageStart,
+               MemoryLayout.SharedSystemPageEnd,
+               MemoryLayout.SystemWorkingSetStart,
+               MemoryLayout.SystemWorkingSetEnd,
+               MemoryLayout.LoaderMappingsStart,
+               MemoryLayout.LoaderMappingsEnd,
+               MemoryLayout.NonPagedSystemPoolStart,
+               MemoryLayout.NonPagedSystemPoolEnd,
+               MemoryLayout.PagedPoolStart,
+               MemoryLayout.PagedPoolEnd,
+               MemoryLayout.SessionSpaceStart,
+               MemoryLayout.SessionSpaceEnd,
+               MemoryLayout.SystemCacheStart,
+               MemoryLayout.SystemCacheEnd,
+               MemoryLayout.PfnDatabase,
+               (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE),
+               MemoryLayout.NonPagedPoolStart,
+               MemoryLayout.NonPagedPoolEnd,
+               MemoryLayout.NonPagedExpansionPoolStart,
+               MemoryLayout.NonPagedExpansionPoolEnd,
+               MemoryLayout.HardwarePoolStart,
+               MemoryLayout.HardwarePoolEnd);
+}
+
+/**
+ * Initializes the kernel's virtual memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::InitializeMemoryLayout(VOID)
+{
+    PFN_NUMBER MaximumNonPagedPoolSize;
+    ULONG_PTR PfnDatabaseEnd;
+
+    /* Check if 5-level paging (LA57) is enabled */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Set PML5 base address */
+        MemoryLayout.SelfMapAddress = (PVOID)MM_P5E_LA57_BASE;
+
+        /* Define memory layout for 5-level paging */
+        MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x00FFFFFFFFFEFFFF;
+        MemoryLayout.NonCanonicalStart = (PVOID)0x0080000000000000;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0xFEFFFFFFFFFFFFFF;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFF00000000000000;
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFECFFFFFFFFFFFF;
+        MemoryLayout.PteSpaceStart = (PVOID)0xFFED000000000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xFFEDFFFFFFFFFFFF;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF;
+    }
+    else
+    {
+        /* Set PML4 base address */
+        MemoryLayout.SelfMapAddress = (PVOID)MM_PXE_BASE;
+
+        /* Define memory layout for 4-level paging */
+        MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x000007FFFFFEFFFF;
+        MemoryLayout.NonCanonicalStart = (PVOID)0x0000800000000000;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0xFFFF7FFFFFFFFFFF;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFFFF800000000000;
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFFFF67FFFFFFFFF;
+        MemoryLayout.PteSpaceStart = (PVOID)0xFFFFF68000000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xFFFFF6FFFFFFFFFF;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF;
+    }
+
+    /* Compute allocation size for the PFN database */
+    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
+
+    /* Compute boot image size */
+    ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize);
+
+    /* Compute session space size */
+    ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize);
+
+    /* Update loader mappings space end address */
+    MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONGLONG)MemoryLayout.LoaderMappingsStart +
+                                             MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE);
+
+    /* Update session space start address */
+    MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd -
+                                             MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE);
+
+    /* Compute system PTE size */
+    ComputeSystemPteSize(&NumberOfSystemPtes);
+
+    /* Compute non-paged pool size */
+    ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize);
+    ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize);
+
+    /* Compute paged pool size */
+    ComputePagedPoolSize(&MemoryLayout.PagedPoolSize);
+
+    /* Insert the PFN database at the beginning of the non-paged pool */
+    MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.NonPagedPoolStart;
+
+    /* Compute the PFN database page-aligned end address */
+    PfnDatabaseEnd = (ULONGLONG)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+    PfnDatabaseEnd = ROUND_UP(PfnDatabaseEnd, MM_PAGE_SIZE);
+
+    /* Shrink the non-paged pool to fit the PFN database */
+    MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd;
+
+    /* Assign the rest of the non-paged pool to the expansion pool */
+    MemoryLayout.NonPagedExpansionPoolStart = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart +
+                                                      MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE);
+    MemoryLayout.NonPagedPoolEnd = MemoryLayout.NonPagedExpansionPoolStart;
+    MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart +
+                                                    MaximumNonPagedPoolSize * MM_PAGE_SIZE);
+    MemoryLayout.NonPagedExpansionPoolSize = ((ULONGLONG)MemoryLayout.NonPagedExpansionPoolEnd -
+                                              (ULONGLONG)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE;
+
+    /* Update paged pool end address */
+    MemoryLayout.PagedPoolEnd = (PVOID)(((ULONGLONG)MemoryLayout.PagedPoolStart +
+                                         MemoryLayout.PagedPoolSize * MM_PAGE_SIZE) - 1);
+}

xtoskrnl/mm/amd64/pagemap.cc

@@ -9,6 +9,28 @@
 #include <xtos.hh>
 
 
+/**
+ * Advances a PTE pointer by a given number of entries, considering the actual PTE size.
+ *
+ * @param Pte
+ *        The PTE pointer to advance.
+ *
+ * @param Count
+ *        The number of PTE entries to advance by.
+ *
+ * @return The advanced PTE pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::PageMap::AdvancePte(IN PMMPTE Pte,
+                        IN LONG Count)
+{
+    /* Return advanced PTE pointer */
+    return (PMMPTE)((ULONG_PTR)Pte + (Count * sizeof(MMPTE)));
+}
+
 /**
  * Clears the contents of a page table entry (PTE).
  *
@@ -21,13 +43,64 @@
  */
 XTAPI
 VOID
-MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer)
+MM::PageMap::ClearPte(IN PMMPTE PtePointer)
 {
-    PtePointer->CacheDisable = 0;
+    /* Clear PTE */
-    PtePointer->PageFrameNumber = 0;
+    PtePointer->Long = 0;
-    PtePointer->Valid = 0;
+}
-    PtePointer->Writable = 0;
+
-    PtePointer->WriteThrough = 0;
+/**
+ * Gets the next entry in a PTE list.
+ *
+ * @param Pte
+ *        The PTE pointer to get the next entry from.
+ *
+ * @return This routine returns the next entry in the PTE list.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG_PTR
+MM::PageMap::GetNextEntry(IN PMMPTE Pte)
+{
+    /* Return next entry in PTE list */
+    return Pte->List.NextEntry;
+}
+
+/**
+ * Advances a PTE pointer, considering the actual PTE size.
+ *
+ * @param Pte
+ *        The PTE pointer to advance.
+ *
+ * @return The advanced PTE pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::PageMap::GetNextPte(IN PMMPTE Pte)
+{
+    /* Return advanced PTE pointer */
+    return AdvancePte(Pte, 1);
+}
+
+/**
+ * Checks if a PTE list contains only one entry.
+ *
+ * @param Pte
+ *        The PTE pointer to check.
+ *
+ * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::PageMap::GetOneEntry(IN PMMPTE Pte)
+{
+    /* Return one entry status */
+    return Pte->List.OneEntry;
 }
 
 /**
@@ -42,7 +115,7 @@ MM::PageMap::ClearPte(PHARDWARE_PTE PtePointer)
  */
 XTAPI
 PMMP5E
-MM::PageMap::GetP5eAddress(PVOID Address)
+MM::PageMap::GetP5eAddress(IN PVOID Address)
 {
     ULONGLONG Offset;
 
@@ -51,6 +124,58 @@ MM::PageMap::GetP5eAddress(PVOID Address)
     return (PMMP5E)((PageMapInfo.P5eBase + Offset) * PageMapInfo.Xpa);
 }
 
+/**
+ * Gets the Offset of the P5E (Page Map Level 5 Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding P5E.
+ *
+ * @return This routine returns the Offset of the P5E, or NULLPTR if LA57 is not enabled.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetP5eOffset(IN PVOID Address)
+{
+    /* Return P5E Offset */
+    return (((((ULONGLONG)Address) >> MM_P5I_SHIFT) & 0x1FF) * PageMapInfo.Xpa);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Map Level 5 Entry.
+ *
+ * @param P5ePointer
+ *        Specifies the address of the P5E.
+ *
+ * @return This routine returns the virtual address mapped by the P5E, or NULLPTR if LA57 is not enabled.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMap::GetP5eVirtualAddress(IN PMMP5E P5ePointer)
+{
+    return (PVOID)((((LONGLONG)P5ePointer << 52) >> 7) * PageMapInfo.Xpa);
+}
+
+/**
+ * Gets the page frame number from a corresponding PTE.
+ *
+ * @param Pte
+ *        The PTE pointer to get the page frame number from.
+ *
+ * @return This routine returns the page frame number.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PFN_NUMBER
+MM::PageMap::GetPageFrameNumber(IN PMMPTE Pte)
+{
+    return Pte->Hardware.PageFrameNumber;
+}
+
 /**
  * Gets the address of the PDE (Page Directory Entry), that maps given address.
  *
@@ -63,7 +188,7 @@ MM::PageMap::GetP5eAddress(PVOID Address)
  */
 XTAPI
 PMMPDE
-MM::PageMap::GetPdeAddress(PVOID Address)
+MM::PageMap::GetPdeAddress(IN PVOID Address)
 {
     ULONGLONG Offset;
 
@@ -73,18 +198,36 @@ MM::PageMap::GetPdeAddress(PVOID Address)
 }
 
 /**
- * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address.
+ * Gets the Offset of the PDE (Page Directory Entry), that maps given address.
  *
  * @param Address
  *        Specifies the virtual address for which to retrieve the corresponding PDE.
  *
+ * @return This routine returns the Offset of the PDE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPdeOffset(IN PVOID Address)
+{
+    /* Return PDE Offset */
+    return ((((ULONGLONG)Address) >> MM_PDI_SHIFT) & 0x1FF);
+}
+
+/**
+ * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding PPE.
+ *
  * @return This routine returns the address of the PPE.
  *
  * @since XT 1.0
  */
 XTAPI
 PMMPPE
-MM::PageMap::GetPpeAddress(PVOID Address)
+MM::PageMap::GetPpeAddress(IN PVOID Address)
 {
     ULONGLONG Offset;
 
@@ -93,6 +236,42 @@ MM::PageMap::GetPpeAddress(PVOID Address)
     return (PMMPPE)(PageMapInfo.PpeBase + Offset);
 }
 
+/**
+ * Gets the Offset of the PPE (Page Directory Pointer Table Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding PPE.
+ *
+ * @return This routine returns the Offset of the PPE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPpeOffset(IN PVOID Address)
+{
+    /* Return PPE Offset */
+    return ((((ULONGLONG)Address) >> MM_PPI_SHIFT) & 0x1FF);
+}
+
+ /**
+ * Gets the entire contents of a Page Table Entry (PTE) as a single value.
+ *
+ * @param PtePointer
+ *        Pointer to the Page Table Entry (PTE) to read.
+ *
+ * @return This routine returns the contents of the PTE as a single value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG_PTR
+MM::PageMap::GetPte(IN PMMPTE PtePointer)
+{
+    /* Return PTE value */
+    return PtePointer->Long;
+}
+
 /**
  * Gets the address of the PTE (Page Table Entry), that maps given address.
  *
@@ -105,7 +284,7 @@ MM::PageMap::GetPpeAddress(PVOID Address)
  */
 XTAPI
 PMMPTE
-MM::PageMap::GetPteAddress(PVOID Address)
+MM::PageMap::GetPteAddress(IN PVOID Address)
 {
     ULONGLONG Offset;
 
@@ -114,6 +293,115 @@ MM::PageMap::GetPteAddress(PVOID Address)
     return (PMMPTE)(PageMapInfo.PteBase + Offset);
 }
 
+/**
+ * Calculates the distance between two PTE pointers.
+ *
+ * @param EndPte
+ *        Pointer to the ending Page Table Entry.
+ *
+ * @param StartPte
+ *        Pointer to the starting Page Table Entry.
+ *
+ * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+LONG
+MM::PageMap::GetPteDistance(PMMPTE EndPte,
+                            PMMPTE StartPte)
+{
+    /* Return distance between PTE pointers */
+    return ((ULONG_PTR)EndPte - (ULONG_PTR)StartPte) / sizeof(MMPTE);
+}
+
+/**
+ * Gets the Offset of the PTE (Page Table Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding PTE.
+ *
+ * @return This routine returns the Offset of the PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPteOffset(IN PVOID Address)
+{
+    /* Return PTE Offset */
+    return ((((ULONGLONG)Address) >> MM_PTI_SHIFT) & 0x1FF);
+}
+
+/**
+ * Gets the size of a PTE.
+ *
+ * @return This routine returns the size of a PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPteSize(VOID)
+{
+    /* Return the size of MMPTE */
+    return sizeof(MMPTE);
+}
+
+/**
+ * Gets the software protection value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software protection value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPteSoftwareProtection(IN PMMPTE PtePointer)
+{
+    /* Return PTE software protection value */
+    return (ULONG)PtePointer->Software.Protection;
+}
+
+/**
+ * Gets the software prototype value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software prototype value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPteSoftwarePrototype(IN PMMPTE PtePointer)
+{
+    /* Return PTE software prototype value */
+    return (ULONG)PtePointer->Software.Prototype;
+}
+
+/**
+ * Gets the software transition value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software transition value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPteSoftwareTransition(IN PMMPTE PtePointer)
+{
+    /* Return PTE software transition value */
+    return (ULONG)PtePointer->Software.Transition;
+}
+
 /**
  * Gets the address of the PXE (Extended Page Entry), that maps given address.
  *
@@ -126,14 +414,47 @@ MM::PageMap::GetPteAddress(PVOID Address)
  */
 XTAPI
 PMMPXE
-MM::PageMap::GetPxeAddress(PVOID Address)
+MM::PageMap::GetPxeAddress(IN PVOID Address)
 {
     ULONGLONG Offset;
 
+    /* Calculate offset and return PXE address */
     Offset = ((((ULONGLONG)Address & (((ULONGLONG)1 << PageMapInfo.VaBits) - 1)) >> MM_PXI_SHIFT) << MM_PTE_SHIFT);
     return (PMMPXE)(PageMapInfo.PxeBase + Offset);
 }
 
+/**
+ * Gets the Offset of the PXE (Extended Page Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding PXE.
+ *
+ * @return This routine returns the Offset of the PXE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::PageMap::GetPxeOffset(IN PVOID Address)
+{
+    /* Return PXE Offset */
+    return ((((ULONGLONG)Address) >> MM_PXI_SHIFT) & 0x1FF);
+}
+
+/**
+ * Gets the status of Extended Paging Address (XPA) mode.
+ *
+ * @return This routine returns TRUE if XPA is enabled, FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::PageMap::GetXpaStatus()
+{
+    return PageMapInfo.Xpa;
+}
+
 /**
  * Checks whether the given PML2 page table entry (PTE) is valid.
  *
@@ -146,22 +467,20 @@ MM::PageMap::GetPxeAddress(PVOID Address)
  */
 XTAPI
 BOOLEAN
-MM::PageMap::PteValid(PHARDWARE_PTE PtePointer)
+MM::PageMap::PteValid(IN PMMPTE PtePointer)
 {
-    return (BOOLEAN)PtePointer->Valid;
+    /* Check if PTE is valid */
+    return (BOOLEAN)PtePointer->Hardware.Valid;
 }
 
 /**
- * Sets a PML2 page table entry (PTE) with the specified physical page and access flags.
+ * Sets the next entry in a PTE list.
  *
- * @param PtePointer
+ * @param Pte
- *        Pointer to the page table entry (PTE) to set.
+ *        The PTE pointer to modify.
  *
- * @param PageFrameNumber
+ * @param Value
- *        Physical frame number to map.
+ *        The value to set as the next entry.
- *
- * @param Writable
- *        Indicates whether the page should be writable.
  *
  * @return This routine does not return any value.
  *
@@ -169,13 +488,82 @@ MM::PageMap::PteValid(PHARDWARE_PTE PtePointer)
  */
 XTAPI
 VOID
-MM::PageMap::SetPte(PHARDWARE_PTE PtePointer,
+MM::PageMap::SetNextEntry(IN PMMPTE Pte,
-                    PFN_NUMBER PageFrameNumber,
+                          IN ULONG_PTR Value)
-                    BOOLEAN Writable)
 {
-    PtePointer->PageFrameNumber = PageFrameNumber;
+    /* Set next entry in PTE list */
-    PtePointer->Valid = 1;
+    Pte->List.NextEntry = Value;
-    PtePointer->Writable = Writable;
+}
+
+/**
+ * Sets the flag indicating whether a PTE list contains only one entry.
+ *
+ * @param Pte
+ *        The PTE pointer to modify.
+ *
+ * @param Value
+ *        The value to set. TRUE if the list has only one entry, FALSE otherwise.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::PageMap::SetOneEntry(IN PMMPTE Pte,
+                         IN BOOLEAN Value)
+{
+    /* Set one entry status */
+    Pte->List.OneEntry = Value;
+}
+
+/**
+ * Sets a Page Table Entry (PTE) with the specified physical page and access flags.
+ *
+ * @param PtePointer
+ *        Pointer to the page table entry (PTE) to set.
+ *
+ * @param PageFrameNumber
+ *        Physical frame number to map.
+ *
+ * @param AttributesMask
+ *        Specifies the attributes mask to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::PageMap::SetPte(IN PMMPTE PtePointer,
+                    IN PFN_NUMBER PageFrameNumber,
+                    IN ULONG_PTR AttributesMask)
+{
+    /* Set PTE */
+    PtePointer->Hardware.PageFrameNumber = PageFrameNumber;
+    PtePointer->Hardware.Valid = 1;
+    PtePointer->Long |= AttributesMask;
+}
+
+/**
+ * Sets a Page Table Entry (PTE) with the specified attributes.
+ *
+ * @param PtePointer
+ *        Pointer to the page table entry (PTE) to set.
+ *
+ * @param Attributes
+ *        Specifies the attributes to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::PageMap::SetPte(IN PMMPTE PtePointer,
+                    IN ULONG_PTR Attributes)
+{
+    PtePointer->Long = Attributes;
 }
 
 /**
@@ -196,12 +584,138 @@ MM::PageMap::SetPte(PHARDWARE_PTE PtePointer,
  */
 XTAPI
 VOID
-MM::PageMap::SetPteCaching(PHARDWARE_PTE PtePointer,
+MM::PageMap::SetPteCaching(IN PMMPTE PtePointer,
-                           BOOLEAN CacheDisable,
+                           IN BOOLEAN CacheDisable,
-                           BOOLEAN WriteThrough)
+                           IN BOOLEAN WriteThrough)
 {
-    PtePointer->CacheDisable = CacheDisable;
+    /* Set caching attributes */
-    PtePointer->WriteThrough = WriteThrough;
+    PtePointer->Hardware.CacheDisable = CacheDisable;
+    PtePointer->Hardware.WriteThrough = WriteThrough;
+}
+
+/**
+ * Transitions a Page Table Entry (PTE) to invalid state
+ *
+ * @param PointerPte
+ *        Pointer to the page table entry (PTE) to transition.
+ *
+ * @param Protection
+ *        Specifies the protection attribute to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::PageMap::TransitionPte(IN PMMPTE PointerPte,
+                           IN ULONG_PTR Protection)
+{
+    MMPTE TempPte;
+
+    /* Set transition PTE */
+    TempPte = *PointerPte;
+    TempPte.Software.Protection = Protection;
+    TempPte.Software.Prototype = 0;
+    TempPte.Software.Transition = 1;
+    TempPte.Software.Valid = 0;
+
+    /* Write PTE value */
+    *PointerPte = TempPte;
+}
+
+/**
+ * Writes a Page Table Entry (PTE) with the specified value.
+ *
+ * @param Pte
+ *        Pointer to the page table entry (PTE) to write.
+ *
+ * @param Value
+ *        The value to write to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::PageMap::WritePte(IN PMMPTE Pte,
+                      IN MMPTE Value)
+{
+    /* Write PTE value */
+    Pte->Long = Value.Long;
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Entry (PML4).
+ *
+ * @param PdePointer
+ *        Specifies the address of the PDE.
+ *
+ * @return This routine returns the virtual address mapped by the PDE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapBasic::GetPdeVirtualAddress(IN PMMPDE PdePointer)
+{
+    /* Return PDE virtual address */
+    return (PVOID)(((LONGLONG)PdePointer << 34) >> 16);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry (PML4).
+ *
+ * @param PpePointer
+ *        Specifies the address of the PPE.
+ *
+ * @return This routine returns the virtual address mapped by the PPE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapBasic::GetPpeVirtualAddress(IN PMMPPE PpePointer)
+{
+    /* Return PPE virtual address */
+    return (PVOID)(((LONGLONG)PpePointer << 43) >> 16);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Table Entry (PML4).
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the virtual address mapped by the PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapBasic::GetPteVirtualAddress(IN PMMPTE PtePointer)
+{
+    /* Return PTE virtual address */
+    return (PVOID)(((LONGLONG)PtePointer << 25) >> 16);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Extended Page Entry (PML4).
+ *
+ * @param PxePointer
+ *        Specifies the address of the PXE.
+ *
+ * @return This routine returns the virtual address mapped by the PXE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapBasic::GetPxeVirtualAddress(IN PMMPXE PxePointer)
+{
+    /* Return PXE virtual address */
+    return (PVOID)(((LONGLONG)PxePointer << 52) >> 16);
 }
 
 /**
@@ -229,6 +743,78 @@ MM::PageMapBasic::InitializePageMapInfo(VOID)
     PageMapInfo.VaBits = 48;
 }
 
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Entry (PML5).
+ *
+ * @param PdePointer
+ *        Specifies the address of the PDE.
+ *
+ * @return This routine returns the virtual address mapped by the PDE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapXpa::GetPdeVirtualAddress(IN PMMPDE PdePointer)
+{
+    /* Return PDE virtual address */
+    return (PVOID)(((LONGLONG)PdePointer << 25) >> 7);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry (PML5).
+ *
+ * @param PpePointer
+ *        Specifies the address of the PPE.
+ *
+ * @return This routine returns the virtual address mapped by the PPE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapXpa::GetPpeVirtualAddress(IN PMMPPE PpePointer)
+{
+    /* Return PPE virtual address */
+    return (PVOID)(((LONGLONG)PpePointer << 34) >> 7);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Table Entry (PML5).
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the virtual address mapped by the PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapXpa::GetPteVirtualAddress(IN PMMPTE PtePointer)
+{
+    /* Return PTE virtual address */
+    return (PVOID)(((LONGLONG)PtePointer << 16) >> 7);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Extended Page Entry (PML5).
+ *
+ * @param PxePointer
+ *        Specifies the address of the PXE.
+ *
+ * @return This routine returns the virtual address mapped by the PXE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::PageMapXpa::GetPxeVirtualAddress(IN PMMPXE PxePointer)
+{
+    /* Return PXE virtual address */
+    return (PVOID)(((LONGLONG)PxePointer << 43) >> 7);
+}
+
 /**
  * Initializes page map information for XPA paging (PML5).
  *

xtoskrnl/mm/amd64/paging.cc

@@ -25,6 +25,78 @@ MM::Paging::GetExtendedPhysicalAddressingStatus(VOID)
     return ((AR::CpuFunc::ReadControlRegister(4) & CR4_LA57) != 0) ? TRUE : FALSE;
 }
 
+/**
+ * Gets the address of the P5E (Page Map Level 5 Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding P5E.
+ *
+ * @return This routine returns the address of the P5E, or NULLPTR if LA57 is not enabled.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMP5E
+MM::Paging::GetP5eAddress(IN PVOID Address)
+{
+    /* Return PDE address */
+    return PmlRoutines->GetP5eAddress(Address);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Map Level 5 Entry.
+ *
+ * @param P5ePointer
+ *        Specifies the address of the P5E.
+ *
+ * @return This routine returns the virtual address mapped by the P5E, or NULLPTR if LA57 is not enabled.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::Paging::GetP5eVirtualAddress(IN PMMP5E P5ePointer)
+{
+    /* Return PTE virtual address */
+    return PmlRoutines->GetP5eVirtualAddress(P5ePointer);
+}
+
+/**
+ * Gets the address of the PXE (Extended Page Entry), that maps given address.
+ *
+ * @param Address
+ *        Specifies the virtual address for which to retrieve the corresponding PXE.
+ *
+ * @return This routine returns the address of the PXE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPXE
+MM::Paging::GetPxeAddress(IN PVOID Address)
+{
+    /* Return PXE address */
+    return PmlRoutines->GetPxeAddress(Address);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Extended Page Entry.
+ *
+ * @param PxePointer
+ *        Specifies the address of the PXE.
+ *
+ * @return This routine returns the virtual address mapped by the PXE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::Paging::GetPxeVirtualAddress(IN PMMPXE PxePointer)
+{
+    /* Return PXE virtual address */
+    return PmlRoutines->GetPxeVirtualAddress(PxePointer);
+}
+
 /**
  * Fills a section of memory with zeroes like RtlZeroMemory(), but in more efficient way.
  *

xtoskrnl/mm/amd64/pfault.cc

@@ -0,0 +1,26 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/amd64/pfault.cc
+ * DESCRIPTION:     Page fault support for AMD64 architecture
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Evaluates the PDE for for paged pool and per-session mappings.
+ *
+ * @param VirtualAddress
+ *        Specifies the virtual address to verify.
+ *
+ * @return This routine returns ACCESS_VIOLATION regardless PML4 or PML5 is used.
+ */
+XTFASTCALL
+XTSTATUS
+MM::PageFault::CheckPdeForPagedPool(IN PVOID VirtualAddress)
+{
+    /* Return access violation */
+    return STATUS_ACCESS_VIOLATION;
+}

xtoskrnl/mm/amd64/pfn.cc

@@ -0,0 +1,354 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/amd64/pfn.cc
+ * DESCRIPTION:     Physical Frame Number for AMD64 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ *                  Rafal Kupiec <belliash@codingworkshop.eu.org>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Initializes the PFN database by mapping virtual memory and populating entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::InitializePfnDatabase(VOID)
+{
+    PKERNEL_INITIALIZATION_BLOCK InitializationBlock;
+    PLIST_ENTRY ListEntry;
+    PLOADER_MEMORY_DESCRIPTOR Descriptor;
+    PUCHAR PfnDatabaseEnd;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    PMMPTE ValidPte;
+
+    /* Raise runlevel and acquire the PFN lock */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL);
+    KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+    /* Get the kernel initialization block */
+    InitializationBlock = KE::BootInformation::GetInitializationBlock();
+
+    /* Get the memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Get the PFN database size and calculate the end of the PFN database virtual address space */
+    PfnDatabaseEnd = (PUCHAR)MemoryLayout->PfnDatabase + (MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE) - 1;
+
+    /* Get a template PTE for mapping the PFN database pages */
+    ValidPte = MM::Pte::GetValidPte();
+
+    /* Map the Page Directory and Page Directory Pointer tables for the PFN database */
+    MM::Pte::MapPPE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte);
+    MM::Pte::MapPDE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte);
+    MM::Pte::MapPTE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte);
+
+    /* Zero PFN database virtual space */
+    RTL::Memory::ZeroMemory(MemoryLayout->PfnDatabase, MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE);
+
+    /* Initialize the color tables */
+    MM::Colors::InitializeColorTables();
+
+    /* Iterate over memory descriptors to map the PFN database and initialize entries */
+    ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink;
+    while(ListEntry != &InitializationBlock->MemoryDescriptorListHead)
+    {
+        /* Get the descriptor */
+        Descriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_DESCRIPTOR, ListEntry);
+
+        /* Skip invisible memory regions */
+        if(MM::Manager::VerifyMemoryTypeInvisible(Descriptor->MemoryType))
+        {
+            /* Move to the next descriptor and continue */
+            ListEntry = ListEntry->Flink;
+            continue;
+        }
+
+        /* Check if this is the modified free descriptor */
+        if(Descriptor == FreeDescriptor)
+        {
+            /* Switch to the original descriptor */
+            Descriptor = &OriginalFreeDescriptor;
+        }
+
+        /* Check if the free memory block that was split is being processed */
+        if(Descriptor == &OriginalFreeDescriptor)
+        {
+            /* Skip loop processing, free memory is initialized separately */
+            ListEntry = ListEntry->Flink;
+            continue;
+        }
+
+        /* Map PFN database entries for this physical range */
+        ProcessMemoryDescriptor(Descriptor->BasePage, Descriptor->PageCount, Descriptor->MemoryType);
+
+        /* Move to the next descriptor */
+        ListEntry = ListEntry->Flink;
+    }
+
+    /* Initialize PFNs for the free memory */
+    ProcessMemoryDescriptor(FreeDescriptor->BasePage, FreeDescriptor->PageCount, LoaderFree);
+
+    /* Initialize PFNs for the physical pages backing the PFN database */
+    ProcessMemoryDescriptor(OriginalFreeDescriptor.BasePage,
+                            FreeDescriptor->BasePage - OriginalFreeDescriptor.BasePage,
+                            LoaderMemoryData);
+
+    /* Restore original free descriptor */
+    *FreeDescriptor = OriginalFreeDescriptor;
+
+    /* Initialize PFNs backing page tables */
+    InitializePageTablePfns();
+}
+
+/**
+ * Initializes PFN database entries for the system page tables.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::InitializePageTablePfns(VOID)
+{
+    PFN_NUMBER PageFrameIndex;
+    PMMPFN Pfn;
+    ULONG RootLevel;
+    PMMPTE RootPte;
+
+    /* Determine root structure based on paging mode */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* XPA enabled, 5-level paging (LA57) */
+        RootLevel = 5;
+
+        /* Retrieve the PFN of the PML5 table and its virtual base address */
+        PageFrameIndex = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress((PVOID)MM_P5E_LA57_BASE));
+        RootPte = (PMMPTE)MM::Paging::GetP5eAddress(NULLPTR);
+    }
+    else
+    {
+        /* XPA disabled, 4-level paging */
+        RootLevel = 4;
+
+        /* Retrieve the PFN of the PML4 table and its virtual base address */
+        PageFrameIndex = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress((PVOID)MM_PXE_BASE));
+        RootPte = (PMMPTE)MM::Paging::GetPxeAddress(NULLPTR);
+    }
+
+    /* Initialize the PFN entry for the root page table itself */
+    Pfn = GetPfnEntry(PageFrameIndex);
+    if(Pfn)
+    {
+        /* Initialize the PFN entry */
+        Pfn->PteAddress = NULLPTR;
+        Pfn->u1.WsIndex = 0;
+        Pfn->u2.ShareCount = 1;
+        Pfn->u3.e1.CacheAttribute = PfnNonCached;
+        Pfn->u3.e2.ReferenceCount = 1;
+        Pfn->u4.PteFrame = 0;
+    }
+
+    /* Start recursive scan from the top level */
+    if(RootPte)
+    {
+        /* Scan the root page table */
+        ScanPageTable(RootPte, RootLevel);
+    }
+}
+
+/**
+ * Processes a memory descriptor and initializes the corresponding PFN database entries
+ *
+ * @param BasePage
+ *        The starting physical page number of the memory run
+ *
+ * @param PageCount
+ *        The number of pages in the memory run
+ *
+ * @param MemoryType
+ *        The type of memory as reported by the bootloader (e.g., free, ROM, in-use)
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
+                                 IN PFN_NUMBER PageCount,
+                                 IN LOADER_MEMORY_TYPE MemoryType)
+{
+    PFN_NUMBER PageNumber;
+    PMMPDE PointerPde;
+    PMMPFN Pfn;
+
+    /* Check if the memory descriptor describes a free memory region */
+    if(MM::Manager::VerifyMemoryTypeFree(MemoryType))
+    {
+        /* Iterate over each page in this free memory run */
+        for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+        {
+            /* Get the PFN entry for the current page and ensure it is not referenced */
+            Pfn = GetPfnEntry(BasePage + PageNumber);
+            if(Pfn->u3.e2.ReferenceCount == 0)
+            {
+                /* Add the page to the free list to make it available for allocation */
+                LinkFreePage(BasePage + PageNumber);
+            }
+        }
+    }
+    else
+    {
+        /* Handle all other (non-free) memory types */
+        switch(MemoryType)
+        {
+            case LoaderBad:
+                /* This memory is marked as bad and should not be used */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Link the page to the bad pages list */
+                    LinkPage(&BadPagesList, BasePage + PageNumber);
+                }
+                break;
+            case LoaderXIPRom:
+                /* This memory range contains Read-Only Memory (ROM) */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Get the PFN entry for the current ROM page */
+                    Pfn = GetPfnEntry(BasePage + PageNumber);
+
+                    /* Ensure that the page is not already in-use */
+                    if(Pfn->u3.e2.ReferenceCount == 0)
+                    {
+                        /* Get the page directory entry for the current page */
+                        PointerPde = MM::Paging::GetPdeAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+
+                        /* Initialize the PFN entry to represent a ROM page */
+                        Pfn->PteAddress = MM::Paging::GetPteAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+                        Pfn->u1.Flink = 0;
+                        Pfn->u2.ShareCount = 0;
+                        Pfn->u3.e1.CacheAttribute = PfnCached;
+                        Pfn->u3.e1.PageLocation = 0;
+                        Pfn->u3.e1.PrototypePte = 1;
+                        Pfn->u3.e1.Rom = 1;
+                        Pfn->u3.e2.ReferenceCount = 0;
+                        Pfn->u4.InPageError = 0;
+                        Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde);
+                    }
+                }
+                break;
+            default:
+                /* All other types are considered in-use (ie, by the kernel, ACPI, etc) */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Get the PFN entry for the current in-use page */
+                    Pfn = GetPfnEntry(BasePage + PageNumber);
+
+                    /* Ensure that the page is not already in-use */
+                    if(Pfn->u3.e2.ReferenceCount == 0)
+                    {
+                        /* Get the page directory entry for the current page */
+                        PointerPde = MM::Paging::GetPdeAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+
+                        /* Initialize the PFN entry to represent an in-use page and prevent it from being allocated */
+                        Pfn->PteAddress = MM::Paging::GetPteAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+                        Pfn->u2.ShareCount++;
+                        Pfn->u3.e1.CacheAttribute = PfnCached;
+                        Pfn->u3.e1.PageLocation = ActiveAndValid;
+                        Pfn->u3.e2.ReferenceCount = 1;
+                        Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde);
+                    }
+                }
+                break;
+        }
+    }
+}
+
+/**
+ * Recursively scans a page table to initialize PFN database entries for active pages.
+ *
+ * @param PointerPte
+ *        Pointer to the base of the page table to scan.
+ *
+ * @param Level
+ *        The paging level of the table being scanned.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::ScanPageTable(IN PMMPTE PointerPte,
+                       IN ULONG Level)
+{
+    PVOID Address;
+    ULONG Index;
+    PMMPTE NextLevelPte;
+    ULONG PtesPerPage;
+
+    /* Get the number of PTEs per page */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+
+    /* Iterate through all entries in the current page table */
+    for(Index = 0; Index < PtesPerPage; Index++)
+    {
+        /* Check if the page table entry is present */
+        if(MM::Paging::PteValid(PointerPte))
+        {
+            /* Mark the PFN pointed to by this entry as active */
+            LinkPfnForPageTable(MM::Paging::GetPageFrameNumber(PointerPte), PointerPte);
+
+            /* Recurse to the next level, if this is not a leaf node (PTE) */
+            if(Level > 1)
+            {
+                /* Calculate the virtual address mapped by this entry to find the next table */
+                switch(Level)
+                {
+                    case 5:
+                        /* Calculate PXE */
+                        Address = MM::Paging::GetP5eVirtualAddress((PMMP5E)PointerPte);
+                        NextLevelPte = (PMMPTE)MM::Paging::GetPxeAddress(Address);
+                        break;
+                    case 4:
+                        /* Calculate PPE */
+                        Address = MM::Paging::GetPxeVirtualAddress((PMMPXE)PointerPte);
+                        NextLevelPte = (PMMPTE)MM::Paging::GetPpeAddress(Address);
+                        break;
+                    case 3:
+                        /* Calculate PDE */
+                        Address = MM::Paging::GetPpeVirtualAddress((PMMPPE)PointerPte);
+                        NextLevelPte = (PMMPTE)MM::Paging::GetPdeAddress(Address);
+                        break;
+                    case 2:
+                        /* Calculate PTE */
+                        Address = MM::Paging::GetPdeVirtualAddress((PMMPDE)PointerPte);
+                        NextLevelPte = MM::Paging::GetPteAddress(Address);
+                        break;
+                    default:
+                        /* Nothing to calculate, return NULLPTR */
+                        NextLevelPte = NULLPTR;
+                        break;
+                }
+
+                /* Recurse deeper if not at the bottom level (PTE) already */
+                if(NextLevelPte)
+                {
+                    /* Recursively scan the next level page table */
+                    ScanPageTable(NextLevelPte, Level - 1);
+                }
+            }
+        }
+
+        /* Move to the next entry in the current table */
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+}

xtoskrnl/mm/amd64/pte.cc

@@ -0,0 +1,295 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/amd64/pte.cc
+ * DESCRIPTION:     Page Table Entry (PTE) for AMD64 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Checks if the virtual address is valid and mapped in the page tables.
+ *
+ * @param VirtualAddress
+ *        The virtual address to check.
+ *
+ * @return This routine returns TRUE if the address is valid, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Pte::AddressValid(IN PVOID VirtualAddress)
+{
+    /* Check XPA status */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Check if the P5E is valid */
+        if(!MM::Paging::PteValid(MM::Paging::GetP5eAddress(VirtualAddress)))
+        {
+            /* Invalid P5E, return FALSE */
+            return FALSE;
+        }
+    }
+
+    /* Check if PXE, PPE, PDE and PTE are valid */
+    if(!MM::Paging::PteValid(MM::Paging::GetPxeAddress(VirtualAddress)) ||
+       !MM::Paging::PteValid(MM::Paging::GetPpeAddress(VirtualAddress)) ||
+       !MM::Paging::PteValid(MM::Paging::GetPdeAddress(VirtualAddress)) ||
+       !MM::Paging::PteValid(MM::Paging::GetPteAddress(VirtualAddress)))
+    {
+        /* Invalid PXE, PPE, PDE or PTE, return FALSE */
+        return FALSE;
+    }
+
+    /* Address is valid, return TRUE */
+    return TRUE;
+}
+
+/**
+ * Retrieves the base virtual address of the system PTEs.
+ *
+ * @return This routine returns a pointer to the first PTE in the system PTE space.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Pte::GetSystemPteBaseAddress(VOID)
+{
+    PMMMEMORY_LAYOUT MemoryLayout;
+
+    /* Retrieve the system's memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Determine the base address for system PTEs based on the paging mode */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* For 5-level paging, system PTEs start at the beginning of system space */
+        return MM::Paging::GetPteAddress((PVOID)MemoryLayout->NonPagedSystemPoolStart);
+    }
+    else
+    {
+        /* For 4-level paging, system PTEs start at the legacy KSEG0_BASE */
+        return MM::Paging::GetPteAddress((PVOID)KSEG0_BASE);
+    }
+}
+
+/**
+ * Performs the initial setup of the system's page table hierarchy.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::InitializePageTable(VOID)
+{
+    PMMPTE EndSpacePte, PointerPte;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    PVOID MappingRange;
+    MMPTE TemplatePte;
+    BOOLEAN Xpa;
+
+    /* Retrieve current paging mode and memory layout */
+    Xpa = MM::Paging::GetXpaStatus();
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Enable the Global Paging (PGE) feature */
+    AR::CpuFunc::WriteControlRegister(4, AR::CpuFunc::ReadControlRegister(4) | CR4_PGE);
+
+    /* Check XPA status */
+    if(Xpa)
+    {
+        /* Get the PML5 user-space range if 5-level paging is active */
+        PointerPte = MM::Paging::GetP5eAddress(0);
+        EndSpacePte = MM::Paging::GetP5eAddress(MemoryLayout->UserSpaceEnd);
+    }
+    else
+    {
+        /* Otherwise, get the PML4 user-space range for 4-level paging */
+        PointerPte = MM::Paging::GetPxeAddress(0);
+        EndSpacePte = MM::Paging::GetPxeAddress(MemoryLayout->UserSpaceEnd);
+    }
+
+    /* Clear all top-level entries mapping the user address space */
+    while(PointerPte <= EndSpacePte)
+    {
+        MM::Paging::ClearPte(PointerPte);
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+
+    /* Flush the TLB to invalidate all non-global entries */
+    AR::CpuFunc::FlushTlb();
+
+    /* Create a template PTE for mapping kernel pages */
+    MM::Paging::ClearPte(&TemplatePte);
+    MM::Paging::SetPte(&TemplatePte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE);
+
+    /* Check XPA status */
+    if(Xpa)
+    {
+        /* Map the kernel's PML5 entries if 5-level paging is active */
+        MM::Pte::MapP5E(MemoryLayout->HyperSpaceStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte);
+    }
+
+    /* Map the kernel's PML4 entries */
+    MM::Pte::MapPXE(MemoryLayout->HyperSpaceStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte);
+
+    /* Calculate the end address of the hyperspace working set mapping */
+    MappingRange = (PVOID)((ULONG_PTR)MemoryLayout->HyperSpaceStart + MM_HYPERSPACE_PAGE_COUNT * MM_PAGE_SIZE);
+
+    /* Map the PDPT entries for paged pool and hyperspace */
+    MM::Pte::MapPPE(MemoryLayout->PagedPoolStart, MemoryLayout->PagedPoolEnd, &ValidPte);
+    MM::Pte::MapPPE(MemoryLayout->HyperSpaceStart, MemoryLayout->HyperSpaceEnd, &ValidPte);
+
+    /* Map the PDEs for the hyperspace working set */
+    MM::Pte::MapPDE(MemoryLayout->HyperSpaceStart, MappingRange, &ValidPte);
+
+    /* Set the hyperspace working set's PTE with the total PTE count */
+    MM::Paging::SetPte(MM::Paging::GetPteAddress((PVOID)MemoryLayout->HyperSpaceStart), MM_HYPERSPACE_PAGE_COUNT, 0);
+}
+
+/**
+ * Maps a range of virtual addresses at the P5E (PML5) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplateP5e
+ *        A template P5E to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapP5E(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMP5E TemplateP5e)
+{
+    PMMP5E EndSpace, PointerP5e;
+
+    /* Get P5E addresses */
+    PointerP5e = MM::Paging::GetP5eAddress(StartAddress);
+    EndSpace = MM::Paging::GetP5eAddress(EndAddress);
+
+    /* Iterate over all P5Es */
+    while(PointerP5e <= EndSpace)
+    {
+        /* Check if P5E is already mapped */
+        if(!MM::Paging::PteValid(PointerP5e))
+        {
+            /* Map P5E */
+            MM::Paging::SetPte(TemplateP5e, MM::Pfn::AllocateBootstrapPages(1), 0);
+            *PointerP5e = *TemplateP5e;
+
+            /* Clear the page table */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerP5e), MM_PAGE_SIZE);
+        }
+
+        /* Get next table entry */
+        PointerP5e = MM::Paging::GetNextPte(PointerP5e);
+    }
+}
+
+/**
+ * Maps a range of virtual addresses at the PPE (Page Directory Pointer Entry) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplatePpe
+ *        A template PPE to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapPPE(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMPPE TemplatePpe)
+{
+    PMMPPE EndSpace, PointerPpe;
+
+    /* Get PPE addresses */
+    PointerPpe = MM::Paging::GetPpeAddress(StartAddress);
+    EndSpace = MM::Paging::GetPpeAddress(EndAddress);
+
+    /* Iterate over all PPEs */
+    while(PointerPpe <= EndSpace)
+    {
+        /* Check if PPE is already mapped */
+        if(!MM::Paging::PteValid(PointerPpe))
+        {
+            /* Map PPE */
+            MM::Paging::SetPte(TemplatePpe, MM::Pfn::AllocateBootstrapPages(1), 0);
+            *PointerPpe = *TemplatePpe;
+
+            /* Clear the page table */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPpe), MM_PAGE_SIZE);
+        }
+
+        /* Get next table entry */
+        PointerPpe = MM::Paging::GetNextPte(PointerPpe);
+    }
+}
+
+/**
+ * Maps a range of virtual addresses at the PXE (PML4) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplatePxe
+ *        A template PXE to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapPXE(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMPXE TemplatePxe)
+{
+    PMMPXE EndSpace, PointerPxe;
+
+    /* Get PXE addresses */
+    PointerPxe = MM::Paging::GetPxeAddress(StartAddress);
+    EndSpace = MM::Paging::GetPxeAddress(EndAddress);
+
+    /* Iterate over all PTEs */
+    while(PointerPxe <= EndSpace)
+    {
+        /* Check if PTE is already mapped */
+        if(!MM::Paging::PteValid(PointerPxe))
+        {
+            /* Map PTE */
+            MM::Paging::SetPte(TemplatePxe, MM::Pfn::AllocateBootstrapPages(1), 0);
+            *PointerPxe = *TemplatePxe;
+
+            /* Clear the page table */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPxe), MM_PAGE_SIZE);
+        }
+
+        /* Get next table entry */
+        PointerPxe = MM::Paging::GetNextPte(PointerPxe);
+    }
+}

xtoskrnl/mm/colors.cc

@@ -0,0 +1,178 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/colors.cc
+ * DESCRIPTION:     Memory manager page coloring subsystem
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Computes & initializes the system's page coloring.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Colors::ComputePageColoring(VOID)
+{
+    UNIMPLEMENTED;
+
+    /* Compute L2 paging colors and mask */
+    PagingColors = MM_PAGING_COLORS;
+    PagingColorsMask = PagingColors - 1;
+}
+
+/**
+ * Retrieves a pointer to the color table for a specific page list and color.
+ *
+ * @param PageList
+ *        The page list type (e.g., FreePageList, ZeroedPageList).
+ *
+ * @param Color
+ *        Supplies the specific color index.
+ *
+ * @return This routine returns a pointer to the corresponding MMCOLOR_TABLES structure.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMCOLOR_TABLES
+MM::Colors::GetFreePages(IN MMPAGELISTS PageList,
+                         IN ULONG Color)
+{
+    /* Return a pointer to the requested color table entry */
+    return &FreePages[PageList][Color];
+}
+
+/**
+ * Retrieves a pointer to the modified pages list for a specific color.
+ *
+ * @param Color
+ *        Supplies the specific color index.
+ *
+ * @return This routine returns a pointer to the corresponding MMPFNLIST structure.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPFNLIST
+MM::Colors::GetModifiedPages(IN ULONG Color)
+{
+    return &ModifiedPages[Color];
+}
+
+/**
+ * Retrieves the next available color for page coloring.
+ *
+ * @return This routine returns the next color value, ensuring it stays within the valid color range.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Colors::GetNextColor(VOID)
+{
+    /* Increment the color counter and wrap it around using the mask */
+    return ((++PagingColors) & PagingColorsMask);
+}
+
+/**
+ * Retrieves the total number of page colors configured in the system.
+ *
+ * @return This routine returns the number of page colors.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Colors::GetPagingColors(VOID)
+{
+    /* Return the total number of page colors */
+    return PagingColors;
+}
+
+/**
+ * Retrieves the bitmask used for calculating a page's color.
+ *
+ * @return This routine returns the page color mask.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Colors::GetPagingColorsMask(VOID)
+{
+    /* Return the mask used for page coloring calculations */
+    return PagingColorsMask;
+}
+
+/**
+ * Initializes the data structures for page coloring.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Colors::InitializeColorTables(VOID)
+{
+    PMMMEMORY_LAYOUT MemoryLayout;
+    PMMPTE PointerPte, LastPte;
+    ULONG Color;
+    PMMPTE ValidPte;
+
+    /* Get the memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Set the base address of the color tables to start right after the PFN database */
+    FreePages[0] = (PMMCOLOR_TABLES)&((PMMPFN)MemoryLayout->PfnDatabase)[MM::Pfn::GetHighestPhysicalPage() + 1];
+
+    /* Calculate the virtual address range for both color tables */
+    PointerPte = MM::Paging::GetPteAddress(&FreePages[0][0]);
+    LastPte = MM::Paging::GetPteAddress((PVOID)((ULONG_PTR)FreePages[0] +
+                                        (2 * PagingColors * sizeof(MMCOLOR_TABLES)) - 1));
+
+    /* Get a pointer to a PTE template */
+    ValidPte = MM::Pte::GetValidPte();
+
+    /* Ensure the entire virtual address range for the color tables is mapped */
+    while(PointerPte <= LastPte)
+    {
+        /* Check if a page in the range is not mapped */
+        if(!MM::Paging::PteValid(PointerPte))
+        {
+            /* Use the bootstrap allocator to get a physical page */
+            MM::Paging::SetPte(ValidPte, MM::Pfn::AllocateBootstrapPages(1), 0);
+            *PointerPte = *ValidPte;
+
+            /* Zero out the newly mapped page */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPte), MM_PAGE_SIZE);
+        }
+
+        /* Move to the next PTE in the range */
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+
+    /* Set the pointer for the second list */
+    FreePages[1] = &FreePages[0][PagingColors];
+
+    /* Initialize all entries in both color tables */
+    for(Color = 0; Color < PagingColors; Color++)
+    {
+        /* Initialize the FreePageList entry for the current color */
+        FreePages[FreePageList][Color].Flink = MAXULONG_PTR;
+        FreePages[FreePageList][Color].Blink = (PVOID)MAXULONG_PTR;
+        FreePages[FreePageList][Color].Count = 0;
+
+        /* Initialize the ZeroedPageList entry for the current color */
+        FreePages[ZeroedPageList][Color].Flink = MAXULONG_PTR;
+        FreePages[ZeroedPageList][Color].Blink = (PVOID)MAXULONG_PTR;
+        FreePages[ZeroedPageList][Color].Count = 0;
+    }
+}

xtoskrnl/mm/data.cc

@@ -9,6 +9,18 @@
 #include <xtos.hh>
 
 
+/* Array of free page lists segregated by cache color */
+PMMCOLOR_TABLES MM::Colors::FreePages[FreePageList + 1];
+
+/* Array of modified pages segregated by cache color */
+MMPFNLIST MM::Colors::ModifiedPages[MM_PAGING_COLORS] = {{0, ModifiedPageList, MAXULONG_PTR, MAXULONG_PTR}};
+
+/* Number of supported page colors */
+ULONG MM::Colors::PagingColors;
+
+/* Bitmask used to calculate the cache color index */
+ULONG MM::Colors::PagingColorsMask;
+
 /* Allocation descriptors dedicated for hardware layer */
 LOADER_MEMORY_DESCRIPTOR MM::HardwarePool::HardwareAllocationDescriptors[MM_HARDWARE_ALLOCATION_DESCRIPTORS];
 
@@ -18,23 +30,81 @@ PVOID MM::HardwarePool::HardwareHeapStart = MM_HARDWARE_HEAP_START_ADDRESS;
 /* Number of used hardware allocation descriptors */
 ULONG MM::HardwarePool::UsedHardwareAllocationDescriptors = 0;
 
-/* Biggest free memory descriptor */
-PLOADER_MEMORY_DESCRIPTOR MM::Init::FreeDescriptor;
-
-/* Highest physical page number */
-ULONG_PTR MM::Init::HighestPhysicalPage;
-
-/* Lowest physical page number */
-ULONG_PTR MM::Init::LowestPhysicalPage = -1;
-
-/* Number of physical pages */
-ULONG MM::Init::NumberOfPhysicalPages;
-
-/* Old biggest free memory descriptor */
-LOADER_MEMORY_DESCRIPTOR MM::Init::OldFreeDescriptor;
-
 /* Processor structures data (THIS IS A TEMPORARY HACK) */
 UCHAR MM::KernelPool::ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE] = {{0}};
 
+/* Global structure describing the virtual memory layout of the system */
+MMMEMORY_LAYOUT MM::Manager::MemoryLayout;
+
+/* Total number of PTEs reserved for system space mapping */
+PFN_NUMBER MM::Manager::NumberOfSystemPtes;
+
 /* Instance of the page map routines for the current PML level */
 MM::PPAGEMAP MM::Paging::PmlRoutines;
+
+/* Total number of physical pages available for allocation */
+PFN_NUMBER MM::Pfn::AvailablePages;
+
+/* Head of the list containing physical pages marked as defective */
+MMPFNLIST MM::Pfn::BadPagesList = {0, BadPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* Biggest free memory descriptor */
+PLOADER_MEMORY_DESCRIPTOR MM::Pfn::FreeDescriptor;
+
+/* List containing free physical pages */
+MMPFNLIST MM::Pfn::FreePagesList = {0, FreePageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* Highest physical page number */
+ULONG_PTR MM::Pfn::HighestPhysicalPage;
+
+/* Lowest physical page number */
+ULONG_PTR MM::Pfn::LowestPhysicalPage = -1;
+
+/* List containing modified pages */
+MMPFNLIST MM::Pfn::ModifiedPagesList = {0, ModifiedPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* List containing modified pages mapped as read-only */
+MMPFNLIST MM::Pfn::ModifiedReadOnlyPagesList = {0, ModifiedReadOnlyPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* Number of physical pages */
+ULONGLONG MM::Pfn::NumberOfPhysicalPages;
+
+/* Old biggest free memory descriptor */
+LOADER_MEMORY_DESCRIPTOR MM::Pfn::OriginalFreeDescriptor;
+
+/* Array of pointers to PFN lists */
+PMMPFNLIST MM::Pfn::PageLocationList[] = {&ZeroedPagesList,
+                                          &FreePagesList,
+                                          &StandbyPagesList,
+                                          &ModifiedPagesList,
+                                          &ModifiedReadOnlyPagesList,
+                                          &BadPagesList,
+                                          NULLPTR,
+                                          NULLPTR};
+
+/* List containing pages mapped as Read-Only (ROM) */
+MMPFNLIST MM::Pfn::RomPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* List containing standby pages (clean, can be reclaimed or repurposed) */
+MMPFNLIST MM::Pfn::StandbyPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* List containing free physical pages that have been zeroed out */
+MMPFNLIST MM::Pfn::ZeroedPagesList = {0, ZeroedPageList, MAXULONG_PTR, MAXULONG_PTR};
+
+/* Array of lists for available System PTEs, separated by pool type */
+MMPTE MM::Pte::FirstSystemFreePte[MaximumPtePoolTypes];
+
+/* Virtual base address of the System PTE space */
+PMMPTE MM::Pte::SystemPteBase;
+
+/* End addresses for the System PTE ranges */
+PMMPTE MM::Pte::SystemPtesEnd[MaximumPtePoolTypes];
+
+/* Start addresses for the System PTE ranges */
+PMMPTE MM::Pte::SystemPtesStart[MaximumPtePoolTypes];
+
+/* Total count of available System PTEs */
+PFN_COUNT MM::Pte::TotalSystemFreePtes[MaximumPtePoolTypes];
+
+/* Template PTE entry containing standard flags for a valid, present kernel page */
+MMPTE MM::Pte::ValidPte;

xtoskrnl/mm/hlpool.cc

@@ -170,7 +170,7 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress,
 {
     PVOID BaseAddress, ReturnAddress;
     PFN_NUMBER MappedPages;
-    PHARDWARE_PTE PtePointer;
+    PMMPTE PtePointer;
 
     /* Initialize variables */
     BaseAddress = HardwareHeapStart;
@@ -189,7 +189,7 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress,
         }
 
         /* Get PTE pointer and advance to next page */
-        PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(ReturnAddress);
+        PtePointer = MM::Paging::GetPteAddress(ReturnAddress);
         ReturnAddress = (PVOID)((ULONG_PTR)ReturnAddress + MM_PAGE_SIZE);
 
         /* Check if PTE is valid */
@@ -219,10 +219,10 @@ MM::HardwarePool::MapHardwareMemory(IN PHYSICAL_ADDRESS PhysicalAddress,
     while(MappedPages--)
     {
         /* Get PTE pointer */
-        PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(BaseAddress);
+        PtePointer = MM::Paging::GetPteAddress(BaseAddress);
 
         /* Fill the PTE */
-        MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), TRUE);
+        MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), MM_PTE_READWRITE);
 
         /* Advance to the next address */
         PhysicalAddress.QuadPart += MM_PAGE_SIZE;
@@ -259,18 +259,18 @@ VOID
 MM::HardwarePool::MarkHardwareMemoryWriteThrough(IN PVOID VirtualAddress,
                                                  IN PFN_NUMBER PageCount)
 {
-    PHARDWARE_PTE PtePointer;
+    PMMPTE PtePointer;
     PFN_NUMBER Page;
 
     /* Get PTE address from virtual address */
-    PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress);
+    PtePointer = MM::Paging::GetPteAddress(VirtualAddress);
 
     /* Iterate through mapped pages */
     for(Page = 0; Page < PageCount; Page++)
     {
         /* Mark pages as CD/WT */
         MM::Paging::SetPteCaching(PtePointer, TRUE, TRUE);
-        PtePointer++;
+        MM::Paging::GetNextEntry(PtePointer);
     }
 }
 
@@ -296,13 +296,13 @@ MM::HardwarePool::RemapHardwareMemory(IN PVOID VirtualAddress,
                                       IN PHYSICAL_ADDRESS PhysicalAddress,
                                       IN BOOLEAN FlushTlb)
 {
-    PHARDWARE_PTE PtePointer;
+    PMMPTE PtePointer;
 
     /* Get PTE address from virtual address */
-    PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress);
+    PtePointer = MM::Paging::GetPteAddress(VirtualAddress);
 
     /* Remap the PTE */
-    MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), TRUE);
+    MM::Paging::SetPte(PtePointer, (PFN_NUMBER)(PhysicalAddress.QuadPart >> MM_PAGE_SHIFT), MM_PTE_READWRITE);
 
     /* Check if TLB needs to be flushed */
     if(FlushTlb)
@@ -334,7 +334,7 @@ MM::HardwarePool::UnmapHardwareMemory(IN PVOID VirtualAddress,
                                       IN PFN_NUMBER PageCount,
                                       IN BOOLEAN FlushTlb)
 {
-    PHARDWARE_PTE PtePointer;
+    PMMPTE PtePointer;
     PFN_NUMBER Page;
 
     /* Check if address is valid hardware memory */
@@ -348,7 +348,7 @@ MM::HardwarePool::UnmapHardwareMemory(IN PVOID VirtualAddress,
     VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress & ~(MM_PAGE_SIZE - 1));
 
     /* Get PTE address from virtual address */
-    PtePointer = (PHARDWARE_PTE)MM::Paging::GetPteAddress(VirtualAddress);
+    PtePointer = MM::Paging::GetPteAddress(VirtualAddress);
 
     /* Iterate through mapped pages */
     for(Page = 0; Page < PageCount; Page++)

xtoskrnl/mm/i686/alloc.cc

@@ -0,0 +1,17 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/i686/alloc.cc
+ * DESCRIPTION:     Memory manager pool allocation
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+XTAPI
+VOID
+MM::Allocator::InitializeNonPagedPool(VOID)
+{
+    UNIMPLEMENTED;
+}

xtoskrnl/mm/i686/init.cc

@@ -1,25 +0,0 @@
-/**
- * PROJECT:         ExectOS
- * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/mm/i686/init.cc
- * DESCRIPTION:     Architecture specific Memory Manager initialization routines
- * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
- *                  Aiken Harris <harraiken91@gmail.com>
- */
-
-#include <xtos.hh>
-
-
-/**
- * Performs architecture specific initialization of the XTOS Memory Manager.
- *
- * @return This routine does not return any value.
- *
- * @since XT 1.0
- */
-XTAPI
-VOID
-MM::Init::InitializeArchitecture(VOID)
-{
-    UNIMPLEMENTED;
-}

xtoskrnl/mm/i686/mmgr.cc

@@ -0,0 +1,445 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/i686/mmgr.cc
+ * DESCRIPTION:     Memory Manager
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Calculates the maximum possible size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR MaximumNonPagedPoolSize;
+    ULONGLONG PhysicalPages;
+
+    /* Get number of physical pages */
+    PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages();
+
+    /* Start with 1MiB and reserve space for PFN database */
+    MaximumNonPagedPoolSize = 1048576;
+
+    /* Check if system has at least 512MiB of physical memory */
+    if(PhysicalPages >= 126976)
+    {
+        /* Add 200KiB for each MiB above 4MiB */
+        MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 204800;
+
+        /* Check if non-paged pool has at least 128MiB */
+        if(MaximumNonPagedPoolSize < 134217728)
+        {
+            /* Expand non-paged pool size to 128MiB */
+            MaximumNonPagedPoolSize = 134217728;
+        }
+    }
+    else
+    {
+        /* Add 400KiB for each MiB above 4MiB */
+        MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 409600;
+    }
+
+    /* Check if non-paged pool does not exceed 256MiB */
+    if(MaximumNonPagedPoolSize > 268435456)
+    {
+        /* Limit non-paged pool size to 256MiB */
+        MaximumNonPagedPoolSize = 268435456;
+    }
+
+    /* Return maximum non-paged pool size in pages */
+    *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize);
+}
+
+/**
+ * Calculates the size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR NonPagedPoolSize;
+    ULONGLONG PhysicalPages;
+
+    /* Get number of physical pages */
+    PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages();
+
+    /* Verify if system has less than 256MiB of physical memory */
+    if(PhysicalPages <= 65536)
+    {
+        /* Reduce initial non-paged pool size to 2MiB to save memory */
+        NonPagedPoolSize = 2097152;
+    }
+    else
+    {
+        /* Start with 256KiB and add 32KiB for each MiB above 4MiB */
+        NonPagedPoolSize = 262144 + (((PhysicalPages - 1024) / 256) * 32768);
+
+        if(NonPagedPoolSize > 134217728)
+        {
+            /* Limit non-paged pool size to 128MiB */
+            NonPagedPoolSize = 134217728;
+        }
+    }
+
+    /* Return non-paged pool size in pages, aligned down to page size boundary */
+    *PoolSize = SIZE_TO_PAGES(ROUND_DOWN(NonPagedPoolSize, MM_PAGE_SIZE));
+}
+
+/**
+ * Calculates the size of the paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR PagedPoolSize, PteCount;
+    ULONG PtesPerPage;
+
+    /* Start with maximum non-paged pool size */
+    ComputeMaximumNonPagedPoolSize(&PagedPoolSize);
+    PagedPoolSize *= MM_PAGE_SIZE;
+
+    /* Check XPA status */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Four times the non-paged pool size on PAE-enabled systems */
+        PagedPoolSize *= 4;
+    }
+    else
+    {
+        /* Double the non-paged pool size on PAE-disabled systems */
+        PagedPoolSize *= 2;
+    }
+
+    /* Check if paged pool does not overlap non-paged pool */
+    if(PagedPoolSize > (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart)
+    {
+        /* Limit paged pool size to maximum possible */
+        PagedPoolSize = (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart;
+    }
+
+    /* Ensure that paged pool size is at least 32MiB */
+    if(PagedPoolSize < 33554432)
+    {
+        /* Increase paged pool size to at least 32MiB */
+        PagedPoolSize = 33554432;
+    }
+
+    /* Get the number of PTEs per page and calculate size of paged pool */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+    PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage);
+
+    /* Return paged pool size */
+    *PoolSize = (PFN_NUMBER)(PteCount * PtesPerPage);
+}
+
+/**
+ * Calculates the size of the session space.
+ *
+ * @param SpaceSize
+ *        A pointer to a variable that will receive the number of pages available by the session space.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize)
+{
+    PFN_NUMBER SessionSpaceSize;
+
+    /* Session Pool, Session View, Session Image, Session Working Set and System View takes 108MiB */
+    SessionSpaceSize = 113246208;
+
+    /* Return number of pages used by the session space */
+    *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE;
+}
+
+/**
+ * Calculates the size of the system PTEs.
+ *
+ * @param PteSize
+ *        A pointer to a variable that will receive the number of system PTEs.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize)
+{
+    PFN_NUMBER SystemPteSize;
+
+    /* Check if system has less than 19MiB of physical memory */
+    if(MM::Pfn::GetNumberOfPhysicalPages() < 4864)
+    {
+        /* Set minimal system PTE size */
+        SystemPteSize = 7000;
+    }
+    else
+    {
+        /* Use standard system PTE size */
+        SystemPteSize = 11000;
+
+        /* Check if system has more than 32MiB of physical memory */
+        if(MM::Pfn::GetNumberOfPhysicalPages() > 8192)
+        {
+            /* Double system PTE size */
+            SystemPteSize *= 2;
+
+            /* Check if system has more than 256MiB of physical memory */
+            if(MM::Pfn::GetNumberOfPhysicalPages() > 65536)
+            {
+                /* Double system PTE size */
+                SystemPteSize *= 2;
+            }
+        }
+    }
+
+    /* Return system PTE size */
+    *PteSize = SystemPteSize;
+}
+
+/**
+ * Dumps the kernel's memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::DumpMemoryLayout(VOID)
+{
+    /* Dump memory layout */
+    DebugPrint(L"System with %zu MiB of installed memory:\n"
+               L"User Space:               %.8P - %.8P\n"
+               L"Loader Mappings:          %.8P - %.8P\n"
+               L"PFN Database:             %.8P - %.8P\n"
+               L"Non-Paged Pool:           %.8P - %.8P\n"
+               L"Session Space:            %.8P - %.8P\n"
+               L"PTE Space:                %.8P - %.8P\n"
+               L"Hyper Space:              %.8P - %.8P\n"
+               L"System Working Set:       %.8P - %.8P\n"
+               L"System Cache:             %.8P - %.8P\n"
+               L"Paged Pool:               %.8P - %.8P\n"
+               L"Non-Paged System Pool:    %.8P - %.8P\n"
+               L"Non-Paged Expansion Pool: %.8P - %.8P\n"
+               L"Shared System Page:       %.8P - %.8P\n"
+               L"Hardware Pool:            %.8P - %.8P\n",
+               GetInstalledMemorySize(),
+               MemoryLayout.UserSpaceStart,
+               MemoryLayout.UserSpaceEnd,
+               MemoryLayout.LoaderMappingsStart,
+               MemoryLayout.LoaderMappingsEnd,
+               MemoryLayout.PfnDatabase,
+               (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE),
+               MemoryLayout.NonPagedPoolStart,
+               MemoryLayout.NonPagedPoolEnd,
+               MemoryLayout.SessionSpaceStart,
+               MemoryLayout.SessionSpaceEnd,
+               MemoryLayout.PteSpaceStart,
+               MemoryLayout.PteSpaceEnd,
+               MemoryLayout.HyperSpaceStart,
+               MemoryLayout.HyperSpaceEnd,
+               MemoryLayout.SystemWorkingSetStart,
+               MemoryLayout.SystemWorkingSetEnd,
+               MemoryLayout.SystemCacheStart,
+               MemoryLayout.SystemCacheEnd,
+               MemoryLayout.PagedPoolStart,
+               MemoryLayout.PagedPoolEnd,
+               MemoryLayout.NonPagedSystemPoolStart,
+               MemoryLayout.NonPagedSystemPoolEnd,
+               MemoryLayout.NonPagedExpansionPoolStart,
+               MemoryLayout.NonPagedExpansionPoolEnd,
+               MemoryLayout.SharedSystemPageStart,
+               MemoryLayout.SharedSystemPageEnd,
+               MemoryLayout.HardwarePoolStart,
+               MemoryLayout.HardwarePoolEnd);
+}
+
+/**
+ * Initializes the kernel's virtual memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::InitializeMemoryLayout(VOID)
+{
+    PFN_NUMBER MaximumNonPagedPoolSize;
+    ULONG_PTR PfnDatabaseEnd;
+
+    /* Check if 3-level paging (PAE) is enabled */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Set PML3 base address */
+        MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE;
+
+        /* Define memory layout for 3-level paging */
+        MemoryLayout.NonCanonicalStart = (PVOID)0x00000000;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000;
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000;
+        MemoryLayout.UserSpaceStart = (PVOID)0x00010000;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceStart = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xC07FFFFF;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xC0800000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xC0BFFFFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF;
+        MemoryLayout.SystemCacheStart = (PVOID)0xC1000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xE1000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF;
+    }
+    else
+    {
+        /* Set PML2 base address */
+        MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE;
+
+        /* Define memory layout for 2-level paging */
+        MemoryLayout.NonCanonicalStart = (PVOID)0x00000000;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000;
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000;
+        MemoryLayout.UserSpaceStart = (PVOID)0x00010000;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceStart = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xC03FFFFF;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xC0400000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xC07FFFFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF;
+        MemoryLayout.SystemCacheStart = (PVOID)0xC1000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xE1000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF;
+    }
+
+    /* Compute allocation size for the PFN database */
+    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
+
+    /* Compute boot image size */
+    ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize);
+
+    /* Compute session space size */
+    ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize);
+
+    /* Update loader mappings space end address */
+    MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONG_PTR)MemoryLayout.LoaderMappingsStart +
+                                             MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE);
+
+    /* Update session space start address */
+    MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd -
+                                             MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE);
+
+    /* Compute system PTE size */
+    ComputeSystemPteSize(&NumberOfSystemPtes);
+
+    /* Compute the initial and maximum non-paged pool sizes */
+    ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize);
+    ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize);
+
+    /* Compute paged pool size */
+    ComputePagedPoolSize(&MemoryLayout.PagedPoolSize);
+
+    /* Position the PFN database right after the loader mappings */
+    MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.LoaderMappingsEnd;
+
+    /* Compute the PFN database end address */
+    PfnDatabaseEnd = (ULONG_PTR)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+
+    /* Position the initial non-paged pool immediately after the PFN database */
+    MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd;
+
+    /* Check if the calculated non-paged pool size fits in the KVA */
+    if((MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE) >
+       ((ULONG_PTR)MemoryLayout.SessionSpaceStart - (ULONG_PTR)MemoryLayout.NonPagedPoolStart))
+    {
+        /* Set the final size for the non-paged pool */
+        MemoryLayout.NonPagedPoolSize = ((ULONG_PTR)MemoryLayout.NonPagedPoolEnd -
+                                        (ULONG_PTR)MemoryLayout.NonPagedPoolStart) / MM_PAGE_SIZE;
+    }
+
+    /* Set the final non-paged pool end address */
+    MemoryLayout.NonPagedPoolEnd = (PVOID)((ULONG_PTR)MemoryLayout.NonPagedPoolStart +
+                                           MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE);
+
+    /* Check if non-paged expansion pool overflows */
+    if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize *
+        MM_PAGE_SIZE >= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart)
+    {
+        /* Check if non-paged expansion pool fits */
+        if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize *
+            MM_PAGE_SIZE <= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd)
+        {
+            /* Set new non-paged expansion pool end address */
+            MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart +
+                                                            MaximumNonPagedPoolSize * MM_PAGE_SIZE);
+        }
+    }
+
+    /* Compute non-paged expansion pool size */
+    MemoryLayout.NonPagedExpansionPoolSize = ((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd -
+                                                (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE;
+}

xtoskrnl/mm/i686/pfault.cc

@@ -0,0 +1,36 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/i686/pfault.cc
+ * DESCRIPTION:     Page fault support for i686 architecture
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Evaluates the PDE for for paged pool and per-session mappings.
+ *
+ * @param VirtualAddress
+ *        Specifies the virtual address to verify.
+ *
+ * @return This routine returns ACCESS_VIOLATION on PML3 or status code on PML2.
+ */
+XTFASTCALL
+XTSTATUS
+MM::PageFault::CheckPdeForPagedPool(IN PVOID VirtualAddress)
+{
+    /* Check if XPA is enabled */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Access violation for PML3 */
+        return STATUS_ACCESS_VIOLATION;
+    }
+
+    /* Unimplemented path for PML2 */
+    UNIMPLEMENTED;
+
+    /* Temporarily, just return access violation */
+    return STATUS_ACCESS_VIOLATION;
+}

xtoskrnl/mm/i686/pfn.cc

@@ -0,0 +1,351 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/i686/pfn.cc
+ * DESCRIPTION:     Physical Frame Number for i686 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ *                  Rafal Kupiec <belliash@codingworkshop.eu.org>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Initializes the PFN database by mapping virtual memory and populating entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::InitializePfnDatabase(VOID)
+{
+    PLOADER_MEMORY_DESCRIPTOR Descriptor;
+    PKERNEL_INITIALIZATION_BLOCK InitializationBlock;
+    PLIST_ENTRY ListEntry;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    PUCHAR PfnDatabaseEnd;
+    PMMPTE ValidPte;
+
+    /* Raise runlevel and acquire PFN lock */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL);
+    KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+    /* Get the kernel initialization block */
+    InitializationBlock = KE::BootInformation::GetInitializationBlock();
+
+    /* Get memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Get the PFN database size and calculate the end of the PFN database virtual address space */
+    PfnDatabaseEnd = (PUCHAR)MemoryLayout->PfnDatabase + (MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE) - 1;
+
+    /* Get a template PTE for mapping the PFN database pages */
+    ValidPte = MM::Pte::GetValidPte();
+
+    /* Map the Page Directory and Page Directory Pointer tables for the PFN database */
+    MM::Pte::MapPDE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte);
+    MM::Pte::MapPTE(MemoryLayout->PfnDatabase, PfnDatabaseEnd, ValidPte);
+
+    /* Zero PFN database virtual space */
+    RTL::Memory::ZeroMemory(MemoryLayout->PfnDatabase, MemoryLayout->PfnDatabaseSize * MM_PAGE_SIZE);
+
+    /* Initialize the color tables */
+    MM::Colors::InitializeColorTables();
+
+    /* Iterate over memory descriptors to map the PFN database and initialize entries */
+    ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink;
+    while(ListEntry != &InitializationBlock->MemoryDescriptorListHead)
+    {
+        /* Get the descriptor */
+        Descriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_DESCRIPTOR, ListEntry);
+
+        /* Skip invisible memory regions */
+        if(MM::Manager::VerifyMemoryTypeInvisible(Descriptor->MemoryType))
+        {
+            /* Move to the next descriptor and continue */
+            ListEntry = ListEntry->Flink;
+            continue;
+        }
+
+        /* Check if this is the modified free descriptor */
+        if(Descriptor == FreeDescriptor)
+        {
+            /* Switch to the original descriptor */
+            Descriptor = &OriginalFreeDescriptor;
+        }
+
+        /* Check if the free memory block that was split is being processed */
+        if(Descriptor == &OriginalFreeDescriptor)
+        {
+            /* Skip loop processing, free memory is initialized separately */
+            ListEntry = ListEntry->Flink;
+            continue;
+        }
+
+        /* Initialize PFNs for this memory range */
+        ProcessMemoryDescriptor(Descriptor->BasePage, Descriptor->PageCount, Descriptor->MemoryType);
+
+        /* Move to the next descriptor */
+        ListEntry = ListEntry->Flink;
+    }
+
+    /* Initialize PFNs for the free memory */
+    ProcessMemoryDescriptor(FreeDescriptor->BasePage, FreeDescriptor->PageCount, LoaderFree);
+
+    /* Initialize PFNs for the physical pages backing the PFN database */
+    ProcessMemoryDescriptor(OriginalFreeDescriptor.BasePage,
+                            FreeDescriptor->BasePage - OriginalFreeDescriptor.BasePage,
+                            LoaderMemoryData);
+
+    /* Restore original free descriptor */
+    *FreeDescriptor = OriginalFreeDescriptor;
+
+    /* Initialize PFNs backing page tables */
+    InitializePageTablePfns();
+}
+
+/**
+ * Initializes PFN database entries for the system page tables.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::InitializePageTablePfns(VOID)
+{
+    PFN_NUMBER PageFrameIndex;
+    PMMPFN Pfn;
+    ULONG RootLevel;
+    PMMPTE RootPte;
+
+    /* Determine root structure based on paging mode */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* XPA enabled, 3-level paging (PAE) */
+        RootLevel = 3;
+
+        /* Retrieve the PFN of the PML3 table and its virtual base address */
+        PageFrameIndex = AR::CpuFunc::ReadControlRegister(3) >> MM_PAGE_SHIFT;
+        RootPte = (PMMPTE)MM::Paging::GetPpeAddress(NULLPTR);
+    }
+    else
+    {
+        /* XPA disabled, 2-level paging */
+        RootLevel = 2;
+
+        /* Retrieve the PFN of the PML2 table and its virtual base address */
+        PageFrameIndex = AR::CpuFunc::ReadControlRegister(3) >> MM_PAGE_SHIFT;
+        RootPte = (PMMPTE)MM::Paging::GetPdeAddress(NULLPTR);
+    }
+
+    /* Initialize the PFN entry for the root page table itself */
+    Pfn = GetPfnEntry(PageFrameIndex);
+    if(Pfn)
+    {
+        /* Initialize the PFN entry */
+        Pfn->PteAddress = NULLPTR;
+        Pfn->u1.WsIndex = 0;
+        Pfn->u2.ShareCount = 1;
+        Pfn->u3.e1.CacheAttribute = PfnNonCached;
+        Pfn->u3.e1.PageLocation = ActiveAndValid;
+        Pfn->u3.e2.ReferenceCount = 1;
+        Pfn->u4.PteFrame = 0;
+    }
+
+    /* Start recursive scan from the top level */
+    if(RootPte)
+    {
+        /* Scan the root page table */
+        ScanPageTable(RootPte, RootLevel);
+    }
+}
+
+/**
+ * Processes a memory descriptor and initializes the corresponding PFN database entries
+ *
+ * @param BasePage
+ *        The starting physical page number of the memory run
+ *
+ * @param PageCount
+ *        The number of pages in the memory run
+ *
+ * @param MemoryType
+ *        The type of memory as reported by the bootloader (e.g., free, ROM, in-use)
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
+                                 IN PFN_NUMBER PageCount,
+                                 IN LOADER_MEMORY_TYPE MemoryType)
+{
+    PFN_NUMBER PageNumber;
+    PMMPDE PointerPde;
+    PMMPFN Pfn;
+
+    /* Check if the memory descriptor describes a free memory region */
+    if(MM::Manager::VerifyMemoryTypeFree(MemoryType))
+    {
+        /* Iterate over each page in this free memory run */
+        for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+        {
+            /* Get the PFN entry for the current page and ensure it is not referenced */
+            Pfn = GetPfnEntry(BasePage + PageNumber);
+            if(Pfn->u3.e2.ReferenceCount == 0)
+            {
+                /* Add the page to the free list to make it available for allocation */
+                LinkFreePage(BasePage + PageNumber);
+            }
+        }
+    }
+    else
+    {
+        /* Handle all other (non-free) memory types */
+        switch(MemoryType)
+        {
+            case LoaderBad:
+                /* This memory is marked as bad and should not be used */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Link the page to the bad pages list */
+                    LinkPage(&BadPagesList, BasePage + PageNumber);
+                }
+                break;
+            case LoaderXIPRom:
+                /* This memory range contains Read-Only Memory (ROM) */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Get the PFN entry for the current ROM page */
+                    Pfn = GetPfnEntry(BasePage + PageNumber);
+
+                    /* Ensure that the page is not already in-use */
+                    if(Pfn->u3.e2.ReferenceCount == 0)
+                    {
+                        /* Get the page directory entry for the current page */
+                        PointerPde = MM::Paging::GetPdeAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+
+                        /* Initialize the PFN entry to represent a ROM page */
+                        Pfn->PteAddress = MM::Paging::GetPteAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+                        Pfn->u1.Flink = 0;
+                        Pfn->u2.ShareCount = 0;
+                        Pfn->u3.e1.CacheAttribute = PfnNonCached;
+                        Pfn->u3.e1.PageLocation = 0;
+                        Pfn->u3.e1.PrototypePte = 1;
+                        Pfn->u3.e1.Rom = 1;
+                        Pfn->u3.e2.ReferenceCount = 0;
+                        Pfn->u4.InPageError = 0;
+                        Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde);
+                    }
+                }
+                break;
+            default:
+                /* All other types are considered in-use (ie, by the kernel, ACPI, etc) */
+                for(PageNumber = 0; PageNumber < PageCount; PageNumber++)
+                {
+                    /* Get the PFN entry for the current in-use page */
+                    Pfn = GetPfnEntry(BasePage + PageNumber);
+
+                    /* Ensure that the page is not already in-use */
+                    if(Pfn->u3.e2.ReferenceCount == 0)
+                    {
+                        /* Get the page directory entry for the current page */
+                        PointerPde = MM::Paging::GetPdeAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+
+                        /* Initialize the PFN entry to represent an in-use page and prevent it from being allocated */
+                        Pfn->PteAddress = MM::Paging::GetPteAddress((PVOID)(KSEG0_BASE + (BasePage << MM_PAGE_SHIFT)));
+                        Pfn->u2.ShareCount++;
+                        Pfn->u3.e1.CacheAttribute = PfnNonCached;
+                        Pfn->u3.e1.PageLocation = ActiveAndValid;
+                        Pfn->u3.e2.ReferenceCount = 1;
+                        Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(PointerPde);
+                    }
+                }
+                break;
+        }
+    }
+}
+
+/**
+ * Recursively scans a page table to initialize PFN database entries for active pages.
+ *
+ * @param PointerPte
+ *        Pointer to the base of the page table to scan.
+ *
+ * @param Level
+ *        The paging level of the table being scanned.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::ScanPageTable(IN PMMPTE PointerPte,
+                       IN ULONG Level)
+{
+    PVOID Address;
+    ULONG Index;
+    PMMPTE NextLevelPte;
+    ULONG PtesPerPage;
+
+    /* Get the number of PTEs per page */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+
+    /* Check if PML3 is enabled and current level is PDPT */
+    if(Level == 3)
+    {
+        /* PAE PDPT has only 4 entries */
+        PtesPerPage = 4;
+    }
+
+    /* Iterate through all entries in the current page table */
+    for(Index = 0; Index < PtesPerPage; Index++)
+    {
+        /* Check if the page table entry is present */
+        if(MM::Paging::PteValid(PointerPte))
+        {
+            /* Mark the PFN pointed to by this entry as active */
+            LinkPfnForPageTable(MM::Paging::GetPageFrameNumber(PointerPte), PointerPte);
+
+            /* Recurse to the next level, if this is not a leaf node (PTE) */
+            if(Level > 1)
+            {
+                /* Calculate the virtual address mapped by this entry to find the next table */
+                switch(Level)
+                {
+                    case 3:
+                        /* Calculate PDE */
+                        Address = MM::Paging::GetPpeVirtualAddress(PointerPte);
+                        NextLevelPte = (PMMPTE)MM::Paging::GetPdeAddress(Address);
+                        break;
+                    case 2:
+                        /* Calculate PTE */
+                        Address = MM::Paging::GetPdeVirtualAddress(PointerPte);
+                        NextLevelPte = (PMMPTE)MM::Paging::GetPteAddress(Address);
+                        break;
+                    default:
+                        /* Nothing to calculate, return NULLPTR */
+                        NextLevelPte = NULLPTR;
+                        break;
+                }
+
+                /* Recurse deeper if not at the bottom level (PTE) already */
+                if(NextLevelPte)
+                {
+                    /* Recursively scan the next level page table */
+                    ScanPageTable(NextLevelPte, Level - 1);
+                }
+            }
+        }
+
+        /* Move to the next entry in the current table */
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+}

xtoskrnl/mm/i686/pte.cc

@@ -0,0 +1,128 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/i686/pte.cc
+ * DESCRIPTION:     Page Table Entry (PTE) for i686 support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Checks if the virtual address is valid and mapped in the page tables.
+ *
+ * @param VirtualAddress
+ *        The virtual address to check.
+ *
+ * @return This routine returns TRUE if the address is valid, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Pte::AddressValid(IN PVOID VirtualAddress)
+{
+    /* Check if PDE and PTE are valid */
+    if(!MM::Paging::PteValid(MM::Paging::GetPdeAddress(VirtualAddress)) ||
+       !MM::Paging::PteValid(MM::Paging::GetPteAddress(VirtualAddress)))
+    {
+        /* Invalid PDE or PTE, return FALSE */
+        return FALSE;
+    }
+
+    /* Address is valid, return TRUE */
+    return TRUE;
+}
+
+/**
+ * Retrieves the base virtual address of the system PTEs.
+ *
+ * @return This routine returns a pointer to the first PTE in the system PTE space.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Pte::GetSystemPteBaseAddress(VOID)
+{
+    return MM::Paging::GetPteAddress(NULLPTR);
+}
+
+/**
+ * Performs the initial setup of the system's page table hierarchy.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::InitializePageTable(VOID)
+{
+    PMMPTE EndSpacePte, PointerPte;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    CPUID_REGISTERS CpuRegisters;
+    MMPTE TemplatePte;
+
+    /* Retrieve current paging mode and memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Get CPU features */
+    CpuRegisters.Leaf = CPUID_GET_STANDARD1_FEATURES;
+    AR::CpuFunc::CpuId(&CpuRegisters);
+
+    /* Check if Paging Global Extensions (PGE) is supported */
+    if(CpuRegisters.Edx & CPUID_FEATURES_EDX_PGE)
+    {
+        /* Enable the Global Paging (PGE) feature */
+        AR::CpuFunc::WriteControlRegister(4, AR::CpuFunc::ReadControlRegister(4) | CR4_PGE);
+    }
+
+    /* Get the PD user-space range for both legacy and PAE paging */
+    PointerPte = (PMMPTE)MM::Paging::GetPdeAddress(0);
+    EndSpacePte = (PMMPTE)MM::Paging::GetPdeAddress(MemoryLayout->UserSpaceEnd);
+
+    /* Clear all top-level entries mapping the user address space */
+    while(PointerPte <= EndSpacePte)
+    {
+        MM::Paging::ClearPte(PointerPte);
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+
+    /* Flush the TLB to invalidate all non-global entries */
+    AR::CpuFunc::FlushTlb();
+
+    /* Create a template PTE for mapping kernel pages */
+    MM::Paging::ClearPte(&TemplatePte);
+    MM::Paging::SetPte(&TemplatePte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE);
+
+    /* Map the kernel's PD entries */
+    MM::Pte::MapPDE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)MM_HIGHEST_SYSTEM_ADDRESS, &TemplatePte);
+}
+
+/**
+ * Maps a range of virtual addresses at the PPE (Page Directory Pointer Entry) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplatePpe
+ *        A template PPE to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapPPE(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMPPE TemplatePpe)
+{
+    /* Just a stub on i686 platform */
+    return;
+}

xtoskrnl/mm/init.cc

@@ -1,150 +0,0 @@
-/**
- * PROJECT:         ExectOS
- * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/mm/init.cc
- * DESCRIPTION:     Memory Manager initialization routines
- * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
- */
-
-#include <xtos.hh>
-
-
-/**
- * Performs an early initialization of the XTOS Memory Manager.
- *
- * @return This routine does not return any value.
- *
- * @since XT 1.0
- */
-XTAPI
-VOID
-MM::Init::InitializeMemoryManager(VOID)
-{
-    /* Scan memory descriptors provided by the boot loader */
-    ScanMemoryDescriptors();
-
-    /* Check if there are enough physical pages */
-    if(NumberOfPhysicalPages < MM_MINIMUM_PHYSICAL_PAGES)
-    {
-        /* Insufficient physical pages, kernel panic */
-        DebugPrint(L"Insufficient physical pages! Install additional memory\n");
-        KE::Crash::Panic(0);
-    }
-
-    /* Proceed with architecture specific initialization */
-    InitializeArchitecture();
-}
-
-/**
- * Scans memory descriptors provided by the boot loader.
- *
- * @return This routine does not return any value.
- *
- * @since XT 1.0
- */
-XTAPI
-VOID
-MM::Init::ScanMemoryDescriptors(VOID)
-{
-    PLIST_ENTRY LoaderMemoryDescriptors, MemoryMappings;
-    PLOADER_MEMORY_DESCRIPTOR MemoryDescriptor;
-    PFN_NUMBER FreePages;
-
-    /* Initially, set number of free pages to 0 */
-    FreePages = 0;
-
-    /* Get a list of memory descriptors provided by the boot loader */
-    LoaderMemoryDescriptors = KE::BootInformation::GetMemoryDescriptors();
-
-    /* Iterate through memory mappings provided by the boot loader */
-    MemoryMappings = LoaderMemoryDescriptors->Flink;
-    while(MemoryMappings != LoaderMemoryDescriptors)
-    {
-        /* Get memory descriptor */
-        MemoryDescriptor = CONTAIN_RECORD(MemoryMappings, LOADER_MEMORY_DESCRIPTOR, ListEntry);
-
-        /* Check if memory type is invisible or cached */
-        if(VerifyMemoryTypeInvisible(MemoryDescriptor->MemoryType) ||
-           (MemoryDescriptor->MemoryType == LoaderHardwareCachedMemory))
-        {
-            /* Skip this mapping */
-            MemoryMappings = MemoryMappings->Flink;
-            continue;
-        }
-
-        /* Make sure that memory type is not bad */
-        if(MemoryDescriptor->MemoryType != LoaderBad)
-        {
-            /* Increment number of physical pages */
-            NumberOfPhysicalPages += MemoryDescriptor->PageCount;
-        }
-
-        /* Find lowest physical page */
-        if(MemoryDescriptor->BasePage < LowestPhysicalPage)
-        {
-            /* Update lowest physical page */
-            LowestPhysicalPage = MemoryDescriptor->BasePage;
-        }
-
-        /* Find highest physical page */
-        if(MemoryDescriptor->BasePage + MemoryDescriptor->PageCount > HighestPhysicalPage)
-        {
-            /* Update highest physical page */
-            HighestPhysicalPage = (MemoryDescriptor->BasePage + MemoryDescriptor->PageCount) - 1;
-        }
-
-        /* Check if memory type should be considered as free */
-        if(VerifyMemoryTypeFree(MemoryDescriptor->MemoryType))
-        {
-            /* Check if this descriptor contains more free pages */
-            if(MemoryDescriptor->PageCount >= FreePages)
-            {
-                /* Update free descriptor */
-                FreePages = MemoryDescriptor->PageCount;
-                FreeDescriptor = MemoryDescriptor;
-            }
-        }
-
-        /* Get next memory descriptor */
-        MemoryMappings = MemoryMappings->Flink;
-    }
-
-    /* Store original free descriptor */
-    RTL::Memory::CopyMemory(&OldFreeDescriptor, FreeDescriptor, sizeof(LOADER_MEMORY_DESCRIPTOR));
-}
-
-/** Checks whether the specified memory type should be considered as free.
- *
- * @param MemoryType
- *        Specifies the memory type to verify.
- *
- * @return This routine returns TRUE if the specified memory type should be considered as free, or FALSE otherwise.
- *
- * @since XT 1.0
- */
-XTAPI
-BOOLEAN
-MM::Init::VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType)
-{
-    return ((MemoryType == LoaderFree) || (MemoryType == LoaderFirmwareTemporary) ||
-            (MemoryType == LoaderLoadedProgram) || (MemoryType == LoaderOsloaderStack));
-}
-
-/**
- * Checks whether the specified memory type should be considered as invisible for the memory manager.
- *
- * @param MemoryType
- *        Specifies the memory type to verify.
- *
- * @return This routine returns TRUE if the specified memory type should be considered as invisible, or FALSE otherwise.
- *
- * @since XT 1.0
- */
-XTAPI
-BOOLEAN
-MM::Init::VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType)
-{
-    return ((MemoryType == LoaderFirmwarePermanent) ||
-            (MemoryType == LoaderSpecialMemory) ||
-            (MemoryType == LoaderBBTMemory));
-}

xtoskrnl/mm/kpool.cc

@@ -4,6 +4,7 @@
  * FILE:            xtoskrnl/mm/kpool.cc
  * DESCRIPTION:     Kernel pool memory management
  * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
+ *                  Aiken Harris <harraiken91@gmail.com>
  */
 
 #include <xtos.hh>
@@ -27,12 +28,68 @@
  */
 XTAPI
 XTSTATUS
-MM::KernelPool::AllocateKernelStack(IN PVOID *Stack,
+MM::KernelPool::AllocateKernelStack(OUT PVOID *Stack,
-                                    IN BOOLEAN LargeStack,
+                                    IN ULONG StackSize)
-                                    IN UCHAR SystemNode)
 {
-    UNIMPLEMENTED;
+    PFN_COUNT StackPages;
-    return STATUS_NOT_IMPLEMENTED;
+    PMMPTE PointerPte, StackPte;
+    MMPTE TempPte, InvalidPte;
+    PFN_NUMBER PageFrameIndex;
+    ULONG Index;
+
+    /* Initialize the output stack pointer to NULLPTR */
+    *Stack = NULLPTR;
+
+    /* Convert the requested stack size into a page count */
+    StackPages = SIZE_TO_PAGES(StackSize);
+
+    /* Reserve PTEs for the stack pages, plus a guard page */
+    StackPte = MM::Pte::ReserveSystemPtes(StackPages + 1, SystemPteSpace);
+    if(!StackPte)
+    {
+        /* Failed to reserve PTEs for the new kernel stack */
+        return STATUS_INSUFFICIENT_RESOURCES;
+    }
+
+    /* Set up a template for an invalid PTE */
+    MM::Paging::SetPte(&InvalidPte, 0, 0x18);
+
+    /* Set up a template for a valid, writable stack PTE */
+    MM::Paging::ClearPte(&TempPte);
+    MM::Paging::SetPte(&TempPte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE);
+
+    /* Acquire the PFN database lock and raise runlevel to DISPATCH_LEVEL */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL);
+    KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+    /* Start iterating from the base of the reserved PTE block */
+    PointerPte = StackPte;
+
+    /* Loop through each page of the stack that needs to be allocated */
+    for(Index = 0; Index < StackPages; Index++)
+    {
+        /* Advance to the next PTE */
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+
+        /* Allocate a physical page and temporarily mark the PTE as invalid */
+        PageFrameIndex = MM::Pfn::AllocatePhysicalPage(MM::Colors::GetNextColor());
+        *PointerPte = InvalidPte;
+
+        /* Associate the physical page with its corresponding PTE in the PFN database */
+        MM::Pfn::LinkPfnToPte(PageFrameIndex, PointerPte, 1);
+
+        /* Make the PTE valid, mapping the virtual address to the physical page */
+        MM::Paging::SetPte(&TempPte, PageFrameIndex, 0);
+        *PointerPte = TempPte;
+    }
+
+    /* Zero the newly allocated stack memory, skipping the guard page */
+    RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(MM::Paging::GetNextPte(StackPte)),
+                            MM_PAGE_SIZE * StackPages);
+
+    /* Return a pointer to the top of the new stack */
+    *Stack = MM::Paging::GetPteVirtualAddress(MM::Paging::AdvancePte(StackPte, StackPages + 1));
+    return STATUS_SUCCESS;
 }
 
 /**
@@ -96,9 +153,40 @@ MM::KernelPool::AllocateProcessorStructures(IN ULONG CpuNumber,
 XTAPI
 VOID
 MM::KernelPool::FreeKernelStack(IN PVOID Stack,
-                                IN BOOLEAN LargeStack)
+                                IN ULONG StackSize)
 {
-    UNIMPLEMENTED;
+    PFN_COUNT StackPages;
+    PMMPTE PointerPte;
+    ULONG Index;
+
+    /* Get the PTE for the top of the stack, including the guard page */
+    PointerPte = MM::Paging::AdvancePte(MM::Paging::GetPteAddress(Stack), -1);
+
+    /* Convert the stack size into a page count */
+    StackPages = SIZE_TO_PAGES(StackSize);
+
+    /* Start guarded code block */
+    {
+        /* Acquire the PFN database lock */
+        KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+        /* Loop through each page of the stack that needs to be freed */
+        for(Index = 0; Index < StackPages; Index++)
+        {
+            /* Ensure the PTE is valid */
+            if(MM::Paging::PteValid(PointerPte))
+            {
+                /* Free the physical page */
+                MM::Pfn::FreePhysicalPage(PointerPte);
+            }
+
+            /* Advance to the next PTE */
+            PointerPte = MM::Paging::AdvancePte(PointerPte, -1);
+        }
+    }
+
+    /* Release all system PTEs used by the stack, including the guard page */
+    MM::Pte::ReleaseSystemPtes(PointerPte, StackPages + 1, SystemPteSpace);
 }
 
 /**

xtoskrnl/mm/mmgr.cc

@@ -0,0 +1,172 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/mmgr.cc
+ * DESCRIPTION:     Memory Manager
+ * DEVELOPERS:      Rafal Kupiec <belliash@codingworkshop.eu.org>
+ *                  Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Computes the size of the boot image.
+ *
+ * @param BootImageSize
+ *        Supplies a pointer to a variable that will receive the size of the boot image in pages.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeBootImageSize(OUT PPFN_NUMBER BootImageSize)
+{
+    PKERNEL_INITIALIZATION_BLOCK InitializationBlock;
+    PFN_NUMBER ImageSize;
+    ULONG PteSize;
+
+    /* Get the kernel initialization block */
+    InitializationBlock = KE::BootInformation::GetInitializationBlock();
+
+    /* Get the size of a PTE */
+    PteSize = MM::Paging::GetPteSize();
+
+    /* Calculate the size of the boot image */
+    ImageSize = InitializationBlock->BootImageSize * MM_PAGE_SIZE;
+    ImageSize = (ImageSize + ((MM_PAGE_SIZE / PteSize) * MM_PAGE_SIZE) - 1) &
+                ~(((MM_PAGE_SIZE / PteSize) * MM_PAGE_SIZE) - 1);
+
+    /* Return number of pages used by the boot image */
+    *BootImageSize = ImageSize / MM_PAGE_SIZE;
+}
+
+/**
+ * Retrieves the amount of total available memory in the system.
+ *
+ * @return This routine returns the amount of available memory in the system in megabytes.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG_PTR
+MM::Manager::GetInstalledMemorySize(VOID)
+{
+    /* Return the amount of installed memory */
+    return (MM::Pfn::GetNumberOfPhysicalPages() * MM_PAGE_SIZE) / 1048576;
+}
+
+/**
+ * Retrieves a pointer to the system's virtual memory layout structure.
+ *
+ * @return This routine returns a pointer to the memory layout structure.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMMEMORY_LAYOUT
+MM::Manager::GetMemoryLayout(VOID)
+{
+    /* Return a pointer to the global memory layout structure */
+    return &MemoryLayout;
+}
+
+/**
+ * Retrieves the number of system PTEs.
+ *
+ * @return This routine returns the number of system PTEs.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PFN_NUMBER
+MM::Manager::GetNumberOfSystemPtes()
+{
+    return NumberOfSystemPtes;
+}
+
+/**
+ * Performs an early initialization of the XTOS Memory Manager.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::InitializeMemoryManager(VOID)
+{
+    /* Scan memory descriptors provided by the boot loader */
+    MM::Pfn::ScanMemoryDescriptors();
+
+    /* Check if there are enough physical pages */
+    if(MM::Pfn::GetNumberOfPhysicalPages() < MM_MINIMUM_PHYSICAL_PAGES)
+    {
+        /* Insufficient physical pages, kernel panic */
+        DebugPrint(L"Insufficient physical pages! Install additional memory\n");
+        KE::Crash::Panic(0);
+    }
+
+    /* Compute page colors to reduce CPU cache conflicts */
+    MM::Colors::ComputePageColoring();
+
+    /* Initialize and dump memory layout */
+    InitializeMemoryLayout();
+    DumpMemoryLayout();
+
+    /* Initialize PTE template */
+    MM::Pte::InitializeSystemPte();
+
+    /* Initialize page table */
+    MM::Pte::InitializePageTable();
+
+    /* Initialize non-paged pool */
+    MM::Allocator::InitializeNonPagedPool();
+
+    /* Initialize system PTE space */
+    MM::Pte::InitializeSystemPteSpace();
+
+    /* Initialize PFN database */
+    MM::Pfn::InitializePfnDatabase();
+}
+
+/**
+ * Checks whether the specified memory type should be considered as free.
+ *
+ * @param MemoryType
+ *        Specifies the memory type to verify.
+ *
+ * @return This routine returns TRUE if the specified memory type should be considered as free, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Manager::VerifyMemoryTypeFree(LOADER_MEMORY_TYPE MemoryType)
+{
+    return ((MemoryType == LoaderFirmwareTemporary) ||
+            (MemoryType == LoaderFree) ||
+            (MemoryType == LoaderLoadedProgram) ||
+            (MemoryType == LoaderOsloaderStack));
+}
+
+/**
+ * Checks whether the specified memory type should be considered as invisible for the memory manager.
+ *
+ * @param MemoryType
+ *        Specifies the memory type to verify.
+ *
+ * @return This routine returns TRUE if the specified memory type should be considered as invisible, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Manager::VerifyMemoryTypeInvisible(LOADER_MEMORY_TYPE MemoryType)
+{
+    return ((MemoryType == LoaderBBTMemory) ||
+            (MemoryType == LoaderFirmwarePermanent) ||
+            (MemoryType == LoaderSpecialMemory));
+}

xtoskrnl/mm/paging.cc

@@ -9,6 +9,28 @@
 #include <xtos.hh>
 
 
+/**
+ * Advances a PTE pointer by a given number of entries, considering the actual PTE size.
+ *
+ * @param Pte
+ *        The PTE pointer to advance.
+ *
+ * @param Count
+ *        The number of PTE entries to advance by.
+ *
+ * @return The advanced PTE pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Paging::AdvancePte(IN PMMPTE Pte,
+                       IN LONG Count)
+{
+    /* Return advanced PTE pointer */
+    return PmlRoutines->AdvancePte(Pte, Count);
+}
+
 /**
  * Clears the contents of a page table entry (PTE).
  *
@@ -21,8 +43,9 @@
  */
 XTAPI
 VOID
-MM::Paging::ClearPte(PHARDWARE_PTE PtePointer)
+MM::Paging::ClearPte(IN PMMPTE PtePointer)
 {
+    /* Clear PTE */
     PmlRoutines->ClearPte(PtePointer);
 }
 
@@ -78,6 +101,77 @@ MM::Paging::FlushTlb(VOID)
     }
 }
 
+/**
+ * Gets the next entry in a PTE list.
+ *
+ * @param Pte
+ *        The PTE pointer to get the next entry from.
+ *
+ * @return This routine returns the next entry in the PTE list.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG_PTR
+MM::Paging::GetNextEntry(IN PMMPTE Pte)
+{
+    /* Return next entry in PTE list */
+    return PmlRoutines->GetNextEntry(Pte);
+}
+
+/**
+ * Advances a PTE pointer, considering the actual PTE size.
+ *
+ * @param Pte
+ *        The PTE pointer to advance.
+ *
+ * @return The advanced PTE pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Paging::GetNextPte(IN PMMPTE Pte)
+{
+    /* Return advanced PTE pointer */
+    return PmlRoutines->GetNextPte(Pte);
+}
+
+/**
+ * Checks if a PTE list contains only one entry.
+ *
+ * @param Pte
+ *        The PTE pointer to check.
+ *
+ * @return This routine returns TRUE if the PTE list has only one entry, FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Paging::GetOneEntry(IN PMMPTE Pte)
+{
+    /* Return one entry status */
+    return PmlRoutines->GetOneEntry(Pte);
+}
+
+/**
+ * Gets the page frame number from a corresponding PTE.
+ *
+ * @param Pte
+ *        The PTE pointer to get the page frame number from.
+ *
+ * @return This routine returns the page frame number.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PFN_NUMBER
+MM::Paging::GetPageFrameNumber(IN PMMPTE Pte)
+{
+    return PmlRoutines->GetPageFrameNumber(Pte);
+}
+
 /**
  * Gets the page map routines for basic paging mode (non-XPA).
  *
@@ -90,6 +184,8 @@ MM::PPAGEMAP
 MM::Paging::GetPageMapBasicRoutines(VOID)
 {
     static MM::PageMapBasic PageMapBasicRoutines;
+
+    /* Return non-XPA page map routines */
     return &PageMapBasicRoutines;
 }
 
@@ -105,6 +201,8 @@ MM::PPAGEMAP
 MM::Paging::GetPageMapXpaRoutines(VOID)
 {
     static MM::PageMapXpa PageMapXpaRoutines;
+
+    /* Return XPA page map routines */
     return &PageMapXpaRoutines;
 }
 
@@ -120,11 +218,30 @@ MM::Paging::GetPageMapXpaRoutines(VOID)
  */
 XTAPI
 PMMPDE
-MM::Paging::GetPdeAddress(PVOID Address)
+MM::Paging::GetPdeAddress(IN PVOID Address)
 {
+    /* Return PDE address */
     return PmlRoutines->GetPdeAddress(Address);
 }
 
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Entry.
+ *
+ * @param PdePointer
+ *        Specifies the address of the PDE.
+ *
+ * @return This routine returns the virtual address mapped by the PDE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::Paging::GetPdeVirtualAddress(IN PMMPDE PdePointer)
+{
+    /* Return PTE virtual address */
+    return PmlRoutines->GetPdeVirtualAddress(PdePointer);
+}
+
 /**
  * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address.
  *
@@ -137,11 +254,48 @@ MM::Paging::GetPdeAddress(PVOID Address)
  */
 XTAPI
 PMMPPE
-MM::Paging::GetPpeAddress(PVOID Address)
+MM::Paging::GetPpeAddress(IN PVOID Address)
 {
+    /* Return PPE address */
     return PmlRoutines->GetPpeAddress(Address);
 }
 
+/**
+ * Gets the virtual address that is mapped by a given Page Directory Pointer Table Entry.
+ *
+ * @param PpePointer
+ *        Specifies the address of the PPE.
+ *
+ * @return This routine returns the virtual address mapped by the PPE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::Paging::GetPpeVirtualAddress(IN PMMPPE PpePointer)
+{
+    /* Return PTE virtual address */
+    return PmlRoutines->GetPpeVirtualAddress(PpePointer);
+}
+
+ /**
+ * Gets the entire contents of a Page Table Entry (PTE) as a single value.
+ *
+ * @param PtePointer
+ *        Pointer to the Page Table Entry (PTE) to read.
+ *
+ * @return This routine returns the contents of the PTE as a single value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG_PTR
+MM::Paging::GetPte(IN PMMPTE PtePointer)
+{
+    /* Return PTE value */
+    return PmlRoutines->GetPte(PtePointer);
+}
+
 /**
  * Gets the address of the PTE (Page Table Entry), that maps given address.
  *
@@ -154,11 +308,135 @@ MM::Paging::GetPpeAddress(PVOID Address)
  */
 XTAPI
 PMMPTE
-MM::Paging::GetPteAddress(PVOID Address)
+MM::Paging::GetPteAddress(IN PVOID Address)
 {
+    /* Return PTE address */
     return PmlRoutines->GetPteAddress(Address);
 }
 
+/**
+ * Calculates the distance between two PTE pointers.
+ *
+ * @param EndPte
+ *        Pointer to the ending Page Table Entry.
+ *
+ * @param StartPte
+ *        Pointer to the starting Page Table Entry.
+ *
+ * @return This routine returns a signed value representing the number of PTEs between EndPte and StartPte.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+LONG
+MM::Paging::GetPteDistance(PMMPTE EndPte,
+                           PMMPTE StartPte)
+{
+    /* Return distance between PTE pointers */
+    return PmlRoutines->GetPteDistance(EndPte, StartPte);
+}
+
+/**
+ * Gets the size of a PTE.
+ *
+ * @return This routine returns the size of a PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Paging::GetPteSize(VOID)
+{
+    /* Return the size of MMPTE */
+    return PmlRoutines->GetPteSize();
+}
+
+/**
+ * Gets the software protection value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software protection value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Paging::GetPteSoftwareProtection(IN PMMPTE PtePointer)
+{
+    /* Return PTE software protection value */
+    return PmlRoutines->GetPteSoftwareProtection(PtePointer);
+}
+
+/**
+ * Gets the software prototype value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software prototype value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Paging::GetPteSoftwarePrototype(IN PMMPTE PtePointer)
+{
+    /* Return PTE software prototype value */
+    return PmlRoutines->GetPteSoftwarePrototype(PtePointer);
+}
+
+/**
+ * Gets the software transition value of the corresponding Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the PTE software transition value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Paging::GetPteSoftwareTransition(IN PMMPTE PtePointer)
+{
+    /* Return PTE software transition value */
+    return PmlRoutines->GetPteSoftwareTransition(PtePointer);
+}
+
+/**
+ * Gets the virtual address that is mapped by a given Page Table Entry.
+ *
+ * @param PtePointer
+ *        Specifies the address of the PTE.
+ *
+ * @return This routine returns the virtual address mapped by the PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PVOID
+MM::Paging::GetPteVirtualAddress(IN PMMPTE PtePointer)
+{
+    /* Return PTE virtual address */
+    return PmlRoutines->GetPteVirtualAddress(PtePointer);
+}
+
+/**
+ * Gets current status of eXtended Physical Addressing (XPA).
+ *
+ * @return This routine returns TRUE if PAE or LA57 (XPA) is enabled, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Paging::GetXpaStatus()
+{
+    return PmlRoutines->GetXpaStatus();
+}
+
 /**
  * Detects if eXtended Physical Addressing (XPA) is enabled and initializes page map support.
  *
@@ -198,22 +476,20 @@ MM::Paging::InitializePageMapSupport(VOID)
  */
 XTAPI
 BOOLEAN
-MM::Paging::PteValid(PHARDWARE_PTE PtePointer)
+MM::Paging::PteValid(IN PMMPTE PtePointer)
 {
+    /* Check if PTE is valid */
     return PmlRoutines->PteValid(PtePointer);
 }
 
 /**
- * Sets a PML2 page table entry (PTE) with the specified physical page and access flags.
+ * Sets the next entry in a PTE list.
  *
- * @param PtePointer
+ * @param Pte
- *        Pointer to the page table entry (PTE) to set.
+ *        The PTE pointer to modify.
  *
- * @param PageFrameNumber
+ * @param Value
- *        Physical frame number to map.
+ *        The value to set as the next entry.
- *
- * @param Writable
- *        Indicates whether the page should be writable.
  *
  * @return This routine does not return any value.
  *
@@ -221,11 +497,80 @@ MM::Paging::PteValid(PHARDWARE_PTE PtePointer)
  */
 XTAPI
 VOID
-MM::Paging::SetPte(PHARDWARE_PTE PtePointer,
+MM::Paging::SetNextEntry(IN PMMPTE Pte,
-                   PFN_NUMBER PageFrameNumber,
+                         IN ULONG_PTR Value)
-                   BOOLEAN Writable)
 {
-    PmlRoutines->SetPte(PtePointer, PageFrameNumber, Writable);
+    /* Set next entry in PTE list */
+    PmlRoutines->SetNextEntry(Pte, Value);
+}
+
+/**
+ * Sets the flag indicating whether a PTE list contains only one entry.
+ *
+ * @param Pte
+ *        The PTE pointer to modify.
+ *
+ * @param Value
+ *        The value to set. TRUE if the list has only one entry, FALSE otherwise.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::SetOneEntry(IN PMMPTE Pte,
+                        IN BOOLEAN Value)
+{
+    /* Set one entry status */
+    PmlRoutines->SetOneEntry(Pte, Value);
+}
+
+/**
+ * Sets a Page Table Entry (PTE) with the specified physical page and access flags.
+ *
+ * @param PtePointer
+ *        Pointer to the page table entry (PTE) to set.
+ *
+ * @param PageFrameNumber
+ *        Physical frame number to map.
+ *
+ * @param AttributesMask
+ *        Specifies the attributes mask to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::SetPte(IN PMMPTE PtePointer,
+                   IN PFN_NUMBER PageFrameNumber,
+                   IN ULONG_PTR AttributesMask)
+{
+    /* Set PTE */
+    PmlRoutines->SetPte(PtePointer, PageFrameNumber, AttributesMask);
+}
+
+/**
+ * Sets a Page Table Entry (PTE) with the specified attributes.
+ *
+ * @param PtePointer
+ *        Pointer to the page table entry (PTE) to set.
+ *
+ * @param Attributes
+ *        Specifies the attributes to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::SetPte(IN PMMPTE PtePointer,
+                   IN ULONG_PTR Attributes)
+{
+    PmlRoutines->SetPte(PtePointer, Attributes);
 }
 
 /**
@@ -246,9 +591,54 @@ MM::Paging::SetPte(PHARDWARE_PTE PtePointer,
  */
 XTAPI
 VOID
-MM::Paging::SetPteCaching(PHARDWARE_PTE PtePointer,
+MM::Paging::SetPteCaching(IN PMMPTE PtePointer,
-                          BOOLEAN CacheDisable,
+                          IN BOOLEAN CacheDisable,
-                          BOOLEAN WriteThrough)
+                          IN BOOLEAN WriteThrough)
 {
+    /* Set caching attributes */
     PmlRoutines->SetPteCaching(PtePointer, CacheDisable, WriteThrough);
 }
+
+/**
+ * Transitions a Page Table Entry (PTE) to invalid state
+ *
+ * @param PointerPte
+ *        Pointer to the page table entry (PTE) to transition.
+ *
+ * @param Protection
+ *        Specifies the protection attribute to apply to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::TransitionPte(IN PMMPTE PointerPte,
+                          IN ULONG_PTR Protection)
+{
+    /* Transition PTE */
+    PmlRoutines->TransitionPte(PointerPte, Protection);
+}
+
+/**
+ * Writes a Page Table Entry (PTE) with the specified value.
+ *
+ * @param Pte
+ *        Pointer to the page table entry (PTE) to write.
+ *
+ * @param Value
+ *        The value to write to the PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::WritePte(IN PMMPTE Pte,
+                     IN MMPTE Value)
+{
+    /* Assign PTE value */
+    PmlRoutines->WritePte(Pte, Value);
+}

xtoskrnl/mm/pte.cc

@@ -0,0 +1,546 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/pte.cc
+ * DESCRIPTION:     Page Table Entry (PTE) support
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Finds a free cluster of system PTEs that can satisfy a given size.
+ *
+ * @param NumberOfPtes
+ *        The number of contiguous PTEs required.
+ *
+ * @param SystemPtePoolType
+ *        Specifies the system PTE pool to search within.
+ *
+ * @param FoundCluster
+ *        On success, receives a pointer to the first PTE of the found cluster.
+ *
+ * @param PreviousClusterNode
+ *        On success, receives a pointer to the list node that precedes the found cluster.
+ *
+ * @return This routine returns TRUE if a suitable cluster was found, FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Pte::FindFreeCluster(IN PFN_COUNT NumberOfPtes,
+                         IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+                         OUT PMMPTE *FoundCluster,
+                         OUT PMMPTE *PreviousClusterNode)
+{
+    PMMPTE CurrentCluster;
+    PMMPTE PreviousNode = &FirstSystemFreePte[SystemPtePoolType];
+    ULONG ClusterSize;
+
+    /* Find a free PTE cluster large enough for the request */
+    while(MM::Paging::GetNextEntry(PreviousNode) != MAXULONG)
+    {
+        /* Retrieve the cluster and its size */
+        CurrentCluster = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousNode));
+        ClusterSize = GetClusterSize(CurrentCluster);
+
+        /* Check if this cluster is large enough */
+        if(NumberOfPtes <= ClusterSize)
+        {
+            /* Found a suitable cluster */
+            *FoundCluster = CurrentCluster;
+            *PreviousClusterNode = PreviousNode;
+            return TRUE;
+        }
+
+        /* This cluster is too small, check the next one */
+        PreviousNode = CurrentCluster;
+    }
+
+    /* No suitable cluster was found */
+    return FALSE;
+}
+
+/**
+ * Decodes and returns the size of a free PTE cluster.
+ *
+ * @param Pte
+ *        Supplies a pointer to the first PTE of the free cluster to inspect.
+ *
+ * @return This routine returns the total number of contiguous PTEs in the free cluster.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+ULONG
+MM::Pte::GetClusterSize(IN PMMPTE Pte)
+{
+    /* A special flag in the first PTE indicates a free cluster of size one */
+    if(MM::Paging::GetOneEntry(Pte))
+    {
+        /* Flag is set, so the cluster size is 1 by definition */
+        return 1;
+    }
+
+    /* For larger clusters, the size is encoded in the second PTE of the block */
+    Pte = MM::Paging::GetNextPte(Pte);
+    return MM::Paging::GetNextEntry(Pte);
+}
+
+/**
+ * Calculates the number of Page Table Entries (PTEs) that fit within a single page.
+ *
+ * @return This routine returns the number of PTEs per page.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PFN_COUNT
+MM::Pte::GetPtesPerPage(VOID)
+{
+    /* Calculate and return the number of PTEs per page */
+    return MM_PAGE_SIZE / MM::Paging::GetPteSize();
+}
+
+/**
+ * Returns a pointer to the valid PTE.
+ *
+ * @return This routine returns a pointer to the valid PTE.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Pte::GetValidPte()
+{
+    /* Return a pointer to the valid PTE */
+    return &ValidPte;
+}
+
+/**
+ * Initializes the system's PTE.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::InitializeSystemPte(VOID)
+{
+    /* Initialize the PTE template */
+    MM::Paging::SetPte(&ValidPte, MM_PTE_VALID | MM_PTE_EXECUTE_READWRITE | MM_PTE_DIRTY | MM_PTE_ACCESSED);
+}
+
+/**
+ * Formats a range of PTEs into a freelist-based pool for system allocations.
+ *
+ * @param StartingPte
+ *        Supplies a pointer to the start of the PTE range to be formatted.
+ *
+ * @param NumberOfPtes
+ *        Supplies the total number of PTEs in the contiguous range.
+ *
+ * @param PoolType
+ *        The system PTE pool type that this range will be used for.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::InitializeSystemPtePool(IN PMMPTE StartingPte,
+                                 IN PFN_COUNT NumberOfPtes,
+                                 IN MMSYSTEM_PTE_POOL_TYPE PoolType)
+{
+    /* Set the system PTE base address */
+    SystemPteBase = GetSystemPteBaseAddress();
+
+    /* Record the boundaries of this new PTE pool */
+    SystemPtesStart[PoolType] = StartingPte;
+    SystemPtesEnd[PoolType] = MM::Paging::AdvancePte(StartingPte, NumberOfPtes - 1);
+
+    /* Zero the memory for the new PTE pool before use */
+    RTL::Memory::ZeroMemory(StartingPte, NumberOfPtes * MM::Paging::GetPteSize());
+
+    /* Build the free list head to point to the start of the pool */
+    MM::Paging::SetNextEntry(StartingPte, MAXULONG);
+    MM::Paging::ClearPte(&FirstSystemFreePte[PoolType]);
+    MM::Paging::SetNextEntry(&FirstSystemFreePte[PoolType], MM::Paging::GetPteDistance(StartingPte, SystemPteBase));
+
+    /* Use the second PTE slot to store the total size of this pool */
+    StartingPte = MM::Paging::GetNextPte(StartingPte);
+    MM::Paging::ClearPte(StartingPte);
+    MM::Paging::SetNextEntry(StartingPte, NumberOfPtes);
+
+    /* Record the total number of free PTEs in this pool */
+    TotalSystemFreePtes[PoolType] = NumberOfPtes;
+}
+
+/**
+ * Sets up the entire system PTE address space.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::InitializeSystemPteSpace(VOID)
+{
+    PMMPTE PointerPte;
+    PMMPTE FirstZeroingPte;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    ULONGLONG NonPagedSystemPoolEnd;
+
+    /* Retrieve the system's memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    NonPagedSystemPoolEnd = ((ULONGLONG)MemoryLayout->NonPagedSystemPoolStart +
+                             MM::Manager::GetNumberOfSystemPtes() * MM_PAGE_SIZE);
+
+    /* Map the page table hierarchy for the entire system PTE space */
+    MM::Pte::MapPPE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)NonPagedSystemPoolEnd, &ValidPte);
+    MM::Pte::MapPDE(MemoryLayout->NonPagedSystemPoolStart, (PVOID)NonPagedSystemPoolEnd, &ValidPte);
+
+    /* Format the main block of system PTEs into a free list pool */
+    PointerPte = MM::Paging::GetPteAddress(MemoryLayout->NonPagedSystemPoolStart);
+
+    InitializeSystemPtePool(PointerPte, MM::Manager::GetNumberOfSystemPtes(), SystemPteSpace);
+
+    /* Reserve and zero a dedicated block of system PTEs */
+    FirstZeroingPte = ReserveSystemPtes(MM_RESERVED_ZERO_PTES + 1, SystemPteSpace);
+    RTL::Memory::ZeroMemory(FirstZeroingPte, (MM_RESERVED_ZERO_PTES + 1) * MM::Paging::GetPteSize());
+
+    /* Use the first PTE of this block as a counter for available zeroing PTEs */
+    MM::Paging::SetPte(FirstZeroingPte, MM_RESERVED_ZERO_PTES, 0);
+}
+
+/**
+ * Maps a range of virtual addresses at the PDE (Page Directory Entry) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplatePde
+ *        A template PDE to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapPDE(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMPDE TemplatePde)
+{
+    PMMPDE EndSpace, PointerPde;
+
+    /* Get PDE addresses */
+    PointerPde = MM::Paging::GetPdeAddress(StartAddress);
+    EndSpace = MM::Paging::GetPdeAddress(EndAddress);
+
+    /* Iterate over all PDEs */
+    while(PointerPde <= EndSpace)
+    {
+        /* Check if PDE is already mapped */
+        if(!MM::Paging::PteValid(PointerPde))
+        {
+            /* Map PDE */
+            MM::Paging::SetPte(TemplatePde, MM::Pfn::AllocateBootstrapPages(1), 0);
+            MM::Paging::WritePte(PointerPde, *TemplatePde);
+
+            /* Clear the page table */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPde), MM_PAGE_SIZE);
+        }
+
+        /* Get next table entry */
+        PointerPde = MM::Paging::GetNextPte(PointerPde);
+    }
+}
+
+/**
+ * Maps a range of virtual addresses at the PTE (Page Table Entry) level.
+ *
+ * @param StartAddress
+ *        The beginning of the virtual address range to map.
+ *
+ * @param EndAddress
+ *        The end of the virtual address range to map.
+ *
+ * @param TemplatePte
+ *        A template PTE to use for creating new entries.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::MapPTE(IN PVOID StartAddress,
+                IN PVOID EndAddress,
+                IN PMMPTE TemplatePte)
+{
+    PMMPTE EndSpace, PointerPte;
+
+    /* Get PTE addresses */
+    PointerPte = MM::Paging::GetPteAddress(StartAddress);
+    EndSpace = MM::Paging::GetPteAddress(EndAddress);
+
+    /* Iterate over all PTEs */
+    while(PointerPte <= EndSpace)
+    {
+        /* Check if PTE is already mapped */
+        if(!MM::Paging::PteValid(PointerPte))
+        {
+            /* Map PTE */
+            MM::Paging::SetPte(TemplatePte, MM::Pfn::AllocateBootstrapPages(1), 0);
+            MM::Paging::WritePte(PointerPte, *TemplatePte);
+
+            /* Clear the page table */
+            RTL::Memory::ZeroMemory(MM::Paging::GetPteVirtualAddress(PointerPte), MM_PAGE_SIZE);
+        }
+
+        /* Get next table entry */
+        PointerPte = MM::Paging::GetNextPte(PointerPte);
+    }
+}
+
+
+/**
+ * Releases a block of system PTEs into a specified pool.
+ *
+ * @param StartingPte
+ *        A pointer to the first PTE to release.
+ *
+ * @param NumberOfPtes
+ *        The number of contiguous PTEs to release.
+ *
+ * @param SystemPtePoolType
+ *        Specifies the system PTE pool to release into.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pte::ReleaseSystemPtes(IN PMMPTE StartingPte,
+                           IN PFN_COUNT NumberOfPtes,
+                           IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)
+{
+    PMMPTE NextPte, PreviousPte, ReleasedPte;
+    ULONG ClusterSize;
+
+    /* Clear the PTEs before releasing them */
+    RtlZeroMemory(StartingPte, NumberOfPtes * MM::Paging::GetPteSize());
+
+    /* Raise runlevel and acquire lock to protect the PTE pool */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL);
+    KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+    /* Increment the total number of available PTEs in this pool */
+    TotalSystemFreePtes[SystemPtePoolType] += NumberOfPtes;
+
+    /* Start at the head of the free list for this pool */
+    PreviousPte = &FirstSystemFreePte[SystemPtePoolType];
+    ReleasedPte = NULLPTR;
+
+    /* Iterate through the free list to find adjacent blocks */
+    while(MM::Paging::GetNextEntry(PreviousPte) != MAXULONG)
+    {
+        /* Get the next free cluster to check its size */
+        NextPte = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousPte));
+        ClusterSize = GetClusterSize(NextPte);
+
+        /* Check if the released block is adjacent to the current free block */
+        if((MM::Paging::AdvancePte(NextPte, ClusterSize) == StartingPte) ||
+           (MM::Paging::AdvancePte(StartingPte, NumberOfPtes) == NextPte))
+        {
+            /* Merge the blocks by adding their sizes */
+            NumberOfPtes += ClusterSize;
+
+            /* Check if the current free block is before the released block */
+            if(NextPte < StartingPte)
+            {
+                /* The new merged block starts at the current free block's address */
+                StartingPte = NextPte;
+            }
+
+            /* Unlink the current free block as it is being merged */
+            MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetNextEntry(NextPte));
+
+            /* Check if the block represents more than one PTE */
+            if(!MM::Paging::GetOneEntry(NextPte))
+            {
+                /* Clear block header and move to the size PTE */
+                MM::Paging::ClearPte(NextPte);
+                NextPte = MM::Paging::GetNextPte(NextPte);
+            }
+
+            /* Clear the merged block */
+            MM::Paging::ClearPte(NextPte);
+
+            /* Reset insertion point since block size/address changed due to merge */
+            ReleasedPte = NULLPTR;
+        }
+        else
+        {
+            /* Select the first free block large enough as insertion point */
+            if((ReleasedPte == NULLPTR) && (NumberOfPtes <= ClusterSize))
+            {
+                /* Mark this as the insertion point */
+                ReleasedPte = PreviousPte;
+            }
+
+            /* Advance to the next free block */
+            PreviousPte = NextPte;
+        }
+    }
+
+    /* Check if there is only one PTE to release */
+    if(NumberOfPtes == 1)
+    {
+        /* Mark it as a single-PTE block */
+        MM::Paging::SetOneEntry(StartingPte, 1);
+    }
+    else
+    {
+        /* Otherwise, mark it as a multi-PTE block */
+        MM::Paging::SetOneEntry(StartingPte, 0);
+
+        /* The next PTE stores the size of the block */
+        NextPte = MM::Paging::GetNextPte(StartingPte);
+        MM::Paging::SetNextEntry(NextPte, NumberOfPtes);
+    }
+
+    /* Check if no suitable insertion point was found */
+    if(ReleasedPte == NULLPTR)
+    {
+        /* Insert at the end of the list */
+        ReleasedPte = PreviousPte;
+    }
+
+    /* Link the new block into the free list */
+    MM::Paging::SetNextEntry(StartingPte, MM::Paging::GetNextEntry(ReleasedPte));
+    MM::Paging::SetNextEntry(ReleasedPte, MM::Paging::GetPteDistance(StartingPte, SystemPteBase));
+}
+
+/**
+ * Reserves a contiguous block of system PTEs from a specified pool.
+ *
+ * @param NumberOfPtes
+ *        The number of contiguous PTEs to reserve.
+ *
+ * @param SystemPtePoolType
+ *        Specifies the system PTE pool from which to allocate.
+ *
+ * @return This routine returns a pointer to the beginning of the reserved block,
+ *         or NULLPTR if not enough contiguous PTEs are available.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PMMPTE
+MM::Pte::ReserveSystemPtes(IN PFN_COUNT NumberOfPtes,
+                           IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType)
+{
+    PMMPTE NextPte, PreviousPte, ReservedPte;
+    ULONG ClusterSize;
+
+    /* Raise runlevel and acquire lock to protect the PTE pool */
+    KE::RaiseRunLevel RunLevel(DISPATCH_LEVEL);
+    KE::QueuedSpinLockGuard SpinLock(SystemSpaceLock);
+
+    /* Find a free PTE cluster large enough for the request */
+    if(!FindFreeCluster(NumberOfPtes, SystemPtePoolType, &NextPte, &PreviousPte))
+    {
+        /* Out of system PTEs for this pool, return NULLPTR */
+        return NULLPTR;
+    }
+
+    /* We have the cluster, now get its size for the allocation logic below */
+    ClusterSize = GetClusterSize(NextPte);
+
+    /* Unlink the found cluster from the free list for processing */
+    MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetNextEntry(NextPte));
+
+    /* Handle the allocation based on whether the cluster size is an exact match */
+    if(ClusterSize == NumberOfPtes)
+    {
+        /* Exact match, allocate the entire cluster */
+        ReservedPte = NextPte;
+
+        /* Handle metadata cleanup for a single-PTE cluster */
+        if(MM::Paging::GetOneEntry(NextPte))
+        {
+            /* Clear the PTE that held the list metadata */
+            MM::Paging::ClearPte(NextPte);
+            NextPte = MM::Paging::GetNextPte(NextPte);
+        }
+
+        /* Clear the PTE that held the cluster size */
+        MM::Paging::ClearPte(NextPte);
+    }
+    else
+    {
+        /* Cluster is larger than needed, so it will be split */
+        ClusterSize -= NumberOfPtes;
+        ReservedPte = MM::Paging::AdvancePte(NextPte, ClusterSize);
+
+        /* Update metadata for the new, smaller leftover cluster */
+        if(ClusterSize == 1)
+        {
+            /* The leftover fragment is a single PTE */
+            MM::Paging::SetOneEntry(NextPte, 1);
+            MM::Paging::ClearPte(ReservedPte);
+        }
+        else
+        {
+            /* The leftover fragment is larger than one PTE */
+            NextPte = MM::Paging::GetNextPte(NextPte);
+            MM::Paging::SetNextEntry(NextPte, ClusterSize);
+        }
+
+        /* Find the correct sorted position to re-insert the leftover fragment */
+        PreviousPte = &FirstSystemFreePte[SystemPtePoolType];
+        while(MM::Paging::GetNextEntry(PreviousPte) != MAXULONG)
+        {
+            /* Get the next free cluster to check its size */
+            NextPte = MM::Paging::AdvancePte(SystemPteBase, MM::Paging::GetNextEntry(PreviousPte));
+
+            /* Check if the leftover fragment should be inserted here */
+            if(ClusterSize <= GetClusterSize(NextPte))
+            {
+                /* Found the correct sorted position */
+                break;
+            }
+
+            /* Advance to the next entry */
+            PreviousPte = NextPte;
+        }
+
+        /* Get a pointer to the start of the leftover fragment */
+        NextPte = MM::Paging::AdvancePte(ReservedPte, -ClusterSize);
+
+        /* Insert the leftover fragment back into the free list at its sorted position */
+        MM::Paging::SetNextEntry(NextPte, MM::Paging::GetNextEntry(PreviousPte));
+        MM::Paging::SetNextEntry(PreviousPte, MM::Paging::GetPteDistance(NextPte, SystemPteBase));
+    }
+
+    /* Decrement the total number of available PTEs in this pool */
+    TotalSystemFreePtes[SystemPtePoolType] -= NumberOfPtes;
+
+    /* Flush the TLB to ensure address translation consistency */
+    AR::CpuFunc::FlushTlb();
+
+    /* Return a pointer to the start of the reserved PTE block */
+    return ReservedPte;
+}