Merge branch 'master' into memmgr

boot/xtldr/arch/amd64/memory.cc

@@ -28,13 +28,12 @@ EFI_STATUS
 Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
                      IN ULONG_PTR SelfMapAddress)
 {
-    PLIST_ENTRY ListEntry, ModulesList, ModulesListEntry;
+    PLIST_ENTRY ModulesList, ModulesListEntry;
-    PXTBL_MEMORY_MAPPING Mapping;
     PXTBL_MODULE_INFO ModuleInfo;
     EFI_PHYSICAL_ADDRESS Address;
-    PVOID LoaderBase;
     ULONGLONG LoaderSize;
     EFI_STATUS Status;
+    PVOID LoaderBase;
 
     /* Allocate pages for the Page Map */
     Status = AllocatePages(AllocateAnyPages, 1, &Address);
@@ -65,7 +64,7 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Map the trampoline code area */
-    Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS,MM_TRAMPOLINE_ADDRESS,
+    Status = MapVirtualMemory(PageMap, MM_TRAMPOLINE_ADDRESS, MM_TRAMPOLINE_ADDRESS,
                               1, LoaderFirmwareTemporary);
     if(Status != STATUS_EFI_SUCCESS)
     {
@@ -117,6 +116,28 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         return STATUS_EFI_PROTOCOL_ERROR;
     }
 
+    /* Return success */
+    return STATUS_EFI_SUCCESS;
+}
+
+/**
+ * Iterates through the memory map and physically maps all virtual addresses to page tables.
+ *
+ * @param PageMap
+ *        Supplies a pointer to the page mapping structure.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTCDECL
+EFI_STATUS
+Memory::CommitPageMap(IN PXTBL_PAGE_MAPPING PageMap)
+{
+    PXTBL_MEMORY_MAPPING Mapping;
+    PLIST_ENTRY ListEntry;
+    EFI_STATUS Status;
+
     /* Iterate through and map all the mappings*/
     Debug::Print(L"Mapping and dumping EFI memory:\n");
     ListEntry = PageMap->MemoryMap.Flink;

boot/xtldr/arch/i686/memory.cc

@@ -25,13 +25,12 @@ EFI_STATUS
 Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
                      IN ULONG_PTR SelfMapAddress)
 {
-    PLIST_ENTRY ListEntry, ModulesList, ModulesListEntry;
     EFI_PHYSICAL_ADDRESS Address, DirectoryAddress;
+    PLIST_ENTRY ModulesList, ModulesListEntry;
     PXTBL_MODULE_INFO ModuleInfo;
-    PXTBL_MEMORY_MAPPING Mapping;
-    PVOID LoaderBase;
     ULONGLONG LoaderSize;
     EFI_STATUS Status;
+    PVOID LoaderBase;
     ULONG Index;
 
     /* Check the page map level to determine which paging structure to create */
@@ -45,6 +44,14 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
             return Status;
         }
 
+        /* Add new memory mapping for the page map itself */
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+        if(Status != STATUS_EFI_SUCCESS)
+        {
+            /* Memory mapping failure */
+            return Status;
+        }
+
         /* Assign the allocated page to the page map and zero it out */
         PageMap->PtePointer = (PVOID)(UINT_PTR)Address;
         RTL::Memory::ZeroMemory(PageMap->PtePointer, EFI_PAGE_SIZE);
@@ -57,6 +64,14 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
             return Status;
         }
 
+        /* Add new memory mapping for the Page Directories (PDs) */
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, DirectoryAddress, 4, LoaderMemoryData);
+        if(Status != STATUS_EFI_SUCCESS)
+        {
+            /* Memory mapping failure */
+            return Status;
+        }
+
         /* Zero-fill the allocated memory for the Page Directories */
         RTL::Memory::ZeroMemory((PVOID)DirectoryAddress, EFI_PAGE_SIZE * 4);
 
@@ -79,6 +94,14 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
             return Status;
         }
 
+        /* Add new memory mapping for the page map itself */
+        Status = MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+        if(Status != STATUS_EFI_SUCCESS)
+        {
+            /* Memory mapping failure */
+            return Status;
+        }
+
         /* Assign the allocated page to the page map and zero it out */
         PageMap->PtePointer = (PVOID)(UINT_PTR)Address;
         RTL::Memory::ZeroMemory(PageMap->PtePointer, EFI_PAGE_SIZE);
@@ -144,6 +167,28 @@ Memory::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
         return STATUS_EFI_PROTOCOL_ERROR;
     }
 
+    /* Return success */
+    return STATUS_EFI_SUCCESS;
+}
+
+/**
+ * Iterates through the memory map and physically maps all virtual addresses to page tables.
+ *
+ * @param PageMap
+ *        Supplies a pointer to the page mapping structure.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTCDECL
+EFI_STATUS
+Memory::CommitPageMap(IN PXTBL_PAGE_MAPPING PageMap)
+{
+    PXTBL_MEMORY_MAPPING Mapping;
+    PLIST_ENTRY ListEntry;
+    EFI_STATUS Status;
+
     /* Iterate through and map all the mappings*/
     Debug::Print(L"Mapping and dumping EFI memory:\n");
     ListEntry = PageMap->MemoryMap.Flink;

boot/xtldr/includes/xtldr.hh

@@ -165,6 +165,7 @@ class Memory
                                                OUT PVOID *Memory);
         STATIC XTCDECL EFI_STATUS BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap,
                                                IN ULONG_PTR SelfMapAddress);
+        STATIC XTCDECL EFI_STATUS CommitPageMap(IN PXTBL_PAGE_MAPPING PageMap);
         STATIC XTCDECL EFI_STATUS FreePages(IN ULONGLONG NumberOfPages,
                                             IN EFI_PHYSICAL_ADDRESS Memory);
         STATIC XTCDECL EFI_STATUS FreePool(IN PVOID Memory);

boot/xtldr/memory.cc

@@ -547,7 +547,8 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             /* Make sure it's memory type is the same */
             if(Mapping1->MemoryType == Mapping2->MemoryType)
             {
-                /* It is already mapped */
+                /* Free the unused mapping structure and return success */
+                FreePool(Mapping1);
                 return STATUS_EFI_SUCCESS;
             }
         }
@@ -574,12 +575,15 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
                     return Status;
                 }
 
-                /* Set mapping fields and insert it on the top */
+                /* Set mapping fields */
                 Mapping3->PhysicalAddress = PhysicalAddressEnd + 1;
                 Mapping3->VirtualAddress = (ULONGLONG)NULLPTR;
                 Mapping3->NumberOfPages = NumberOfMappedPages;
                 Mapping3->MemoryType = Mapping2->MemoryType;
+
+                /* Insert new mapping in front of the list and increase page map size */
                 RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry);
+                PageMap->MapSize++;
             }
 
             /* Calculate number of pages and the end of the physical address */
@@ -610,12 +614,15 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
                     return Status;
                 }
 
-                /* Set mapping fields and insert it on the top */
+                /* Set mapping fields */
                 Mapping3->PhysicalAddress = Mapping1->PhysicalAddress;
                 Mapping3->VirtualAddress = (ULONGLONG)NULLPTR;
                 Mapping3->NumberOfPages = NumberOfMappedPages;
                 Mapping3->MemoryType = Mapping2->MemoryType;
+
+                /* Insert new mapping in front of the list and increase page map size */
                 RTL::LinkedList::InsertHeadList(&Mapping2->ListEntry, &Mapping3->ListEntry);
+                PageMap->MapSize++;
             }
 
             /* Calculate number of pages and the end of the physical address */
@@ -643,15 +650,19 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
             Status = FreePool(Mapping2);
             ListEntry = MappingListEntry;
 
-            /* Go to the next mapping */
+            /* Decrease page map size and go to the next mapping */
+            PageMap->MapSize--;
             continue;
         }
 
         /* Determine physical address order */
         if(Mapping2->PhysicalAddress > Mapping1->PhysicalAddress)
         {
-            /* Insert new mapping in front */
+            /* Insert new mapping in front of the list and increase page map size */
             RTL::LinkedList::InsertHeadList(Mapping2->ListEntry.Blink, &Mapping1->ListEntry);
+            PageMap->MapSize++;
+
+            /* Return success */
             return STATUS_EFI_SUCCESS;
         }
 
@@ -659,7 +670,7 @@ Memory::MapVirtualMemory(IN OUT PXTBL_PAGE_MAPPING PageMap,
         ListEntry = ListEntry->Flink;
     }
 
-    /* Insert new mapping to the list and increase page map size */
+    /* Insert new mapping to the tail of the list and increase page map size */
     RTL::LinkedList::InsertTailList(&PageMap->MemoryMap, &Mapping1->ListEntry);
     PageMap->MapSize++;
 

boot/xtldr/modules/xtos_o/amd64/memory.cc

@@ -10,6 +10,34 @@
 #include <xtos.hh>
 
 
+XTCDECL
+EFI_STATUS
+Xtos::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap)
+{
+    EFI_STATUS Status;
+
+    /* Build page map */
+    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, (PageMap->PageMapLevel > 4) ? MM_P5E_LA57_BASE : MM_PXE_BASE);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to build page map */
+        XtLdrProtocol->Debug.Print(L"Failed to build page map (Status code: %zX)\n", Status);
+        return Status;
+    }
+
+    /* Map memory for hardware layer */
+    Status = MapHardwareMemoryPool(PageMap);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to map memory for hardware layer */
+        XtLdrProtocol->Debug.Print(L"Failed to map memory for hardware leyer (Status code: %zX)\n", Status);
+        return Status;
+    }
+
+    /* Return success */
+    return STATUS_EFI_SUCCESS;
+}
+
 /**
  * Determines the appropriate EFI memory mapping strategy for the AMD64 architecture.
  *
@@ -91,24 +119,6 @@ Xtos::EnablePaging(IN PXTBL_PAGE_MAPPING PageMap)
     ULONG_PTR TrampolineSize;
     PVOID TrampolineCode;
 
-    /* Build page map */
-    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, (PageMap->PageMapLevel > 4) ? MM_P5E_LA57_BASE : MM_PXE_BASE);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Failed to build page map */
-        XtLdrProtocol->Debug.Print(L"Failed to build page map (Status code: %zX)\n", Status);
-        return Status;
-    }
-
-    /* Map memory for hardware layer */
-    Status = MapHardwareMemoryPool(PageMap);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Failed to map memory for hardware layer */
-        XtLdrProtocol->Debug.Print(L"Failed to map memory for hardware leyer (Status code: %zX)\n", Status);
-        return Status;
-    }
-
     /* Check the configured page map level to set the LA57 state accordingly */
     if(PageMap->PageMapLevel == 5)
     {
@@ -188,6 +198,7 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
     PHARDWARE_PTE P5eBase, PdeBase, PpeBase, PxeBase;
     EFI_PHYSICAL_ADDRESS Address;
     EFI_STATUS Status;
+    ULONG Index;
 
     if(PageMap->PageMapLevel == 5)
     {
@@ -205,6 +216,9 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
                 return Status;
             }
 
+            /* Map hardware memory */
+            XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+
             /* Zero fill memory used by P5E */
             XtLdrProtocol->Memory.ZeroMemory((PVOID)Address, EFI_PAGE_SIZE);
 
@@ -239,6 +253,9 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
             return Status;
         }
 
+        /* Map hardware memory */
+        XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+
         /* Zero fill memory used by PXE */
         XtLdrProtocol->Memory.ZeroMemory((PVOID)Address, EFI_PAGE_SIZE);
 
@@ -267,6 +284,9 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
             return Status;
         }
 
+        /* Map hardware memory */
+        XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+
         /* Zero fill memory used by PPE */
         XtLdrProtocol->Memory.ZeroMemory((PVOID)Address, EFI_PAGE_SIZE);
 
@@ -285,7 +305,7 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
     }
 
     /* Loop through 2 PDE entries */
-    for(UINT Index = 0 ; Index < 2 ; Index++)
+    for(Index = 0 ; Index < 2 ; Index++)
     {
         /* Check if PDE entry already exists */
         if(!PdeBase[((MM_HARDWARE_VA_START >> MM_PDI_SHIFT) & 0x1FF) + Index].Valid)
@@ -298,6 +318,9 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
                 return Status;
             }
 
+            /* Map hardware memory */
+            XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+
             /* Zero fill memory used by PDE */
             XtLdrProtocol->Memory.ZeroMemory((PVOID)Address, EFI_PAGE_SIZE);
 

boot/xtldr/modules/xtos_o/i686/memory.cc

@@ -9,6 +9,47 @@
 #include <xtos.hh>
 
 
+XTCDECL
+EFI_STATUS
+Xtos::BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap)
+{
+    ULONG_PTR SelfMapAddress;
+    EFI_STATUS Status;
+
+    /* Initialize self map address */
+    if(PageMap->PageMapLevel == 3)
+    {
+        /* For PML3 (PAE) use PTE base address */
+        SelfMapAddress = MM_PTE_BASE;
+    }
+    else
+    {
+        /* For PML2 (PAE disabled) use legacy PDE base address */
+        SelfMapAddress = MM_PDE_LEGACY_BASE;
+    }
+
+    /* Build page map */
+    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, SelfMapAddress);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to build page map */
+        XtLdrProtocol->Debug.Print(L"Failed to build page map (Status code: %zX)\n", Status);
+        return Status;
+    }
+
+    /* Map memory for hardware layer */
+    Status = MapHardwareMemoryPool(PageMap);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to map memory for hardware layer */
+        XtLdrProtocol->Debug.Print(L"Failed to map memory for hardware layer (Status code: %zX)\n", Status);
+        return Status;
+    }
+
+    /* Return success */
+    return STATUS_EFI_SUCCESS;
+}
+
 /**
  * Determines the appropriate EFI memory mapping strategy for the i686 architecture.
  *
@@ -74,37 +115,6 @@ EFI_STATUS
 Xtos::EnablePaging(IN PXTBL_PAGE_MAPPING PageMap)
 {
     EFI_STATUS Status;
-    ULONG_PTR SelfMapAddress;
-
-    /* Initialize self map address */
-    if(PageMap->PageMapLevel == 3)
-    {
-        /* For PML3 (PAE) use PTE base address */
-        SelfMapAddress = MM_PTE_BASE;
-    }
-    else
-    {
-        /* For PML2 (PAE disabled) use legacy PDE base address */
-        SelfMapAddress = MM_PDE_LEGACY_BASE;
-    }
-
-    /* Build page map */
-    Status = XtLdrProtocol->Memory.BuildPageMap(PageMap, SelfMapAddress);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Failed to build page map */
-        XtLdrProtocol->Debug.Print(L"Failed to build page map (Status code: %zX)\n", Status);
-        return Status;
-    }
-
-    /* Map memory for hardware layer */
-    Status = MapHardwareMemoryPool(PageMap);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Failed to map memory for hardware layer */
-        XtLdrProtocol->Debug.Print(L"Failed to map memory for hardware layer (Status code: %zX)\n", Status);
-        return Status;
-    }
 
     /* Exit EFI Boot Services */
     XtLdrProtocol->Debug.Print(L"Exiting EFI boot services\n");
@@ -173,6 +183,9 @@ Xtos::MapHardwareMemoryPool(IN PXTBL_PAGE_MAPPING PageMap)
     /* Zero fill allocated memory */
     XtLdrProtocol->Memory.ZeroMemory((PVOID)Address, EFI_PAGE_SIZE);
 
+    /* Map hardware memory */
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)NULLPTR, Address, 1, LoaderMemoryData);
+
     /* Check if PAE is enabled (3-level paging) */
     if(PageMap->PageMapLevel == 3)
     {

boot/xtldr/modules/xtos_o/includes/xtos.hh

@@ -38,6 +38,7 @@ class Xtos
                                                           IN PVOID PhysicalAddress,
                                                           IN UINT NumberOfPages,
                                                           IN LOADER_MEMORY_TYPE MemoryType);
+        STATIC XTCDECL EFI_STATUS BuildPageMap(IN PXTBL_PAGE_MAPPING PageMap);
         STATIC XTCDECL LOADER_MEMORY_TYPE ConvertEfiMemoryType(IN EFI_MEMORY_TYPE EfiMemoryType);
         STATIC XTCDECL BOOLEAN DetermineMappingStrategy();
         STATIC XTCDECL ULONG DeterminePagingLevel(IN CONST PWCHAR Parameters);
@@ -47,10 +48,13 @@ class Xtos
                                                   IN PULONG_PTR FrameBufferSize,
                                                   IN PXTBL_FRAMEBUFFER_MODE_INFORMATION FrameBufferModeInfo);
         STATIC XTCDECL EFI_STATUS GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap,
-                                                          IN PVOID *VirtualAddress,
+                                                          IN EFI_PHYSICAL_ADDRESS PhysicalBase,
+                                                          IN PVOID VirtualBase,
                                                           OUT PLIST_ENTRY MemoryDescriptorList);
         STATIC XTCDECL EFI_STATUS GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
-                                                         IN PVOID *VirtualAddress,
+                                                         IN EFI_PHYSICAL_ADDRESS PhysicalBase,
+                                                         IN PVOID VirtualBase,
+                                                         IN PVOID FrameBufferVirtualBase,
                                                          OUT PLIST_ENTRY SystemResourcesList);
         STATIC XTCDECL EFI_STATUS GetVirtualAddress(IN PLIST_ENTRY MemoryMappings,
                                                     IN PVOID PhysicalAddress,

boot/xtldr/modules/xtos_o/xtos.cc

@@ -191,38 +191,16 @@ Xtos::GetDisplayInformation(OUT PSYSTEM_RESOURCE_FRAMEBUFFER FrameBufferResource
 XTCDECL
 EFI_STATUS
 Xtos::GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap,
-                              IN PVOID *VirtualAddress,
+                              IN EFI_PHYSICAL_ADDRESS PhysicalBase,
+                              IN PVOID VirtualBase,
                               OUT PLIST_ENTRY MemoryDescriptorList)
 {
     PLOADER_MEMORY_DESCRIPTOR Descriptor;
     PXTBL_MEMORY_MAPPING MemoryMapping;
-    EFI_PHYSICAL_ADDRESS Address;
     PLIST_ENTRY ListEntry;
-    EFI_STATUS Status;
-    ULONGLONG Pages;
-
-    /* Calculate the number of pages required to store the memory descriptor array */
-    Pages = (ULONGLONG)EFI_SIZE_TO_PAGES((PageMap->MapSize + 1) * sizeof(LOADER_MEMORY_DESCRIPTOR));
-
-    /* Allocate physical pages to hold the memory descriptor list */
-    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, Pages, &Address);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Page allocation failed, return the status code */
-        return Status;
-    }
-
-    /* Create a virtual memory mapping for the allocated descriptor buffer */
-    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderMemoryData);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        /* Release the allocated pages as the virtual mapping failed and return status code */
-        XtLdrProtocol->Memory.FreePages(Address, Pages);
-        return Status;
-    }
 
     /* Initialize the descriptor pointer to the start of the allocated physical buffer */
-    Descriptor = (PLOADER_MEMORY_DESCRIPTOR)Address;
+    Descriptor = (PLOADER_MEMORY_DESCRIPTOR)PhysicalBase;
 
     /* Get the first entry from the internal boot loader memory map */
     ListEntry = PageMap->MemoryMap.Flink;
@@ -247,17 +225,18 @@ Xtos::GetMemoryDescriptorList(IN PXTBL_PAGE_MAPPING PageMap,
     }
 
     /* Convert all physical link pointers in the list to their corresponding virtual addresses */
-    XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, MemoryDescriptorList, (PVOID)Address, *VirtualAddress);
+    XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, MemoryDescriptorList, (PVOID)PhysicalBase, VirtualBase);
 
-    /* Advance the virtual address pointer to the next available free region and return success */
+    /* Return success */
-    *VirtualAddress = (PUINT8)*VirtualAddress + (Pages * EFI_PAGE_SIZE);
     return STATUS_EFI_SUCCESS;
 }
 
 XTCDECL
 EFI_STATUS
 Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
-                             IN PVOID *VirtualAddress,
+                             IN EFI_PHYSICAL_ADDRESS PhysicalBase,
+                             IN PVOID VirtualBase,
+                             IN PVOID FrameBufferVirtualBase,
                              OUT PLIST_ENTRY SystemResourcesList)
 {
     XTSTATUS Status;
@@ -268,39 +247,18 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
     PXTBL_FRAMEBUFFER_PROTOCOL FrameBufProtocol;
     XTBL_FRAMEBUFFER_MODE_INFORMATION FbModeInfo;
     EFI_PHYSICAL_ADDRESS FbAddress;
+    EFI_PHYSICAL_ADDRESS OriginalPhysicalBase;
     ULONG_PTR FbSize;
-    UINT FrameBufferPages;
     PSYSTEM_RESOURCE_FRAMEBUFFER FrameBufferResource;
     PSYSTEM_RESOURCE_ACPI AcpiResource;
-    ULONGLONG Pages;
-    EFI_PHYSICAL_ADDRESS Address;
-    PVOID PhysicalBase, VirtualBase;
 
-    Pages = (ULONGLONG)EFI_SIZE_TO_PAGES(sizeof(SYSTEM_RESOURCE_ACPI) + sizeof(SYSTEM_RESOURCE_FRAMEBUFFER));
+    /* Save original physical base */
-
+    OriginalPhysicalBase = PhysicalBase;
-    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, Pages, &Address);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        return Status;
-    }
-    Status = XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, Address, Pages, LoaderFirmwarePermanent);
-    if(Status != STATUS_EFI_SUCCESS)
-    {
-        XtLdrProtocol->Memory.FreePages(Address, Pages);
-        return Status;
-    }
-
-    PhysicalBase = (PVOID)Address;
-    VirtualBase = *VirtualAddress;
-
-    /* Calculate next valid virtual address */
-    *VirtualAddress = (PUINT8)*VirtualAddress + (Pages * EFI_PAGE_SIZE);
-
-    AcpiResource = (PSYSTEM_RESOURCE_ACPI)Address;
 
+    AcpiResource = (PSYSTEM_RESOURCE_ACPI)PhysicalBase;
     XtLdrProtocol->Memory.ZeroMemory(AcpiResource, sizeof(SYSTEM_RESOURCE_ACPI));
 
-    /* Load FrameBuffer protocol */
+    /* Load ACPI protocol */
     Status = XtLdrProtocol->Protocol.Open(&ProtocolHandle, (PVOID*)&AcpiProtocol, &AcpiGuid);
     if(Status != STATUS_EFI_SUCCESS)
     {
@@ -319,13 +277,11 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
 
     XtLdrProtocol->LinkedList.InsertTail(SystemResourcesList, &AcpiResource->Header.ListEntry);
 
-    /* Close FrameBuffer protocol */
+    /* Close ACPI protocol */
-    XtLdrProtocol->Protocol.Close(&ProtocolHandle, &FrameBufGuid);
+    XtLdrProtocol->Protocol.Close(&ProtocolHandle, &AcpiGuid);
-
-    Address = Address + sizeof(SYSTEM_RESOURCE_ACPI);
-
-    FrameBufferResource = (PSYSTEM_RESOURCE_FRAMEBUFFER)Address;
 
+    PhysicalBase = PhysicalBase + sizeof(SYSTEM_RESOURCE_ACPI);
+    FrameBufferResource = (PSYSTEM_RESOURCE_FRAMEBUFFER)PhysicalBase;
     XtLdrProtocol->Memory.ZeroMemory(FrameBufferResource, sizeof(SYSTEM_RESOURCE_FRAMEBUFFER));
 
     /* Load FrameBuffer protocol */
@@ -346,26 +302,16 @@ Xtos::GetSystemResourcesList(IN PXTBL_PAGE_MAPPING PageMap,
         return Status;
     }
 
-    /* Calculate pages needed to map framebuffer */
+    /* Assign the pre-mapped virtual address to the resource block */
-    FrameBufferPages = EFI_SIZE_TO_PAGES(FbSize);
+    FrameBufferResource->Header.VirtualAddress = FrameBufferVirtualBase;
-
-    /* Rewrite framebuffer address by using virtual address */
-    FrameBufferResource->Header.VirtualAddress = *VirtualAddress;
-
-    /* Map frame buffer memory */
-    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)FrameBufferResource->Header.VirtualAddress,
-                                           (ULONGLONG)FrameBufferResource->Header.PhysicalAddress,
-                                           FrameBufferPages, LoaderFirmwarePermanent);
-
-    /* Close FrameBuffer protocol */
     XtLdrProtocol->Protocol.Close(&ProtocolHandle, &FrameBufGuid);
 
-    *VirtualAddress = (PUINT8)*VirtualAddress + (FrameBufferPages * EFI_PAGE_SIZE);
-
     XtLdrProtocol->LinkedList.InsertTail(SystemResourcesList, &FrameBufferResource->Header.ListEntry);
 
-    XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, SystemResourcesList, PhysicalBase, VirtualBase);
+    /* Convert list pointers to virtual */
+    XtLdrProtocol->Memory.PhysicalListToVirtual(PageMap, SystemResourcesList, (PVOID)OriginalPhysicalBase, VirtualBase);
 
+    /* Return success */
     return STATUS_EFI_SUCCESS;
 }
 
@@ -426,34 +372,100 @@ Xtos::InitializeApicBase(IN PXTBL_PAGE_MAPPING PageMap)
 XTCDECL
 EFI_STATUS
 Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap,
-                            IN PVOID *VirtualAddress,
+                            IN OUT PVOID *VirtualAddress,
                             IN PXTBL_BOOT_PARAMETERS Parameters)
 {
+    EFI_PHYSICAL_ADDRESS FbPhysicalAddress, PhysicalBlock, PhysicalDescriptor, PhysicalResources;
+    PVOID FbVirtualAddress, VirtualBlock, VirtualResources, VirtualDescriptor;
+    UINT BlockPages, DescriptorPages, FbPages, ParametersSize, ResourcesPages;
+    XTBL_FRAMEBUFFER_MODE_INFORMATION FbModeInfo;
+    PXTBL_FRAMEBUFFER_PROTOCOL FrameBufProtocol;
     PKERNEL_INITIALIZATION_BLOCK LoaderBlock;
-    EFI_PHYSICAL_ADDRESS Address;
+    EFI_HANDLE ProtocolHandle;
     EFI_STATUS Status;
-    UINT BlockPages;
+    ULONG_PTR FbSize;
-    UINT ParametersSize;
+
+    EFI_GUID FrameBufGuid = XT_FRAMEBUFFER_PROTOCOL_GUID;
+
+    /* Initialize Framebuffer information */
+    FbPhysicalAddress = 0;
+    FbSize = 0;
+    FbVirtualAddress = NULLPTR;
+    FbPages = 0;
 
     /* Calculate size of parameters */
     ParametersSize = (XtLdrProtocol->WideString.Length(Parameters->Parameters, 0) + 1) * sizeof(WCHAR);
 
     /* Calculate number of pages needed for initialization block */
     BlockPages = EFI_SIZE_TO_PAGES(sizeof(KERNEL_INITIALIZATION_BLOCK) + ParametersSize);
+    ResourcesPages = EFI_SIZE_TO_PAGES(sizeof(SYSTEM_RESOURCE_ACPI) + sizeof(SYSTEM_RESOURCE_FRAMEBUFFER));
 
-    /* Allocate memory for kernel initialization block */
+    /* Query Framebuffer size for allocation */
-    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, BlockPages, &Address);
+    if(XtLdrProtocol->Protocol.Open(&ProtocolHandle, (PVOID*)&FrameBufProtocol, &FrameBufGuid) == STATUS_EFI_SUCCESS)
+    {
+        /* Get FrameBuffer information */
+        FrameBufProtocol->GetDisplayInformation(&FbPhysicalAddress, &FbSize, &FbModeInfo);
+        XtLdrProtocol->Protocol.Close(&ProtocolHandle, &FrameBufGuid);
+    }
+    FbPages = EFI_SIZE_TO_PAGES(FbSize);
+
+    /* Precommit page map to allocate memory */
+    XtLdrProtocol->Memory.CommitPageMap(PageMap);
+
+    /* Calculate number of pages needed for memory descriptor list */
+    DescriptorPages = EFI_SIZE_TO_PAGES(PageMap->MapSize * sizeof(LOADER_MEMORY_DESCRIPTOR) * 2);
+
+    /* Allocate memory for the kernel initialization block and boot parameters */
+    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, BlockPages, &PhysicalBlock);
     if(Status != STATUS_EFI_SUCCESS)
     {
-        /* Memory allocation failure */
+        /* Memory allocation failure, return status code */
         return Status;
     }
 
-    /* Initialize and zero-fill kernel initialization block */
+    /* Allocate memory for the system resources data structures */
-    LoaderBlock = (PKERNEL_INITIALIZATION_BLOCK)(UINT_PTR)Address;
+    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, ResourcesPages, &PhysicalResources);
-    XtLdrProtocol->Memory.ZeroMemory(LoaderBlock, sizeof(KERNEL_INITIALIZATION_BLOCK) + ParametersSize);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Memory allocation failure, return status code */
+        return Status;
+    }
+
+    /* Allocate memory for the memory descriptor list */
+    Status = XtLdrProtocol->Memory.AllocatePages(AllocateAnyPages, DescriptorPages, &PhysicalDescriptor);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Memory allocation failure, return status code */
+        return Status;
+    }
+
+    /* Map the Kernel Initialization Block into virtual memory and advance the virtual address pointer */
+    VirtualBlock = *VirtualAddress;
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)VirtualBlock, PhysicalBlock, BlockPages, LoaderSystemBlock);
+    *VirtualAddress = (PUINT8)*VirtualAddress + (BlockPages * EFI_PAGE_SIZE);
+
+    /* Map the system resources physical memory into virtual address space and update the allocation pointer */
+    VirtualResources = *VirtualAddress;
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)VirtualResources, PhysicalResources, ResourcesPages, LoaderFirmwarePermanent);
+    *VirtualAddress = (PUINT8)*VirtualAddress + (ResourcesPages * EFI_PAGE_SIZE);
+
+    /* Check if a framebuffer was detected and requires memory mapping */
+    if(FbPages > 0)
+    {
+        /* Map the framebuffer physical memory range into virtual address space */
+        FbVirtualAddress = *VirtualAddress;
+        XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)FbVirtualAddress, FbPhysicalAddress, FbPages, LoaderFirmwarePermanent);
+        *VirtualAddress = (PUINT8)*VirtualAddress + (FbPages * EFI_PAGE_SIZE);
+    }
+
+    /* Map the allocated physical memory for memory descriptors into the virtual address space */
+    VirtualDescriptor = *VirtualAddress;
+    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)VirtualDescriptor, PhysicalDescriptor, DescriptorPages, LoaderMemoryData);
+    *VirtualAddress = (PUINT8)*VirtualAddress + (DescriptorPages * EFI_PAGE_SIZE);
 
     /* Set basic loader block properties */
+    XtLdrProtocol->Memory.ZeroMemory((PVOID)PhysicalBlock, sizeof(KERNEL_INITIALIZATION_BLOCK) + ParametersSize);
+    LoaderBlock = (PKERNEL_INITIALIZATION_BLOCK)PhysicalBlock;
     LoaderBlock->BlockSize = sizeof(KERNEL_INITIALIZATION_BLOCK);
     LoaderBlock->BlockVersion = INITIALIZATION_BLOCK_VERSION;
     LoaderBlock->ProtocolVersion = BOOT_PROTOCOL_VERSION;
@@ -467,24 +479,30 @@ Xtos::InitializeLoaderBlock(IN PXTBL_PAGE_MAPPING PageMap,
     LoaderBlock->FirmwareInformation.EfiFirmware.EfiRuntimeServices = NULLPTR;
 
     /* Copy parameters to kernel initialization block */
-    LoaderBlock->KernelParameters = (PWCHAR)((UINT_PTR)*VirtualAddress + sizeof(KERNEL_INITIALIZATION_BLOCK));
+    LoaderBlock->KernelParameters = (PWCHAR)((UINT_PTR)VirtualBlock + sizeof(KERNEL_INITIALIZATION_BLOCK));
     XtLdrProtocol->Memory.CopyMemory((PVOID)((UINT_PTR)LoaderBlock + sizeof(KERNEL_INITIALIZATION_BLOCK)),
-                  Parameters->Parameters,
+                                     Parameters->Parameters, ParametersSize);
-                  ParametersSize);
 
-    /* Map kernel initialization block */
+    /* Commit mappings */
-    XtLdrProtocol->Memory.MapVirtualMemory(PageMap, (ULONGLONG)*VirtualAddress, (ULONGLONG)LoaderBlock,
+    XtLdrProtocol->Memory.CommitPageMap(PageMap);
-                                           BlockPages, LoaderSystemBlock);
-
-    /* Calculate next valid virtual address */
-    *VirtualAddress = (PUINT8)*VirtualAddress + (BlockPages * EFI_PAGE_SIZE);
 
+    /* Initialize system resources list */
     XtLdrProtocol->LinkedList.InitializeHead(&LoaderBlock->SystemResourcesListHead);
-    GetSystemResourcesList(PageMap, VirtualAddress, &LoaderBlock->SystemResourcesListHead);
+    Status = GetSystemResourcesList(PageMap, PhysicalResources, VirtualResources, FbVirtualAddress, &LoaderBlock->SystemResourcesListHead);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to initialize system resources list, return status code */
+        return Status;
+    }
 
     /* Initialize memory descriptor list */
     XtLdrProtocol->LinkedList.InitializeHead(&LoaderBlock->MemoryDescriptorListHead);
-    GetMemoryDescriptorList(PageMap, VirtualAddress, &LoaderBlock->MemoryDescriptorListHead);
+    Status = GetMemoryDescriptorList(PageMap, PhysicalDescriptor, VirtualDescriptor, &LoaderBlock->MemoryDescriptorListHead);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        /* Failed to initialize memory descriptor list, return status code */
+        return Status;
+    }
 
     /* Set boot image size */
     LoaderBlock->BootImageSize = (PFN_NUMBER)(((ULONGLONG)*VirtualAddress - KSEG0_BASE) / EFI_PAGE_SIZE);
@@ -702,6 +720,14 @@ Xtos::RunBootSequence(IN PEFI_FILE_HANDLE BootDir,
         return Status;
     }
 
+    /* Build page map */
+    Status = BuildPageMap(&PageMap);
+    if(Status != STATUS_EFI_SUCCESS)
+    {
+        XtLdrProtocol->Debug.Print(L"Failed to build page map (Status code: %zX)\n", Status);
+        return Status;
+    }
+
     /* Store virtual address of kernel initialization block for future kernel call */
     KernelParameters = (PKERNEL_INITIALIZATION_BLOCK)VirtualAddress;
 

boot/xtldr/protocol.cc

@@ -1054,6 +1054,7 @@ Protocol::InstallXtLoaderProtocol()
     LoaderProtocol.Memory.AllocatePages = Memory::AllocatePages;
     LoaderProtocol.Memory.AllocatePool = Memory::AllocatePool;
     LoaderProtocol.Memory.BuildPageMap = Memory::BuildPageMap;
+    LoaderProtocol.Memory.CommitPageMap = Memory::CommitPageMap;
     LoaderProtocol.Memory.CompareMemory = RTL::Memory::CompareMemory;
     LoaderProtocol.Memory.CopyMemory = RTL::Memory::CopyMemory;
     LoaderProtocol.Memory.FreePages = Memory::FreePages;

sdk/xtdk/amd64/mmtypes.h

@@ -110,6 +110,15 @@
 /* Trampoline code address */
 #define MM_TRAMPOLINE_ADDRESS                      0x80000
 
+/* Pool block size */
+#define MM_POOL_BLOCK_SIZE                         16
+
+/* Number of pool lists per page */
+#define MM_POOL_LISTS_PER_PAGE                     (MM_PAGE_SIZE / MM_POOL_BLOCK_SIZE)
+
+/* Number of pool tracking tables */
+#define MM_POOL_TRACKING_TABLES                    64
+
 /* Page size enumeration list */
 typedef enum _PAGE_SIZE
 {
@@ -317,4 +326,22 @@ typedef struct _MMPFN
     } u4;
 } MMPFN, *PMMPFN;
 
+/* Pool descriptor structure definition */
+typedef struct _POOL_DESCRIPTOR
+{
+    LIST_ENTRY ListHeads[MM_POOL_LISTS_PER_PAGE];
+    PVOID LockAddress;
+    ULONG PoolIndex;
+    LONG PendingFreeDepth;
+    PVOID PendingFrees;
+    MMPOOL_TYPE PoolType;
+    ULONG RunningFrees;
+    ULONG RunningAllocations;
+    ULONG Threshold;
+    ULONG TotalPages;
+    ULONG TotalBigAllocations;
+    SIZE_T TotalBytes;
+    SIZE_T Reserved;
+} POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
+
 #endif /* __XTDK_AMD64_MMTYPES_H */

sdk/xtdk/amd64/xtstruct.h

@@ -61,6 +61,7 @@ typedef struct _MMPTE_PROTOTYPE MMPTE_PROTOTYPE, *PMMPTE_PROTOTYPE;
 typedef struct _MMPTE_SOFTWARE MMPTE_SOFTWARE, *PMMPTE_SOFTWARE;
 typedef struct _MMPTE_SUBSECTION MMPTE_SUBSECTION, *PMMPTE_SUBSECTION;
 typedef struct _MMPTE_TRANSITION MMPTE_TRANSITION, *PMMPTE_TRANSITION;
+typedef struct _POOL_DESCRIPTOR POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 typedef struct _THREAD_ENVIRONMENT_BLOCK THREAD_ENVIRONMENT_BLOCK, *PTHREAD_ENVIRONMENT_BLOCK;
 
 /* Unions forward references */

sdk/xtdk/bltypes.h

@@ -51,6 +51,7 @@ typedef EFI_STATUS (XTCDECL *PBL_BOOTMENU_INITIALIZE_OS_LIST)(IN ULONG MaxNameLe
 typedef BOOLEAN (XTCDECL *PBL_BOOTUTILS_GET_BOOLEAN_PARAMETER)(IN PCWSTR Parameters, IN PCWSTR Needle);
 typedef VOID (XTAPI *PBL_BOOTUTILS_GET_TRAMPOLINE_INFORMATION)(IN TRAMPOLINE_TYPE TrampolineType, OUT PVOID *TrampolineCode, OUT PULONG_PTR TrampolineSize);
 typedef EFI_STATUS (XTCDECL *PBL_BUILD_PAGE_MAP)(IN PXTBL_PAGE_MAPPING PageMap, IN ULONG_PTR SelfMapAddress);
+typedef EFI_STATUS (XTCDECL *PBL_COMMIT_PAGE_MAP)(IN PXTBL_PAGE_MAPPING PageMap);
 typedef EFI_STATUS (XTCDECL *PBL_CLOSE_VOLUME)(IN PEFI_HANDLE VolumeHandle);
 typedef VOID (XTCDECL *PBL_CLEAR_CONSOLE_LINE)(IN ULONGLONG LineNo);
 typedef BOOLEAN (XTCDECL *PBL_CPU_CPUID)(IN OUT PCPUID_REGISTERS Registers);
@@ -449,6 +450,7 @@ typedef struct _XTBL_LOADER_PROTOCOL
         PBL_ALLOCATE_PAGES AllocatePages;
         PBL_ALLOCATE_POOL AllocatePool;
         PBL_BUILD_PAGE_MAP BuildPageMap;
+        PBL_COMMIT_PAGE_MAP CommitPageMap;
         PBL_COMPARE_MEMORY CompareMemory;
         PBL_COPY_MEMORY CopyMemory;
         PBL_FREE_PAGES FreePages;

sdk/xtdk/i686/mmtypes.h

@@ -108,6 +108,15 @@
 /* Trampoline code address */
 #define MM_TRAMPOLINE_ADDRESS                      0x80000
 
+/* Pool block size */
+#define MM_POOL_BLOCK_SIZE                         8
+
+/* Number of pool lists per page */
+#define MM_POOL_LISTS_PER_PAGE                     (MM_PAGE_SIZE / MM_POOL_BLOCK_SIZE)
+
+/* Number of pool tracking tables */
+#define MM_POOL_TRACKING_TABLES                    32
+
 /* Page size enumeration list */
 typedef enum _PAGE_SIZE
 {
@@ -410,4 +419,22 @@ typedef struct _MMPFN
     } u4;
 } MMPFN, *PMMPFN;
 
+/* Pool descriptor structure definition */
+typedef struct _POOL_DESCRIPTOR
+{
+    LIST_ENTRY ListHeads[MM_POOL_LISTS_PER_PAGE];
+    PVOID LockAddress;
+    ULONG PoolIndex;
+    LONG PendingFreeDepth;
+    PVOID PendingFrees;
+    MMPOOL_TYPE PoolType;
+    ULONG RunningFrees;
+    ULONG RunningAllocations;
+    ULONG Threshold;
+    ULONG TotalPages;
+    ULONG TotalBigAllocations;
+    SIZE_T TotalBytes;
+    SIZE_T Reserved;
+} POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
+
 #endif /* __XTDK_I686_MMTYPES_H */

sdk/xtdk/i686/xtstruct.h

@@ -70,6 +70,7 @@ typedef struct _MMPML3_PTE_PROTOTYPE MMPML3_PTE_PROTOTYPE, *PMMPML3_PTE_PROTOTYP
 typedef struct _MMPML3_PTE_SOFTWARE MMPML3_PTE_SOFTWARE, *PMMPML3_PTE_SOFTWARE;
 typedef struct _MMPML3_PTE_SUBSECTION MMPML3_PTE_SUBSECTION, *PMMPML3_PTE_SUBSECTION;
 typedef struct _MMPML3_PTE_TRANSITION MMPML3_PTE_TRANSITION, *PMMPML3_PTE_TRANSITION;
+typedef struct _POOL_DESCRIPTOR POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 typedef struct _THREAD_ENVIRONMENT_BLOCK THREAD_ENVIRONMENT_BLOCK, *PTHREAD_ENVIRONMENT_BLOCK;
 
 /* Unions forward references */

sdk/xtdk/mmfuncs.h

@@ -0,0 +1,40 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            sdk/xtdk/mmfuncs.h
+ * DESCRIPTION:     XTOS memory manager routine definitions
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTDK_MMFUNCS_H
+#define __XTDK_MMFUNCS_H
+
+#include <xtdefs.h>
+#include <xtstruct.h>
+#include <xttypes.h>
+
+
+/* Memory manager routines forward references */
+XTAPI
+XTSTATUS
+MmAllocatePool(IN MMPOOL_TYPE PoolType,
+               IN SIZE_T Bytes,
+               OUT PVOID *Memory);
+
+XTAPI
+XTSTATUS
+MmAllocatePoolWithTag(IN MMPOOL_TYPE PoolType,
+                      IN SIZE_T Bytes,
+                      OUT PVOID *Memory,
+                      IN ULONG Tag);
+
+XTAPI
+XTSTATUS
+MmFreePool(IN PVOID VirtualAddress);
+
+XTAPI
+XTSTATUS
+MmFreePoolWithTag(IN PVOID VirtualAddress,
+                  IN ULONG Tag);
+
+#endif /* __XTDK_MMFUNCS_H */

sdk/xtdk/mmtypes.h

@@ -29,9 +29,32 @@
 /* Memory manager pool type mask definition */
 #define MM_POOL_TYPE_MASK                          1
 
+/* Bad pool caller reasons */
+#define MM_POOL_INVALID_ALLOC_RUNLEVEL             8
+#define MM_POOL_INVALID_FREE_RUNLEVEL              9
+
+/* Pool flags */
+#define MM_POOL_BIG_ALLOCATIONS_ENTRY_FREE         0x1
+#define MM_POOL_PROTECTED                          0x80000000
+#define MM_POOL_RAISE_EXCEPTION                    0x10
+
 /* Number of reserved zeroed PTEs */
 #define MM_RESERVED_ZERO_PTES                      32
 
+/* Memory Manager Protection Bits */
+#define MM_ZERO_ACCESS                             0
+#define MM_READONLY                                1
+#define MM_EXECUTE                                 2
+#define MM_EXECUTE_READ                            3
+#define MM_READWRITE                               4
+#define MM_WRITECOPY                               5
+#define MM_EXECUTE_READWRITE                       6
+#define MM_EXECUTE_WRITECOPY                       7
+#define MM_PROTECT_ACCESS                          7
+
+/* Protection field shift */
+#define MM_PROTECT_FIELD_SHIFT                     5
+
 /* Memory manager page lists */
 typedef enum _MMPAGELISTS
 {
@@ -62,14 +85,14 @@ typedef enum _MMPOOL_TYPE
     NonPagedPoolMustSucceed = 2,
     NonPagedPoolCacheAligned = 4,
     PagedPoolCacheAligned = 5,
-    NonPagedPoolCacheAlignedMustS = 6,
+    NonPagedPoolCacheAlignedMustSucceed = 6,
     MaxPoolType = 7,
     NonPagedPoolSession = 32,
     PagedPoolSession = 33,
     NonPagedPoolMustSucceedSession = 34,
     NonPagedPoolCacheAlignedSession = 36,
     PagedPoolCacheAlignedSession = 37,
-    NonPagedPoolCacheAlignedMustSSession = 38
+    NonPagedPoolCacheAlignedMustSucceedSession = 38
 } MMPOOL_TYPE, *PMMPOOL_TYPE;
 
 /* Page table pool types */
@@ -173,4 +196,66 @@ typedef struct _MMPFNLIST
     PFN_NUMBER Blink;
 } MMPFNLIST, *PMMPFNLIST;
 
+/* Physical memory run structure definition */
+typedef struct _PHYSICAL_MEMORY_RUN
+{
+    PFN_NUMBER BasePage;
+    PFN_NUMBER PageCount;
+} PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
+
+/* Physical memory descriptor structure definition */
+typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
+{
+    ULONG NumberOfRuns;
+    PFN_NUMBER NumberOfPages;
+    PHYSICAL_MEMORY_RUN Run[1];
+} PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
+
+/* Pool header structure definition */
+typedef struct _POOL_HEADER
+{
+    union
+    {
+        struct
+        {
+            USHORT PreviousSize:9;
+            USHORT PoolIndex:7;
+            USHORT BlockSize:9;
+            USHORT PoolType:7;
+        };
+        ULONG Long;
+    };
+    union
+    {
+        ULONG PoolTag;
+        PEPROCESS ProcessBilled;
+        struct
+        {
+            USHORT AllocatorBackTraceIndex;
+            USHORT PoolTagHash;
+        };
+    };
+} POOL_HEADER, *PPOOL_HEADER;
+
+/* Pool descriptor structure definition */
+typedef struct _POOL_TRACKING_BIG_ALLOCATIONS
+{
+    ULONG NumberOfPages;
+    PVOID QuotaObject;
+    ULONG Tag;
+    PVOID VirtualAddress;
+} POOL_TRACKING_BIG_ALLOCATIONS, *PPOOL_TRACKING_BIG_ALLOCATIONS;
+
+/* Pool tracking table structure definition */
+typedef struct _POOL_TRACKING_TABLE
+{
+    LONG NonPagedAllocations;
+    SIZE_T NonPagedBytes;
+    LONG NonPagedFrees;
+    LONG PagedAllocations;
+    SIZE_T PagedBytes;
+    LONG PagedFrees;
+    ULONG Tag;
+} POOL_TRACKING_TABLE, *PPOOL_TRACKING_TABLE;
+
 #endif /* __XTDK_MMTYPES_H */

sdk/xtdk/xtdebug.h

@@ -23,7 +23,7 @@
     #define DebugPrint(Format, ...)     DbgPrint(Format, __VA_ARGS__);
 #else
     #define DEBUG                       0
-    #define DebugPrint(Format, ...)     ((VOID)NULL)
+    #define DebugPrint(Format, ...)     ((VOID)NULLPTR)
 #endif
 
 #endif /* __XTDK_XTDEBUG_H */

sdk/xtdk/xtkmapi.h

@@ -50,6 +50,7 @@
 #include <hlfuncs.h>
 #include <kdfuncs.h>
 #include <kefuncs.h>
+#include <mmfuncs.h>
 #include <rtlfuncs.h>
 
 /* Architecture specific XT routines */

sdk/xtdk/xtstruct.h

@@ -310,6 +310,11 @@ typedef struct _PECOFF_IMAGE_ROM_HEADER PECOFF_IMAGE_ROM_HEADER, *PPECOFF_IMAGE_
 typedef struct _PECOFF_IMAGE_ROM_OPTIONAL_HEADER PECOFF_IMAGE_ROM_OPTIONAL_HEADER, *PPECOFF_IMAGE_ROM_OPTIONAL_HEADER;
 typedef struct _PECOFF_IMAGE_SECTION_HEADER PECOFF_IMAGE_SECTION_HEADER, *PPECOFF_IMAGE_SECTION_HEADER;
 typedef struct _PECOFF_IMAGE_VXD_HEADER PECOFF_IMAGE_VXD_HEADER, *PPECOFF_IMAGE_VXD_HEADER;
+typedef struct _PHYSICAL_MEMORY_DESCRIPTOR PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
+typedef struct _PHYSICAL_MEMORY_RUN PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
+typedef struct _POOL_HEADER POOL_HEADER, *PPOOL_HEADER;
+typedef struct _POOL_TRACKING_BIG_ALLOCATIONS POOL_TRACKING_BIG_ALLOCATIONS, *PPOOL_TRACKING_BIG_ALLOCATIONS;
+typedef struct _POOL_TRACKING_TABLE POOL_TRACKING_TABLE, *PPOOL_TRACKING_TABLE;
 typedef struct _PROCESSOR_IDENTITY PROCESSOR_IDENTITY, *PPROCESSOR_IDENTITY;
 typedef struct _PROCESSOR_POWER_STATE PROCESSOR_POWER_STATE, *PPROCESSOR_POWER_STATE;
 typedef struct _RTL_BITMAP RTL_BITMAP, *PRTL_BITMAP;

xtoskrnl/CMakeLists.txt

@@ -51,21 +51,23 @@ list(APPEND XTOSKRNL_SOURCE
     ${XTOSKRNL_SOURCE_DIR}/ke/spinlock.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/sysres.cc
     ${XTOSKRNL_SOURCE_DIR}/ke/timer.cc
-    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/alloc.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/mmgr.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pagemap.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/paging.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfault.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pfn.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pool.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/${ARCH}/pte.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/alloc.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/colors.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/data.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/exports.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/hlpool.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/kpool.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/mmgr.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/paging.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/pfn.cc
+    ${XTOSKRNL_SOURCE_DIR}/mm/pool.cc
     ${XTOSKRNL_SOURCE_DIR}/mm/pte.cc
     ${XTOSKRNL_SOURCE_DIR}/po/idle.cc
     ${XTOSKRNL_SOURCE_DIR}/rtl/${ARCH}/dispatch.cc

xtoskrnl/ar/amd64/archsup.S

@@ -104,12 +104,20 @@ ArTrap\Vector:
     /* Test previous mode and swap GS if needed */
     movl $0, TrapPreviousMode(%rbp)
     mov %cs, %ax
-    and $1, %al
+    and $3, %al
     mov %al, TrapPreviousMode(%rbp)
     jz KernelMode$\Vector
     swapgs
+    jmp UserMode$\Vector
 
 KernelMode$\Vector:
+    /* Save kernel stack pointer (SS:RSP) */
+    movl %ss, %eax
+    mov %eax, TrapSegSs(%rbp)
+    lea TRAP_FRAME_SIZE(%rbp), %rax
+    mov %rax, TrapRsp(%rbp)
+
+UserMode$\Vector:
     /* Push Frame Pointer, clear direction flag and pass to trap dispatcher */
     mov %rsp, %rcx
     cld

xtoskrnl/ar/i686/archsup.S

@@ -75,12 +75,20 @@ _ArTrap\Vector:
     /* Test previous mode and swap GS if needed */
     movl $0, TrapPreviousMode(%ebp)
     mov %cs, %ax
-    and $1, %al
+    and $3, %al
     mov %al, TrapPreviousMode(%ebp)
     jz KernelMode$\Vector
     swapgs
+    jmp UserMode$\Vector
 
 KernelMode$\Vector:
+    /* Save kernel stack pointer (SS:ESP) as CPU did not push them */
+    movl %ss, %eax
+    mov %eax, TrapSegSs(%ebp)
+    lea TrapEsp(%ebp), %eax
+    mov %eax, TrapEsp(%ebp)
+
+UserMode$\Vector:
     /* Push Frame Pointer, clear direction flag and pass to trap dispatcher */
     push %esp
     cld
@@ -100,6 +108,7 @@ KernelModeReturn$\Vector:
     mov TrapSegDs(%ebp), %ds
     mov TrapSegEs(%ebp), %es
     mov TrapSegFs(%ebp), %fs
+    mov TrapSegGs(%ebp), %gs
 
     /* Free stack space */
     add $(TRAP_FRAME_SIZE - TRAP_REGISTERS_SIZE), %esp

xtoskrnl/hl/x86/pic.cc

@@ -9,6 +9,41 @@
 #include <xtos.hh>
 
 
+/**
+ * Checks whether the APIC is supported by the processor.
+ *
+ * @return This routine returns TRUE if APIC is supported, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+HL::Pic::CheckApicSupport(VOID)
+{
+    CPUID_REGISTERS CpuRegisters;
+
+    /* Prepare CPUID registers */
+    CpuRegisters.Leaf = CPUID_GET_STANDARD1_FEATURES;
+    CpuRegisters.SubLeaf = 0;
+    CpuRegisters.Eax = 0;
+    CpuRegisters.Ebx = 0;
+    CpuRegisters.Ecx = 0;
+    CpuRegisters.Edx = 0;
+
+    /* Get CPUID */
+    AR::CpuFunc::CpuId(&CpuRegisters);
+
+    /* Check APIC status from the CPUID results */
+    if(!(CpuRegisters.Edx & CPUID_FEATURES_EDX_APIC))
+    {
+        /* APIC is not supported */
+        return FALSE;
+    }
+
+    /* APIC is supported */
+    return TRUE;
+}
+
 /**
  * Checks whether the x2APIC extension is supported by the processor.
  *
@@ -124,6 +159,14 @@ HL::Pic::InitializeApic(VOID)
     APIC_LVT_REGISTER LvtRegister;
     ULONG CpuNumber;
 
+    /* Check APIC support */
+    if(!CheckApicSupport())
+    {
+        /* APIC is not supported, raise kernel panic */
+        DebugPrint(L"FATAL ERROR: Local APIC not present.\n");
+        KE::Crash::Panic(0x5D, CPUID_GET_STANDARD1_FEATURES, 0x0, 0x0, CPUID_FEATURES_EDX_APIC);
+    }
+
     /* Determine APIC mode (xAPIC compatibility or x2APIC) */
     if(CheckX2ApicSupport())
     {

xtoskrnl/includes/ar/amd64/asmsup.h

@@ -56,6 +56,8 @@
 #define TrapSegEs                               330
 #define TrapSegFs                               332
 #define TrapSegGs                               334
+#define TrapRsp                                 496
+#define TrapSegSs                               504
 
 /* KTRAP_FRAME length related definitions */
 #define TRAP_FRAME_SIZE                         512

xtoskrnl/includes/ar/amd64/procsup.hh

@@ -31,6 +31,13 @@ namespace AR
                                                        OUT PVOID *TrampolineCode,
                                                        OUT PULONG_PTR TrampolineSize);
             STATIC XTAPI VOID InitializeProcessor(IN PVOID ProcessorStructures);
+            STATIC XTAPI VOID SetIdtGate(IN PKIDTENTRY Idt,
+                                         IN USHORT Vector,
+                                         IN PVOID Handler,
+                                         IN USHORT Selector,
+                                         IN USHORT Ist,
+                                         IN USHORT Dpl,
+                                         IN USHORT Type);
 
         private:
             STATIC XTAPI VOID IdentifyProcessor(VOID);
@@ -62,13 +69,6 @@ namespace AR
             STATIC XTAPI VOID SetGdtEntryBase(IN PKGDTENTRY Gdt,
                                               IN USHORT Selector,
                                               IN ULONG_PTR Base);
-            STATIC XTAPI VOID SetIdtGate(IN PKIDTENTRY Idt,
-                                         IN USHORT Vector,
-                                         IN PVOID Handler,
-                                         IN USHORT Selector,
-                                         IN USHORT Ist,
-                                         IN USHORT Dpl,
-                                         IN USHORT Type);
     };
 }
 

xtoskrnl/includes/ar/i686/asmsup.h

@@ -24,9 +24,11 @@
 #define TrapSegEs                               38
 #define TrapSegFs                               40
 #define TrapSegGs                               42
+#define TrapEsp                                 92
+#define TrapSegSs                               96
 
 /* KTRAP_FRAME length related definitions */
 #define TRAP_FRAME_SIZE                         100
 #define TRAP_REGISTERS_SIZE                     56
 
-#endif /* __XTOSKRNL_AMD64_ASMSUP_H */
+#endif /* __XTOSKRNL_I686_ASMSUP_H */

xtoskrnl/includes/ar/i686/procsup.hh

@@ -34,6 +34,13 @@ namespace AR
                                                        OUT PVOID *TrampolineCode,
                                                        OUT PULONG_PTR TrampolineSize);
             STATIC XTAPI VOID InitializeProcessor(IN PVOID ProcessorStructures);
+            STATIC XTAPI VOID SetIdtGate(IN PKIDTENTRY Idt,
+                                         IN USHORT Vector,
+                                         IN PVOID Handler,
+                                         IN USHORT Selector,
+                                         IN USHORT Ist,
+                                         IN USHORT Dpl,
+                                         IN USHORT Type);
 
         private:
             STATIC XTAPI VOID IdentifyProcessor(VOID);
@@ -67,13 +74,6 @@ namespace AR
             STATIC XTAPI VOID SetGdtEntryBase(IN PKGDTENTRY Gdt,
                                               IN USHORT Selector,
                                               IN ULONG_PTR Base);
-            STATIC XTAPI VOID SetIdtGate(IN PKIDTENTRY Idt,
-                                         IN USHORT Vector,
-                                         IN PVOID Handler,
-                                         IN USHORT Selector,
-                                         IN USHORT Ist,
-                                         IN USHORT Dpl,
-                                         IN USHORT Type);
             STATIC XTAPI VOID SetNonMaskableInterruptTssEntry(IN PKPROCESSOR_BLOCK ProcessorBlock,
                                                               IN PVOID KernelFaultStack);
 

xtoskrnl/includes/hl/pic.hh

@@ -32,6 +32,7 @@ namespace HL
                                                      IN ULONGLONG Value);
 
         private:
+            STATIC XTAPI BOOLEAN CheckApicSupport(VOID);
             STATIC XTAPI BOOLEAN CheckX2ApicSupport(VOID);
             STATIC XTCDECL VOID HandleApicSpuriousService(VOID);
             STATIC XTCDECL VOID HandlePicSpuriousService(VOID);

xtoskrnl/includes/ke.hh

@@ -16,6 +16,7 @@
 #include <ke/crash.hh>
 #include <ke/dpc.hh>
 #include <ke/event.hh>
+#include <ke/guard.hh>
 #include <ke/irq.hh>
 #include <ke/kprocess.hh>
 #include <ke/krnlinit.hh>

xtoskrnl/includes/ke/crash.hh

@@ -20,7 +20,7 @@ namespace KE
         public:
             STATIC XTAPI VOID HaltSystem(VOID);
             STATIC XTAPI VOID Panic(IN ULONG Code);
-            STATIC XTAPI VOID PanicEx(IN ULONG Code,
+            STATIC XTAPI VOID Panic(IN ULONG Code,
                                     IN ULONG_PTR Parameter1,
                                     IN ULONG_PTR Parameter2,
                                     IN ULONG_PTR Parameter3,

xtoskrnl/includes/ke/guard.hh

@@ -0,0 +1,61 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/ke/guard.hh
+ * DESCRIPTION:     Kernel synchronization guard
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_KE_GUARD_HH
+#define __XTOSKRNL_KE_GUARD_HH
+
+#include <xtos.hh>
+#include <ke/spinlock.hh>
+
+/* Kernel Library */
+namespace KE
+{
+    class QueuedSpinLockGuard
+    {
+        private:
+            KSPIN_LOCK_QUEUE_LEVEL QueuedLockLevel;
+
+        public:
+            QueuedSpinLockGuard(IN OUT KSPIN_LOCK_QUEUE_LEVEL LockLevel)
+            {
+                QueuedLockLevel = LockLevel;
+                KE::SpinLock::AcquireQueuedSpinLock(QueuedLockLevel);
+            }
+
+            ~QueuedSpinLockGuard()
+            {
+                KE::SpinLock::ReleaseQueuedSpinLock(QueuedLockLevel);
+            }
+
+            QueuedSpinLockGuard(const QueuedSpinLockGuard&) = delete;
+            QueuedSpinLockGuard& operator=(const QueuedSpinLockGuard&) = delete;
+    };
+
+    class SpinLockGuard
+    {
+        private:
+            PKSPIN_LOCK Lock;
+
+        public:
+            SpinLockGuard(IN OUT PKSPIN_LOCK SpinLock)
+            {
+                Lock = SpinLock;
+                KE::SpinLock::AcquireSpinLock(Lock);
+            }
+
+            ~SpinLockGuard()
+            {
+                KE::SpinLock::ReleaseSpinLock(Lock);
+            }
+
+            SpinLockGuard(const SpinLockGuard&) = delete;
+            SpinLockGuard& operator=(const SpinLockGuard&) = delete;
+    };
+}
+
+#endif /* __XTOSKRNL_KE_GUARD_HH */

xtoskrnl/includes/ke/spinlock.hh

@@ -44,47 +44,6 @@ namespace KE
             STATIC XTFASTCALL VOID ReleaseSpinLock(IN OUT PKSPIN_LOCK SpinLock);
             STATIC XTFASTCALL BOOLEAN TestSpinLock(IN PKSPIN_LOCK SpinLock);
     };
-
-    class QueuedSpinLockGuard
-    {
-        private:
-            KSPIN_LOCK_QUEUE_LEVEL QueuedLockLevel;
-
-        public:
-            QueuedSpinLockGuard(IN OUT KSPIN_LOCK_QUEUE_LEVEL LockLevel)
-            {
-                QueuedLockLevel = LockLevel;
-                KE::SpinLock::AcquireQueuedSpinLock(QueuedLockLevel);
-            }
-
-            ~QueuedSpinLockGuard()
-            {
-                KE::SpinLock::ReleaseQueuedSpinLock(QueuedLockLevel);
-            }
-
-            QueuedSpinLockGuard(const QueuedSpinLockGuard&) = delete;
-            QueuedSpinLockGuard& operator=(const QueuedSpinLockGuard&) = delete;
-    };
-
-    class SpinLockGuard
-    {
-        private:
-            PKSPIN_LOCK SpinLock;
-
-        public:
-            SpinLockGuard(IN OUT PKSPIN_LOCK SpinLock)
-            {
-                KE::SpinLock::AcquireSpinLock(SpinLock);
-            }
-
-            ~SpinLockGuard()
-            {
-                KE::SpinLock::ReleaseSpinLock(SpinLock);
-            }
-
-            SpinLockGuard(const SpinLockGuard&) = delete;
-            SpinLockGuard& operator=(const SpinLockGuard&) = delete;
-    };
 }
 
 #endif /* __XTOSKRNL_KE_SPINLOCK_HH */

xtoskrnl/includes/mm.hh

@@ -17,10 +17,12 @@
 
 #include <mm/alloc.hh>
 #include <mm/colors.hh>
+#include <mm/guard.hh>
 #include <mm/hlpool.hh>
 #include <mm/kpool.hh>
 #include <mm/mmgr.hh>
 #include <mm/pfault.hh>
 #include <mm/pfn.hh>
+#include <mm/pool.hh>
 
 #endif /* __XTOSKRNL_MM_HH */

xtoskrnl/includes/mm/alloc.hh

@@ -2,7 +2,7 @@
  * PROJECT:         ExectOS
  * COPYRIGHT:       See COPYING.md in the top level directory
  * FILE:            xtoskrnl/includes/mm/alloc.hh
- * DESCRIPTION:     Memory manager pool allocation
+ * DESCRIPTION:     Memory Manager pool allocator
  * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
  */
 
@@ -10,34 +10,80 @@
 #define __XTOSKRNL_MM_ALLOC_HH
 
 #include <xtos.hh>
+#include <mm/pool.hh>
 
 
 /* Memory Manager */
 namespace MM
 {
-    class Allocator
+    class Allocator final : private Pool
     {
         private:
-            STATIC PFN_NUMBER NonPagedPoolFrameEnd;
+            STATIC PPOOL_TRACKING_TABLE AllocationsTrackingExpansionTable;
-            STATIC PFN_NUMBER NonPagedPoolFrameStart;
+            STATIC SIZE_T AllocationsTrackingExpansionTableSize;
-            STATIC LIST_ENTRY NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS];
+            STATIC PPOOL_TRACKING_TABLE AllocationsTrackingTable;
+            STATIC KSPIN_LOCK AllocationsTrackingTableLock;
+            STATIC SIZE_T AllocationsTrackingTableMask;
+            STATIC SIZE_T AllocationsTrackingTableSize;
+            STATIC ULONG BigAllocationsInUse;
+            STATIC PPOOL_TRACKING_BIG_ALLOCATIONS BigAllocationsTrackingTable;
+            STATIC SIZE_T BigAllocationsTrackingTableHash;
+            STATIC KSPIN_LOCK BigAllocationsTrackingTableLock;
+            STATIC SIZE_T BigAllocationsTrackingTableSize;
+            STATIC PPOOL_TRACKING_TABLE TagTables[MM_POOL_TRACKING_TABLES];
 
         public:
             STATIC XTAPI XTSTATUS AllocatePages(IN MMPOOL_TYPE PoolType,
                                                 IN SIZE_T Bytes,
                                                 OUT PVOID *Memory);
+            STATIC XTAPI XTSTATUS AllocatePool(IN MMPOOL_TYPE PoolType,
+                                               IN SIZE_T Bytes,
+                                               OUT PVOID *Memory);
+            STATIC XTAPI XTSTATUS AllocatePool(IN MMPOOL_TYPE PoolType,
+                                               IN SIZE_T Bytes,
+                                               OUT PVOID *Memory,
+                                               IN ULONG Tag);
             STATIC XTAPI XTSTATUS FreePages(IN PVOID VirtualAddress);
-            STATIC XTAPI VOID InitializeNonPagedPool(VOID);
+            STATIC XTAPI XTSTATUS FreePages(IN PVOID VirtualAddress,
-            STATIC XTAPI VOID InitializePagedPool(VOID);
+                                            OUT PPFN_NUMBER PagesFreed);
+            STATIC XTAPI XTSTATUS FreePool(IN PVOID VirtualAddress);
+            STATIC XTAPI XTSTATUS FreePool(IN PVOID VirtualAddress,
+                                           IN ULONG Tag);
+            STATIC XTAPI VOID InitializeAllocationsTracking(VOID);
+            STATIC XTAPI VOID InitializeBigAllocationsTracking(VOID);
 
         private:
             STATIC XTAPI XTSTATUS AllocateNonPagedPoolPages(IN PFN_COUNT Pages,
                                                             OUT PVOID *Memory);
             STATIC XTAPI XTSTATUS AllocatePagedPoolPages(IN PFN_COUNT Pages,
                                                          OUT PVOID *Memory);
-            STATIC XTAPI XTSTATUS FreeNonPagedPoolPages(IN PVOID VirtualAddress);
+            STATIC XTINLINE ULONG ComputeHash(IN PVOID VirtualAddress);
-            STATIC XTAPI XTSTATUS FreePagedPoolPages(IN PVOID VirtualAddress);
+            STATIC XTINLINE ULONG ComputeHash(IN ULONG Tag,
-            STATIC XTAPI VOID MapNonPagedPool(VOID);
+                                              IN ULONG TableMask);
+            STATIC XTAPI BOOLEAN ExpandBigAllocationsTable(VOID);
+            STATIC XTAPI XTSTATUS FreeNonPagedPoolPages(IN PVOID VirtualAddress,
+                                                        OUT PPFN_NUMBER PagesFreed);
+            STATIC XTAPI XTSTATUS FreePagedPoolPages(IN PVOID VirtualAddress,
+                                                     OUT PPFN_NUMBER PagesFreed);
+            STATIC XTAPI VOID RegisterAllocationTag(IN ULONG Tag,
+                                                    IN SIZE_T Bytes,
+                                                    IN MMPOOL_TYPE PoolType);
+            STATIC XTAPI VOID RegisterAllocationTagExpansion(IN ULONG Tag,
+                                                             IN SIZE_T Bytes,
+                                                             IN MMPOOL_TYPE PoolType);
+            STATIC XTAPI BOOLEAN RegisterBigAllocationTag(IN PVOID VirtualAddress,
+                                                          IN ULONG Tag,
+                                                          IN ULONG Pages,
+                                                          IN MMPOOL_TYPE PoolType);
+            STATIC XTAPI VOID UnregisterAllocationTag(IN ULONG Tag,
+                                                      IN SIZE_T Bytes,
+                                                      IN MMPOOL_TYPE PoolType);
+            STATIC XTAPI VOID UnregisterAllocationTagExpansion(IN ULONG Tag,
+                                                               IN SIZE_T Bytes,
+                                                               IN MMPOOL_TYPE PoolType);
+            STATIC XTAPI ULONG UnregisterBigAllocationTag(IN PVOID VirtualAddress,
+                                                          OUT PULONG_PTR Pages,
+                                                          IN MMPOOL_TYPE PoolType);
     };
 }
 

xtoskrnl/includes/mm/amd64/pagemap.hh

@@ -39,7 +39,7 @@ namespace MM
             XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
             XTAPI ULONG GetPpeOffset(IN PVOID Address);
             VIRTUAL XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer) = 0;
-            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer);
             XTAPI PMMPTE GetPteAddress(IN PVOID Address);
             XTAPI LONG GetPteDistance(PMMPTE EndPte,
                                       PMMPTE StartPte);
@@ -61,9 +61,9 @@ namespace MM
                                    IN BOOLEAN Value);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
                               IN PFN_NUMBER PageFrameNumber,
-                              IN ULONG_PTR AttributesMask);
+                              IN ULONGLONG AttributesMask);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                              IN ULONG_PTR Attributes);
+                              IN ULONGLONG Attributes);
             XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                      IN BOOLEAN CacheDisable,
                                      IN BOOLEAN WriteThrough);

xtoskrnl/includes/mm/amd64/paging.hh

@@ -25,6 +25,7 @@ namespace MM
                                            IN LONG Count);
             STATIC XTAPI BOOLEAN CanonicalAddress(IN PVOID VirtualAddress);
             STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID FlushEntireTlb(VOID);
             STATIC XTAPI VOID FlushTlb(VOID);
             STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
             STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
@@ -37,7 +38,7 @@ namespace MM
             STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
             STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
             STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
-            STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer);
             STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address);
             STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte,
                                              PMMPTE StartPte);
@@ -57,9 +58,9 @@ namespace MM
                                           IN BOOLEAN Value);
             STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
                                      IN PFN_NUMBER PageFrameNumber,
-                                     IN ULONG_PTR AttributesMask);
+                                     IN ULONGLONG AttributesMask);
             STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                                     IN ULONG_PTR Attributes);
+                                     IN ULONGLONG Attributes);
             STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                             IN BOOLEAN CacheDisable,
                                             IN BOOLEAN WriteThrough);

xtoskrnl/includes/mm/guard.hh

@@ -0,0 +1,88 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/guard.hh
+ * DESCRIPTION:     Memory Manager synchronization guard
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_GUARD_HH
+#define __XTOSKRNL_MM_GUARD_HH
+
+#include <xtos.hh>
+#include <ke/runlevel.hh>
+#include <ke/spinlock.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class PoolLockGuard
+    {
+        private:
+            BOOLEAN Locked;
+            MMPOOL_TYPE LockPoolType;
+            KRUNLEVEL PreviousRunLevel;
+
+        public:
+            PoolLockGuard(IN MMPOOL_TYPE PoolType)
+            {
+                LockPoolType = PoolType;
+
+                /* Determine the appropriate synchronization mechanism based on the requested pool type */
+                if(LockPoolType == NonPagedPool)
+                {
+                    /* Elevate the runlevel to DISPATCH_LEVEL */
+                    PreviousRunLevel = KE::RunLevel::RaiseRunLevel(DISPATCH_LEVEL);
+
+                    /* Acquire the global queued spinlock protecting the non-paged pool */
+                    KE::SpinLock::AcquireQueuedSpinLock(NonPagedPoolLock);
+                }
+                else
+                {
+                    /* Paged pool requires a mutex, currently unimplemented */
+                    UNIMPLEMENTED;
+                }
+
+                /* Mark the guard as actively holding the lock */
+                Locked = TRUE;
+            }
+
+            ~PoolLockGuard(VOID)
+            {
+                /* Automatically release the held lock upon going out of scope */
+                Release();
+            }
+
+            PoolLockGuard(const PoolLockGuard&) = delete;
+            PoolLockGuard& operator=(const PoolLockGuard&) = delete;
+
+            VOID Release(VOID)
+            {
+                /* Check if the guard is currently holding a lock */
+                if(!Locked)
+                {
+                    /* Return, to prevent a double-free */
+                    return;
+                }
+
+                /* Determine the appropriate synchronization mechanism based on the requested pool type */
+                if(LockPoolType == NonPagedPool)
+                {
+                    /* Release the non-paged pool spinlock and subsequently restore the original runlevel */
+                    KE::SpinLock::ReleaseQueuedSpinLock(NonPagedPoolLock);
+                    KE::RunLevel::LowerRunLevel(PreviousRunLevel);
+                }
+                else
+                {
+                    /* Paged pool requires a mutex, currently unimplemented */
+                    UNIMPLEMENTED;
+                }
+
+                /* Update the internal state, indicating that the lock is no longer held */
+                Locked = FALSE;
+            }
+    };
+}
+
+#endif /* __XTOSKRNL_MM_GUARD_HH */

xtoskrnl/includes/mm/i686/pagemap.hh

@@ -36,7 +36,7 @@ namespace MM
             XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
             XTAPI ULONG GetPpeOffset(IN PVOID Address);
             XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
-            VIRTUAL XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer) = 0;
+            VIRTUAL XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer) = 0;
             XTAPI PMMPTE GetPteAddress(IN PVOID Address);
             XTAPI ULONG GetPteOffset(IN PVOID Address);
             VIRTUAL XTAPI LONG GetPteDistance(PMMPTE EndPte,
@@ -55,9 +55,9 @@ namespace MM
                                            IN BOOLEAN Value) = 0;
             VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer,
                                       IN PFN_NUMBER PageFrameNumber,
-                                      IN ULONG_PTR AttributesMask) = 0;
+                                      IN ULONGLONG AttributesMask) = 0;
             VIRTUAL XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                                      IN ULONG_PTR Attributes) = 0;
+                                      IN ULONGLONG Attributes) = 0;
             VIRTUAL XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                              IN BOOLEAN CacheDisable,
                                              IN BOOLEAN WriteThrough) = 0;
@@ -79,7 +79,7 @@ namespace MM
             XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
             XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
             XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
-            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer);
             XTAPI LONG GetPteDistance(PMMPTE EndPte,
                                       PMMPTE StartPte);
             XTAPI ULONG GetPteSize(VOID);
@@ -95,9 +95,9 @@ namespace MM
                                    IN BOOLEAN Value);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
                               IN PFN_NUMBER PageFrameNumber,
-                              IN ULONG_PTR AttributesMask);
+                              IN ULONGLONG AttributesMask);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                              IN ULONG_PTR Attributes);
+                              IN ULONGLONG Attributes);
             XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                      IN BOOLEAN CacheDisable,
                                      IN BOOLEAN WriteThrough);
@@ -118,7 +118,7 @@ namespace MM
             XTAPI BOOLEAN GetOneEntry(IN PMMPTE Pte);
             XTAPI PFN_NUMBER GetPageFrameNumber(IN PMMPTE Pte);
             XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
-            XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer);
             XTAPI LONG GetPteDistance(PMMPTE EndPte,
                                       PMMPTE StartPte);
             XTAPI ULONG GetPteSize(VOID);
@@ -134,9 +134,9 @@ namespace MM
                                    IN BOOLEAN Value);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
                               IN PFN_NUMBER PageFrameNumber,
-                              IN ULONG_PTR AttributesMask);
+                              IN ULONGLONG AttributesMask);
             XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                              IN ULONG_PTR Attributes);
+                              IN ULONGLONG Attributes);
             XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                      IN BOOLEAN CacheDisable,
                                      IN BOOLEAN WriteThrough);

xtoskrnl/includes/mm/i686/paging.hh

@@ -25,6 +25,7 @@ namespace MM
                                            IN LONG Count);
             STATIC XTAPI BOOLEAN CanonicalAddress(IN PVOID VirtualAddress);
             STATIC XTAPI VOID ClearPte(IN PMMPTE PtePointer);
+            STATIC XTAPI VOID FlushEntireTlb(VOID);
             STATIC XTAPI VOID FlushTlb(VOID);
             STATIC XTAPI ULONG_PTR GetNextEntry(IN PMMPTE Pte);
             STATIC XTAPI PMMPTE GetNextPte(IN PMMPTE Pte);
@@ -35,7 +36,7 @@ namespace MM
             STATIC XTAPI PVOID GetPdeVirtualAddress(IN PMMPDE PdePointer);
             STATIC XTAPI PMMPPE GetPpeAddress(IN PVOID Address);
             STATIC XTAPI PVOID GetPpeVirtualAddress(IN PMMPPE PpePointer);
-            STATIC XTAPI ULONG_PTR GetPte(IN PMMPTE PtePointer);
+            STATIC XTAPI ULONGLONG GetPte(IN PMMPTE PtePointer);
             STATIC XTAPI PMMPTE GetPteAddress(IN PVOID Address);
             STATIC XTAPI LONG GetPteDistance(PMMPTE EndPte,
                                              PMMPTE StartPte);
@@ -53,9 +54,9 @@ namespace MM
                                           IN BOOLEAN Value);
             STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
                                      IN PFN_NUMBER PageFrameNumber,
-                                     IN ULONG_PTR AttributesMask);
+                                     IN ULONGLONG AttributesMask);
             STATIC XTAPI VOID SetPte(IN PMMPTE PtePointer,
-                                     IN ULONG_PTR Attributes);
+                                     IN ULONGLONG Attributes);
             STATIC XTAPI VOID SetPteCaching(IN PMMPTE PtePointer,
                                             IN BOOLEAN CacheDisable,
                                             IN BOOLEAN WriteThrough);

xtoskrnl/includes/mm/kpool.hh

@@ -17,9 +17,6 @@ namespace MM
 {
     class KernelPool
     {
-        private:
-            STATIC UCHAR ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE];
-
         public:
             STATIC XTAPI XTSTATUS AllocateKernelStack(OUT PVOID *Stack,
                                                       IN ULONG StackSize);

xtoskrnl/includes/mm/mmgr.hh

@@ -20,11 +20,13 @@ namespace MM
         private:
             STATIC MMMEMORY_LAYOUT MemoryLayout;
             STATIC PFN_NUMBER NumberOfSystemPtes;
+            STATIC PPHYSICAL_MEMORY_DESCRIPTOR PhysicalMemoryBlock;
 
         public:
             STATIC XTAPI ULONG_PTR GetInstalledMemorySize(VOID);
             STATIC XTAPI PMMMEMORY_LAYOUT GetMemoryLayout(VOID);
-            STATIC XTAPI PFN_NUMBER GetNumberOfSystemPtes();
+            STATIC XTAPI PFN_NUMBER GetNumberOfSystemPtes(VOID);
+            STATIC XTAPI PPHYSICAL_MEMORY_DESCRIPTOR GetPhysicalMemoryBlock(VOID);
             STATIC XTAPI VOID InitializeMemoryLayout(VOID);
             STATIC XTAPI VOID InitializeMemoryManager(VOID);
             STATIC XTAPI BOOLEAN VerifyMemoryTypeFree(IN LOADER_MEMORY_TYPE MemoryType);

xtoskrnl/includes/mm/pfn.hh

@@ -30,6 +30,7 @@ namespace MM
             STATIC ULONGLONG NumberOfPhysicalPages;
             STATIC LOADER_MEMORY_DESCRIPTOR OriginalFreeDescriptor;
             STATIC PMMPFNLIST PageLocationList[];
+            STATIC RTL_BITMAP PfnBitMap;
             STATIC MMPFNLIST RomPagesList;
             STATIC MMPFNLIST StandbyPagesList;
             STATIC MMPFNLIST ZeroedPagesList;
@@ -49,6 +50,7 @@ namespace MM
             STATIC XTAPI ULONG_PTR GetHighestPhysicalPage(VOID);
             STATIC XTAPI ULONGLONG GetNumberOfPhysicalPages(VOID);
             STATIC XTAPI PMMPFN GetPfnEntry(IN PFN_NUMBER Pfn);
+            STATIC XTAPI VOID InitializePfnBitmap(VOID);
             STATIC XTAPI VOID InitializePfnDatabase(VOID);
             STATIC XTAPI VOID LinkPfn(IN PFN_NUMBER PageFrameIndex,
                                       IN PMMPTE PointerPte,

xtoskrnl/includes/mm/pool.hh

@@ -0,0 +1,68 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/includes/mm/pool.hh
+ * DESCRIPTION:     Memory Manager pool manager
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#ifndef __XTOSKRNL_MM_POOL_HH
+#define __XTOSKRNL_MM_POOL_HH
+
+#include <xtos.hh>
+
+
+/* Memory Manager */
+namespace MM
+{
+    class Pool
+    {
+        protected:
+            STATIC POOL_DESCRIPTOR NonPagedPoolDescriptor;
+            STATIC PFN_NUMBER NonPagedPoolFrameEnd;
+            STATIC PFN_NUMBER NonPagedPoolFrameStart;
+            STATIC LIST_ENTRY NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS];
+            STATIC ULONG PoolSecureCookie;
+            STATIC PPOOL_DESCRIPTOR PoolVector[2];
+
+        public:
+            STATIC XTAPI MMPOOL_TYPE DeterminePoolType(IN PVOID VirtualAddress);
+            STATIC XTAPI VOID InitializeNonPagedPool(VOID);
+            STATIC XTAPI VOID InitializePagedPool(VOID);
+            STATIC XTAPI VOID InitializePoolSecurity(VOID);
+
+        protected:
+            STATIC XTAPI PLIST_ENTRY DecodePoolLink(IN PLIST_ENTRY PoolLink);
+            STATIC XTAPI PLIST_ENTRY EncodePoolLink(IN PLIST_ENTRY PoolLink);
+            STATIC XTAPI PPOOL_HEADER GetPoolBlock(IN PPOOL_HEADER Header, IN SSIZE_T Index);
+            STATIC XTAPI PPOOL_HEADER GetPoolEntry(IN PVOID Payload);
+            STATIC XTAPI PLIST_ENTRY GetPoolFreeBlock(IN PPOOL_HEADER Header);
+            STATIC XTAPI PPOOL_HEADER GetPoolNextBlock(IN PPOOL_HEADER Header);
+            STATIC XTAPI PPOOL_HEADER GetPoolPreviousBlock(IN PPOOL_HEADER Header);
+            STATIC XTAPI VOID InsertPoolHeadList(IN PLIST_ENTRY ListHead,
+                                                 IN PLIST_ENTRY Entry);
+            STATIC XTAPI VOID InsertPoolTailList(IN PLIST_ENTRY ListHead,
+                                                 IN PLIST_ENTRY Entry);
+            STATIC XTAPI BOOLEAN PoolListEmpty(IN PLIST_ENTRY ListHead);
+            STATIC XTAPI VOID RemovePoolEntryList(IN PLIST_ENTRY Entry);
+            STATIC XTAPI PLIST_ENTRY RemovePoolHeadList(IN PLIST_ENTRY ListHead);
+            STATIC XTAPI PLIST_ENTRY RemovePoolTailList(IN PLIST_ENTRY ListHead);
+            STATIC XTAPI VOID VerifyPoolBlocks(IN PVOID Block);
+            STATIC XTAPI VOID VerifyPoolHeader(IN PPOOL_HEADER Entry);
+            STATIC XTAPI VOID VerifyPoolLinks(IN PLIST_ENTRY ListHead);
+            STATIC XTAPI VOID VerifyRunLevel(IN MMPOOL_TYPE PoolType,
+                                             IN SIZE_T Bytes,
+                                             IN PVOID Entry);
+
+        private:
+            STATIC XTAPI VOID InitializePoolDescriptor(IN PPOOL_DESCRIPTOR Descriptor,
+                                                       IN MMPOOL_TYPE PoolType,
+                                                       IN ULONG Index,
+                                                       IN ULONG Threshold,
+                                                       IN PVOID LockAddress);
+            STATIC XTAPI VOID InitializePoolListHead(IN PLIST_ENTRY ListHead);
+            STATIC XTAPI VOID MapNonPagedPool(VOID);
+    };
+}
+
+#endif /* __XTOSKRNL_MM_POOL_HH */

xtoskrnl/ke/amd64/irq.cc

@@ -33,7 +33,11 @@ KE::Irq::SetInterruptHandler(IN ULONG Vector,
     ProcessorBlock = KE::Processor::GetCurrentProcessorBlock();
 
     /* Update interrupt handler */
-    ProcessorBlock->IdtBase[(UCHAR) Vector].OffsetLow = ((ULONG_PTR)Handler & 0xFFFF);
+    AR::ProcSup::SetIdtGate(ProcessorBlock->IdtBase,
-    ProcessorBlock->IdtBase[(UCHAR) Vector].OffsetMiddle = (((ULONG_PTR)Handler >> 16) & 0xFFFF);
+                            Vector,
-    ProcessorBlock->IdtBase[(UCHAR) Vector].OffsetHigh = (ULONG_PTR)Handler >> 32;
+                            Handler,
+                            KGDT_R0_CODE,
+                            0,
+                            KIDT_ACCESS_RING0,
+                            AMD64_INTERRUPT_GATE);
 }

xtoskrnl/ke/crash.cc

@@ -43,7 +43,7 @@ XTAPI
 VOID
 KE::Crash::Panic(IN ULONG Code)
 {
-    PanicEx(Code, 0, 0, 0, 0);
+    Panic(Code, 0, 0, 0, 0);
 }
 
 /**
@@ -70,12 +70,13 @@ KE::Crash::Panic(IN ULONG Code)
  */
 XTAPI
 VOID
-KE::Crash::PanicEx(IN ULONG Code,
+KE::Crash::Panic(IN ULONG Code,
                  IN ULONG_PTR Parameter1,
                  IN ULONG_PTR Parameter2,
                  IN ULONG_PTR Parameter3,
                  IN ULONG_PTR Parameter4)
 {
-    KD::DebugIo::KdPrint(L"Fatal System Error: 0x%08lx\nKernel Panic!\n\n", Code);
+    KD::DebugIo::KdPrint(L"Fatal System Error: 0x%08lx (0x%zx 0x%zx 0x%zx 0x%zx)\nKernel Panic!\n\n",
+                         Code, Parameter1, Parameter2, Parameter3, Parameter4);
     HaltSystem();
 }

xtoskrnl/ke/i686/irq.cc

@@ -33,6 +33,11 @@ KE::Irq::SetInterruptHandler(IN ULONG Vector,
     ProcessorBlock = KE::Processor::GetCurrentProcessorBlock();
 
     /* Update interrupt handler */
-    ProcessorBlock->IdtBase[(UCHAR) Vector].Offset = (USHORT)((ULONG)Handler & 0xFFFF);
+    AR::ProcSup::SetIdtGate(ProcessorBlock->IdtBase,
-    ProcessorBlock->IdtBase[(UCHAR) Vector].ExtendedOffset = (USHORT)((ULONG)Handler >> 16);
+                            Vector,
+                            Handler,
+                            KGDT_R0_CODE,
+                            0,
+                            KIDT_ACCESS_RING0,
+                            I686_INTERRUPT_GATE);
 }

xtoskrnl/mm/amd64/pagemap.cc

@@ -302,7 +302,7 @@ MM::PageMap::GetPpeOffset(IN PVOID Address)
  * @since XT 1.0
  */
 XTAPI
-ULONG_PTR
+ULONGLONG
 MM::PageMap::GetPte(IN PMMPTE PtePointer)
 {
     /* Return PTE value */
@@ -574,7 +574,7 @@ XTAPI
 VOID
 MM::PageMap::SetPte(IN PMMPTE PtePointer,
                     IN PFN_NUMBER PageFrameNumber,
-                    IN ULONG_PTR AttributesMask)
+                    IN ULONGLONG AttributesMask)
 {
     /* Set PTE */
     PtePointer->Hardware.PageFrameNumber = PageFrameNumber;
@@ -598,7 +598,7 @@ MM::PageMap::SetPte(IN PMMPTE PtePointer,
 XTAPI
 VOID
 MM::PageMap::SetPte(IN PMMPTE PtePointer,
-                    IN ULONG_PTR Attributes)
+                    IN ULONGLONG Attributes)
 {
     PtePointer->Long = Attributes;
 }

xtoskrnl/mm/amd64/pool.cc

@@ -1,8 +1,8 @@
 /**
  * PROJECT:         ExectOS
  * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/mm/amd64/alloc.cc
+ * FILE:            xtoskrnl/mm/amd64/pool.cc
- * DESCRIPTION:     Memory manager pool allocation
+ * DESCRIPTION:     AMD64 Memory Manager pool manager
  * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
  */
 
@@ -18,7 +18,7 @@
  */
 XTAPI
 VOID
-MM::Allocator::MapNonPagedPool(VOID)
+MM::Pool::MapNonPagedPool(VOID)
 {
     PMMMEMORY_LAYOUT MemoryLayout;
 

xtoskrnl/mm/data.cc

@@ -9,14 +9,41 @@
 #include <xtos.hh>
 
 
-/* PFN marking the initial non-paged pool end boundary */
+/* Expansion table used to track pool memory allocations */
-PFN_NUMBER MM::Allocator::NonPagedPoolFrameEnd;
+PPOOL_TRACKING_TABLE MM::Allocator::AllocationsTrackingExpansionTable;
 
-/* PFN marking the initial non-paged pool start boundary */
+/* Total number of entries in the expansion allocations tracking table */
-PFN_NUMBER MM::Allocator::NonPagedPoolFrameStart;
+SIZE_T MM::Allocator::AllocationsTrackingExpansionTableSize;
 
-/* Array of non-paged pool free list heads */
+/* Global table used to track pool memory allocations */
-LIST_ENTRY MM::Allocator::NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS];
+PPOOL_TRACKING_TABLE MM::Allocator::AllocationsTrackingTable;
+
+/* Spinlock protecting the allocations table */
+KSPIN_LOCK MM::Allocator::AllocationsTrackingTableLock;
+
+/* Bitmask used during the hashing process */
+SIZE_T MM::Allocator::AllocationsTrackingTableMask;
+
+/* Total number of entries in the global allocations tracking table */
+SIZE_T MM::Allocator::AllocationsTrackingTableSize;
+
+/* Active number of big allocations to trigger table expansion */
+ULONG MM::Allocator::BigAllocationsInUse;
+
+/* Pointer to the hash table for tracking page-aligned memory */
+PPOOL_TRACKING_BIG_ALLOCATIONS MM::Allocator::BigAllocationsTrackingTable;
+
+/* Bitmask used for fast modulo arithmetic during hash bucket lookups */
+SIZE_T MM::Allocator::BigAllocationsTrackingTableHash;
+
+/* Spinlock protecting the big allocations table */
+KSPIN_LOCK MM::Allocator::BigAllocationsTrackingTableLock;
+
+/* Maximum capacity of the tracking hash table */
+SIZE_T MM::Allocator::BigAllocationsTrackingTableSize;
+
+/* Array of CPU-local tracking tables */
+PPOOL_TRACKING_TABLE MM::Allocator::TagTables[MM_POOL_TRACKING_TABLES];
 
 /* Array of free page lists segregated by cache color */
 PMMCOLOR_TABLES MM::Colors::FreePages[FreePageList + 1];
@@ -39,15 +66,15 @@ PVOID MM::HardwarePool::HardwareHeapStart = MM_HARDWARE_HEAP_START_ADDRESS;
 /* Number of used hardware allocation descriptors */
 ULONG MM::HardwarePool::UsedHardwareAllocationDescriptors = 0;
 
-/* Processor structures data (THIS IS A TEMPORARY HACK) */
-UCHAR MM::KernelPool::ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE] = {{0}};
-
 /* Global structure describing the virtual memory layout of the system */
 MMMEMORY_LAYOUT MM::Manager::MemoryLayout;
 
 /* Total number of PTEs reserved for system space mapping */
 PFN_NUMBER MM::Manager::NumberOfSystemPtes;
 
+/* Physical memory block descriptor */
+PPHYSICAL_MEMORY_DESCRIPTOR MM::Manager::PhysicalMemoryBlock;
+
 /* Instance of the page map routines for the current PML level */
 MM::PPAGEMAP MM::Paging::PmlRoutines;
 
@@ -91,6 +118,9 @@ PMMPFNLIST MM::Pfn::PageLocationList[] = {&ZeroedPagesList,
                                           NULLPTR,
                                           NULLPTR};
 
+/* Bitmap used to track physical pages */
+RTL_BITMAP MM::Pfn::PfnBitMap;
+
 /* List containing pages mapped as Read-Only (ROM) */
 MMPFNLIST MM::Pfn::RomPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR};
 
@@ -100,6 +130,24 @@ MMPFNLIST MM::Pfn::StandbyPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULON
 /* List containing free physical pages that have been zeroed out */
 MMPFNLIST MM::Pfn::ZeroedPagesList = {0, ZeroedPageList, MAXULONG_PTR, MAXULONG_PTR};
 
+/* Non-paged pool descriptor */
+POOL_DESCRIPTOR MM::Pool::NonPagedPoolDescriptor;
+
+/* PFN marking the initial non-paged pool end boundary */
+PFN_NUMBER MM::Pool::NonPagedPoolFrameEnd;
+
+/* PFN marking the initial non-paged pool start boundary */
+PFN_NUMBER MM::Pool::NonPagedPoolFrameStart;
+
+/* Array of non-paged pool free list heads */
+LIST_ENTRY MM::Pool::NonPagedPoolFreeList[MM_MAX_FREE_PAGE_LIST_HEADS];
+
+/* Random cookie used to obfuscate pool links */
+ULONG MM::Pool::PoolSecureCookie;
+
+/* Array of pool descriptors */
+PPOOL_DESCRIPTOR MM::Pool::PoolVector[2];
+
 /* Array of lists for available System PTEs, separated by pool type */
 MMPTE MM::Pte::FirstSystemFreePte[MaximumPtePoolTypes];
 

xtoskrnl/mm/exports.cc

@@ -0,0 +1,102 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/exports.cc
+ * DESCRIPTION:     C-compatible API wrappers for exported kernel functions
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Allocates a block of memory from the specified pool type.
+ *
+ * @param PoolType
+ *        Specifies the type of pool to allocate from.
+ *
+ * @param Bytes
+ *        Specifies the number of bytes to allocate.
+ *
+ * @param Memory
+ *        Supplies a pointer to the allocated memory.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+XTSTATUS
+MmAllocatePool(IN MMPOOL_TYPE PoolType,
+               IN SIZE_T Bytes,
+               OUT PVOID *Memory)
+{
+    return MM::Allocator::AllocatePool(PoolType, Bytes, Memory);
+}
+
+/**
+ * Allocates a block of memory from the specified pool type.
+ *
+ * @param PoolType
+ *        Specifies the type of pool to allocate from.
+ *
+ * @param Bytes
+ *        Specifies the number of bytes to allocate.
+ *
+ * @param Memory
+ *        Supplies a pointer to the allocated memory.
+ *
+ * @param Tag
+ *        Specifies the allocation identifying tag.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+XTSTATUS
+MmAllocatePoolWithTag(IN MMPOOL_TYPE PoolType,
+                      IN SIZE_T Bytes,
+                      OUT PVOID *Memory,
+                      IN ULONG Tag)
+{
+    return MM::Allocator::AllocatePool(PoolType, Bytes, Memory, Tag);
+}
+
+/**
+ * Frees a previously allocated memory pool.
+ *
+ * @param VirtualAddress
+ *        Supplies the base virtual address of the pool allocation to free.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+XTSTATUS
+MmFreePool(IN PVOID VirtualAddress)
+{
+    return MM::Allocator::FreePool(VirtualAddress);
+}
+
+/**
+ * Frees a previously allocated memory pool.
+ *
+ * @param VirtualAddress
+ *        Supplies the base virtual address of the pool allocation to free.
+ *
+ * @param Tag
+ *        Specifies the allocation identifying tag.
+ *
+ * @return This routine returns a status code.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+XTSTATUS
+MmFreePoolWithTag(IN PVOID VirtualAddress,
+                  IN ULONG Tag)
+{
+    return MM::Allocator::FreePool(VirtualAddress, Tag);
+}

xtoskrnl/mm/i686/pagemap.cc

@@ -80,7 +80,6 @@ MM::PageMap::GetPdeOffset(IN PVOID Address)
     return ((((ULONG_PTR)(Address)) >> PageMapInfo.PdiShift) & (PageMapInfo.Xpa ? 0x1FF : 0x3FF));
 }
 
-
 /**
  * Gets the address of the PPE (Page Directory Pointer Table Entry), that maps given address.
  *
@@ -174,7 +173,6 @@ MM::PageMap::GetPteOffset(IN PVOID Address)
     return ((((ULONG_PTR)(Address)) >> MM_PTI_SHIFT) & (PageMapInfo.Xpa ? 0x1FF : 0x3FF));
 }
 
-
 /**
  * Gets the status of Extended Paging Address (XPA) mode.
  *
@@ -318,7 +316,7 @@ MM::PageMapBasic::GetPdeVirtualAddress(IN PMMPDE PdePointer)
     return ((PVOID)((ULONG)(PdePointer) << 20));
 }
 
- /**
+/**
  * Gets the entire contents of a PML2 Page Table Entry (PTE) as a single value.
  *
  * @param PtePointer
@@ -329,12 +327,13 @@ MM::PageMapBasic::GetPdeVirtualAddress(IN PMMPDE PdePointer)
  * @since XT 1.0
  */
 XTAPI
-ULONG_PTR
+ULONGLONG
 MM::PageMapBasic::GetPte(IN PMMPTE PtePointer)
 {
     /* Return PTE value */
-    return PtePointer->Pml2.Long;
+    return (ULONGLONG)PtePointer->Pml2.Long;
 }
+
 /**
  * Calculates the distance between two PTE pointers.
  *
@@ -549,12 +548,12 @@ XTAPI
 VOID
 MM::PageMapBasic::SetPte(IN PMMPTE PtePointer,
                          IN PFN_NUMBER PageFrameNumber,
-                         IN ULONG_PTR AttributesMask)
+                         IN ULONGLONG AttributesMask)
 {
     /* Set PTE */
     PtePointer->Pml2.Hardware.PageFrameNumber = PageFrameNumber;
     PtePointer->Pml2.Hardware.Valid = 1;
-    PtePointer->Pml2.Long |= AttributesMask;
+    PtePointer->Pml2.Long |= (ULONG)AttributesMask;
 }
 
 /**
@@ -573,9 +572,9 @@ MM::PageMapBasic::SetPte(IN PMMPTE PtePointer,
 XTAPI
 VOID
 MM::PageMapBasic::SetPte(IN PMMPTE PtePointer,
-                         IN ULONG_PTR Attributes)
+                         IN ULONGLONG Attributes)
 {
-    PtePointer->Pml2.Long = Attributes;
+    PtePointer->Pml2.Long = (ULONG)Attributes;
 }
 
 /**
@@ -626,14 +625,14 @@ MM::PageMapBasic::TransitionPte(IN PMMPTE PointerPte,
     MMPTE TempPte;
 
     /* Set transition PTE */
-    TempPte = *PointerPte;
+    TempPte.Pml2.Long = PointerPte->Pml2.Long;
     TempPte.Pml2.Software.Protection = Protection;
     TempPte.Pml2.Software.Prototype = 0;
     TempPte.Pml2.Software.Transition = 1;
     TempPte.Pml2.Software.Valid = 0;
 
     /* Write PTE value */
-    *PointerPte = TempPte;
+    PointerPte->Pml2.Long = TempPte.Pml2.Long;
 }
 
 /**
@@ -798,7 +797,7 @@ MM::PageMapXpa::GetPdeVirtualAddress(IN PMMPDE PdePointer)
  * @since XT 1.0
  */
 XTAPI
-ULONG_PTR
+ULONGLONG
 MM::PageMapXpa::GetPte(IN PMMPTE PtePointer)
 {
     /* Return PTE value */
@@ -1018,7 +1017,7 @@ XTAPI
 VOID
 MM::PageMapXpa::SetPte(IN PMMPTE PtePointer,
                        IN PFN_NUMBER PageFrameNumber,
-                       IN ULONG_PTR AttributesMask)
+                       IN ULONGLONG AttributesMask)
 {
     /* Set PTE */
     PtePointer->Pml3.Hardware.PageFrameNumber = PageFrameNumber;
@@ -1042,7 +1041,7 @@ MM::PageMapXpa::SetPte(IN PMMPTE PtePointer,
 XTAPI
 VOID
 MM::PageMapXpa::SetPte(IN PMMPTE PtePointer,
-                       IN ULONG_PTR Attributes)
+                       IN ULONGLONG Attributes)
 {
     PtePointer->Pml3.Long = Attributes;
 }
@@ -1095,14 +1094,14 @@ MM::PageMapXpa::TransitionPte(IN PMMPTE PointerPte,
     MMPTE TempPte;
 
     /* Set transition PTE */
-    TempPte = *PointerPte;
+    TempPte.Pml3.Long = PointerPte->Pml3.Long;
     TempPte.Pml3.Software.Protection = Protection;
     TempPte.Pml3.Software.Prototype = 0;
     TempPte.Pml3.Software.Transition = 1;
     TempPte.Pml3.Software.Valid = 0;
 
     /* Write PTE value */
-    *PointerPte = TempPte;
+    PointerPte->Pml3.Long = TempPte.Pml3.Long;
 }
 
 /**

xtoskrnl/mm/i686/pool.cc

@@ -1,8 +1,8 @@
 /**
  * PROJECT:         ExectOS
  * COPYRIGHT:       See COPYING.md in the top level directory
- * FILE:            xtoskrnl/mm/i686/alloc.cc
+ * FILE:            xtoskrnl/mm/i686/pool.cc
- * DESCRIPTION:     Memory manager pool allocation
+ * DESCRIPTION:     I686 Memory Manager pool manager
  * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
  */
 
@@ -18,7 +18,7 @@
  */
 XTAPI
 VOID
-MM::Allocator::MapNonPagedPool(VOID)
+MM::Pool::MapNonPagedPool(VOID)
 {
     PMMMEMORY_LAYOUT MemoryLayout;
 

xtoskrnl/mm/kpool.cc

@@ -113,12 +113,15 @@ MM::KernelPool::AllocateProcessorStructures(IN ULONG CpuNumber,
     PKPROCESSOR_BLOCK ProcessorBlock;
     PVOID ProcessorStructures;
     UINT_PTR Address;
+    XTSTATUS Status;
 
-    /* Not implemented yet, this is just a hack */
+    /* Assign memory for processor structures */
-    UNIMPLEMENTED;
+    Status = MM::Allocator::AllocatePool(NonPagedPool, KPROCESSOR_STRUCTURES_SIZE, &ProcessorStructures);
-
+    if(Status != STATUS_SUCCESS)
-    /* Assign memory for processor structures from preallocated buffer */
+    {
-    ProcessorStructures = &ProcessorStructuresData[CpuNumber - 1];
+        /* Failed to allocate memory, return status code */
+        return Status;
+    }
 
     /* Make sure all structures are zeroed */
     RTL::Memory::ZeroMemory(ProcessorStructures, KPROCESSOR_STRUCTURES_SIZE);
@@ -203,5 +206,10 @@ XTAPI
 VOID
 MM::KernelPool::FreeProcessorStructures(IN PVOID StructuresData)
 {
-    UNIMPLEMENTED;
+    /* Check if the provided pointer is valid */
+    if(StructuresData != NULLPTR)
+    {
+        /* Release the contiguous memory block back */
+        MM::Allocator::FreePool(StructuresData, 0);
+    }
 }

xtoskrnl/mm/mmgr.cc

@@ -81,11 +81,146 @@ MM::Manager::GetMemoryLayout(VOID)
  */
 XTAPI
 PFN_NUMBER
-MM::Manager::GetNumberOfSystemPtes()
+MM::Manager::GetNumberOfSystemPtes(VOID)
 {
     return NumberOfSystemPtes;
 }
 
+/**
+ * Initializes and returns the system physical memory descriptor block.
+ *
+ * @return This routine returns a pointer to the structure representing the system usable physical memory block.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PPHYSICAL_MEMORY_DESCRIPTOR
+MM::Manager::GetPhysicalMemoryBlock(VOID)
+{
+    PPHYSICAL_MEMORY_DESCRIPTOR PrimaryBuffer, SecondaryBuffer;
+    PKERNEL_INITIALIZATION_BLOCK InitializationBlock;
+    PLOADER_MEMORY_DESCRIPTOR MemoryDescriptor;
+    PFN_NUMBER PageFrameNumer, NumberOfPages;
+    ULONG DescriptorCount, RunCount;
+    PLIST_ENTRY ListEntry;
+    XTSTATUS Status;
+
+    /* Check if the physical memory block has already been initialized */
+    if(!PhysicalMemoryBlock)
+    {
+        /* Reset local tracking variables */
+        DescriptorCount = 0;
+        NumberOfPages = 0;
+        PageFrameNumer = -1;
+        RunCount = 0;
+
+        /* Retrieve the kernel initialization block */
+        InitializationBlock = KE::BootInformation::GetInitializationBlock();
+
+        /* Iterate through the loader memory descriptor list to determine its size */
+        ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink;
+        while(ListEntry != &InitializationBlock->MemoryDescriptorListHead)
+        {
+            /* Count this descriptor */
+            DescriptorCount++;
+
+            /* Go to the next descriptor */
+            ListEntry = ListEntry->Flink;
+        }
+
+        /* Ensure the memory descriptor list is not empty */
+        if(DescriptorCount == 0)
+        {
+            /* Fail gracefully if no memory descriptors were found, by returning NULLPTR */
+            return NULLPTR;
+        }
+
+        /* Allocate a primary buffer sized for the maximum possible number of runs */
+        Status = MM::Allocator::AllocatePool(NonPagedPool,
+                                             sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
+                                             sizeof(PHYSICAL_MEMORY_RUN) *
+                                             (DescriptorCount - 1),
+                                             (PVOID*)&PrimaryBuffer,
+                                             SIGNATURE32('M', 'M', 'g', 'r'));
+        if(Status != STATUS_SUCCESS || !PrimaryBuffer)
+        {
+            /* Primary pool allocation failed, return NULLPTR */
+            return NULLPTR;
+        }
+
+        /* Traverse the memory descriptor list a second time to build the map */
+        ListEntry = InitializationBlock->MemoryDescriptorListHead.Flink;
+        while(ListEntry != &InitializationBlock->MemoryDescriptorListHead)
+        {
+            /* Resolve the memory descriptor record from the current list entry */
+            MemoryDescriptor = CONTAIN_RECORD(ListEntry, LOADER_MEMORY_DESCRIPTOR, ListEntry);
+
+            /* Filter out bad, reserved, or invisible memory types */
+            if((MemoryDescriptor->MemoryType < LoaderMaximum) &&
+               (MemoryDescriptor->MemoryType != LoaderBad) &&
+               !VerifyMemoryTypeInvisible(MemoryDescriptor->MemoryType))
+            {
+                /* Accumulate the total number of usable physical pages */
+                NumberOfPages += MemoryDescriptor->PageCount;
+
+                /* Check if the current descriptor is contiguous with the previous run */
+                if(RunCount > 0 && MemoryDescriptor->BasePage == PageFrameNumer)
+                {
+                    /* Coalesce the contiguous descriptor into the existing physical run */
+                    PrimaryBuffer->Run[RunCount - 1].PageCount += MemoryDescriptor->PageCount;
+                    PageFrameNumer += MemoryDescriptor->PageCount;
+                }
+                else
+                {
+                    /* Start a new physical run with the new descriptor's boundaries */
+                    PrimaryBuffer->Run[RunCount].BasePage = MemoryDescriptor->BasePage;
+                    PrimaryBuffer->Run[RunCount].PageCount = MemoryDescriptor->PageCount;
+
+                    /* Update the expected next page frame number for future contiguity checks */
+                    PageFrameNumer = PrimaryBuffer->Run[RunCount].BasePage + PrimaryBuffer->Run[RunCount].PageCount;
+
+                    /* Increment the total number of distinct physical memory runs */
+                    RunCount++;
+                }
+            }
+
+            /* Go to the next descriptor */
+            ListEntry = ListEntry->Flink;
+        }
+
+        /* Check if the buffer can be shrunk due to coalesced memory runs */
+        if(DescriptorCount > RunCount)
+        {
+            /* Allocate a secondary, more tightly sized buffer to reduce memory footprint */
+            Status = MM::Allocator::AllocatePool(NonPagedPool,
+                                                 sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
+                                                 sizeof(PHYSICAL_MEMORY_RUN) *
+                                                 (RunCount - 1),
+                                                 (PVOID*)&SecondaryBuffer,
+                                                 SIGNATURE32('M', 'M', 'g', 'r'));
+            if(Status == STATUS_SUCCESS && SecondaryBuffer)
+            {
+                /* Copy the coalesced runs from the oversized primary buffer */
+                RtlCopyMemory(SecondaryBuffer->Run, PrimaryBuffer->Run, sizeof(PHYSICAL_MEMORY_RUN) * RunCount);
+
+                /* Free the primary buffer */
+                MM::Allocator::FreePool(PrimaryBuffer, SIGNATURE32('M', 'M', 'g', 'r'));
+
+                /* Update the primary buffer pointer */
+                PrimaryBuffer = SecondaryBuffer;
+            }
+        }
+
+        /* Populate the final metadata and save the physical memory block globally */
+        PrimaryBuffer->NumberOfRuns = RunCount;
+        PrimaryBuffer->NumberOfPages = NumberOfPages;
+        PhysicalMemoryBlock = PrimaryBuffer;
+    }
+
+    /* Return a pointer to the physical memory block */
+    return PhysicalMemoryBlock;
+}
+
 /**
  * Performs an early initialization of the XTOS Memory Manager.
  *
@@ -105,7 +240,7 @@ MM::Manager::InitializeMemoryManager(VOID)
     {
         /* Insufficient physical pages, kernel panic */
         DebugPrint(L"Insufficient physical pages! Install additional memory\n");
-        KE::Crash::Panic(0);
+        KE::Crash::Panic(0x7D, MM::Pfn::GetNumberOfPhysicalPages(), MM_MINIMUM_PHYSICAL_PAGES, 0x0, 0x2);
     }
 
     /* Compute page colors to reduce CPU cache conflicts */
@@ -124,14 +259,24 @@ MM::Manager::InitializeMemoryManager(VOID)
     /* Initialize system PTE space */
     MM::Pte::InitializeSystemPteSpace();
 
+    /* Initialize memory pool security */
+    MM::Pool::InitializePoolSecurity();
+
     /* Initialize non-paged pool */
-    MM::Allocator::InitializeNonPagedPool();
+    MM::Pool::InitializeNonPagedPool();
 
     /* Initialize PFN database */
     MM::Pfn::InitializePfnDatabase();
 
+    /* Initialize allocations tracking tables */
+    MM::Allocator::InitializeAllocationsTracking();
+    MM::Allocator::InitializeBigAllocationsTracking();
+
+    /* Initialize PFN bitmap */
+    MM::Pfn::InitializePfnBitmap();
+
     /* Initialize paged pool */
-    MM::Allocator::InitializePagedPool();
+    MM::Pool::InitializePagedPool();
 
     /* Flush TLB */
     AR::CpuFunc::FlushTlb();

xtoskrnl/mm/paging.cc

@@ -68,7 +68,22 @@ MM::Paging::ClearPte(IN PMMPTE PtePointer)
 }
 
 /**
- * Flushes current Translation Lookaside Buffer (TLB)
+ * Flushes the entire Translation Lookaside Buffer (TLB) on all processors.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Paging::FlushEntireTlb(VOID)
+{
+    /* Temporarily fallback to FlushTlb() as SMP is not supported yet */
+    FlushTlb();
+}
+
+/**
+ * Flushes current Translation Lookaside Buffer (TLB).
  *
  * @return This routine does not return any value.
  *
@@ -321,7 +336,7 @@ MM::Paging::GetPpeVirtualAddress(IN PMMPPE PpePointer)
  * @since XT 1.0
  */
 XTAPI
-ULONG_PTR
+ULONGLONG
 MM::Paging::GetPte(IN PMMPTE PtePointer)
 {
     /* Return PTE value */
@@ -578,7 +593,7 @@ XTAPI
 VOID
 MM::Paging::SetPte(IN PMMPTE PtePointer,
                    IN PFN_NUMBER PageFrameNumber,
-                   IN ULONG_PTR AttributesMask)
+                   IN ULONGLONG AttributesMask)
 {
     /* Set PTE */
     PmlRoutines->SetPte(PtePointer, PageFrameNumber, AttributesMask);
@@ -600,7 +615,7 @@ MM::Paging::SetPte(IN PMMPTE PtePointer,
 XTAPI
 VOID
 MM::Paging::SetPte(IN PMMPTE PtePointer,
-                   IN ULONG_PTR Attributes)
+                   IN ULONGLONG Attributes)
 {
     PmlRoutines->SetPte(PtePointer, Attributes);
 }

xtoskrnl/mm/pfn.cc

@@ -31,7 +31,11 @@ MM::Pfn::AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages)
     {
         /* Not enough physical memory available, kernel panic */
         DebugPrint(L"Insufficient physical pages! Install additional memory\n");
-        KE::Crash::Panic(0);
+        KE::Crash::Panic(0x7D,
+                         NumberOfPhysicalPages,
+                         FreeDescriptor->PageCount,
+                         OriginalFreeDescriptor.PageCount,
+                         NumberOfPages);
     }
 
     /* Allocate pages from the beginning of the free descriptor */
@@ -173,7 +177,7 @@ MM::Pfn::DecrementReferenceCount(IN PMMPFN PageFrameNumber,
     if(PageFrameNumber->u2.ShareCount)
     {
         /* This indicates a bug; crash the system */
-        KE::Crash::PanicEx(0x4E,
+        KE::Crash::Panic(0x4E,
                          0x07,
                          PageFrameIndex,
                          PageFrameNumber->u2.ShareCount,
@@ -181,13 +185,14 @@ MM::Pfn::DecrementReferenceCount(IN PMMPFN PageFrameNumber,
     }
 
     /* Check if the PTE is marked as being ready for removal */
-    if(MM::Paging::GetPte(PageFrameNumber->PteAddress) & 0x1)
+    if((ULONG_PTR)PageFrameNumber->PteAddress & 0x1)
     {
         /* Check the page's cache attribute */
         if((PageFrameNumber->u3.e1.CacheAttribute != PfnCached) &&
            (PageFrameNumber->u3.e1.CacheAttribute != PfnNotMapped))
         {
-            UNIMPLEMENTED;
+            /* Flush the TLB to prevent cache attribute conflicts from stale non-cached mappings */
+            MM::Paging::FlushEntireTlb();
         }
 
         /* The page is no longer needed, free it by linking it to the free list */
@@ -256,7 +261,7 @@ MM::Pfn::DecrementShareCount(IN PMMPFN PageFrameNumber,
        (PageFrameNumber->u3.e1.PageLocation != StandbyPageList))
     {
         /* This indicates a bug; crash the system */
-        KE::Crash::PanicEx(0x4E,
+        KE::Crash::Panic(0x4E,
                          0x99,
                          PageFrameIndex,
                          PageFrameNumber->u3.e1.PageLocation,
@@ -284,7 +289,7 @@ MM::Pfn::DecrementShareCount(IN PMMPFN PageFrameNumber,
         if(PageFrameNumber->u3.e2.ReferenceCount == 1)
         {
             /* Check if the PTE is marked as being ready for removal */
-            if(MM::Paging::GetPte(PageFrameNumber->PteAddress) & 0x1)
+            if((ULONG_PTR)PageFrameNumber->PteAddress & 0x1)
             {
                 /* Reset the reference count */
                 PageFrameNumber->u3.e2.ReferenceCount = 0;
@@ -293,7 +298,8 @@ MM::Pfn::DecrementShareCount(IN PMMPFN PageFrameNumber,
                 if((PageFrameNumber->u3.e1.CacheAttribute != PfnCached) &&
                    (PageFrameNumber->u3.e1.CacheAttribute != PfnNotMapped))
                 {
-                    UNIMPLEMENTED;
+                    /* Flush the TLB to prevent cache attribute conflicts from stale non-cached mappings */
+                    MM::Paging::FlushEntireTlb();
                 }
 
                 /* Mark the page as active and valid again */
@@ -419,6 +425,13 @@ MM::Pfn::GetPfnEntry(IN PFN_NUMBER Pfn)
         return NULLPTR;
     }
 
+    /* Make sure this page has a PFN entry set */
+    if(PfnBitMap.Buffer && !RTL::BitMap::TestBit(&PfnBitMap, Pfn))
+    {
+        /* The requested page number is not set in the bitmap, return NULLPTR */
+        return NULLPTR;
+    }
+
     /* Get the memory layout */
     MemoryLayout = MM::Manager::GetMemoryLayout();
 
@@ -441,6 +454,60 @@ MM::Pfn::IncrementAvailablePages(VOID)
     AvailablePages++;
 }
 
+/**
+ * Initializes the PFN bitmap to track available physical memory.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pfn::InitializePfnBitmap(VOID)
+{
+    PPHYSICAL_MEMORY_DESCRIPTOR PhysicalMemoryBlock;
+    XTSTATUS Status;
+    PVOID BitMap;
+    ULONG Index;
+
+    /* Retrieve the global physical memory descriptor block */
+    PhysicalMemoryBlock = MM::Manager::GetPhysicalMemoryBlock();
+    if(!PhysicalMemoryBlock)
+    {
+        /* Physical memory layout unavailable, kernel panic */
+        KE::Crash::Panic(0x7D, NumberOfPhysicalPages, LowestPhysicalPage, HighestPhysicalPage, 0x100);
+    }
+
+    /* Calculate the required bitmap size and allocate memory */
+    Status = MM::Allocator::AllocatePool(NonPagedPool,
+                                         (((HighestPhysicalPage + 1) + 31) / 32) * 4,
+                                         (PVOID *)&BitMap,
+                                         SIGNATURE32('M', 'M', 'g', 'r'));
+    if(Status != STATUS_SUCCESS || !BitMap)
+    {
+        /* Memory allocation failed, kernel panic */
+        DebugPrint(L"Insufficient physical pages! Install additional memory\n");
+        KE::Crash::Panic(0x7D, NumberOfPhysicalPages, LowestPhysicalPage, HighestPhysicalPage, 0x101);
+    }
+
+    /* Initialize the PFN bitmap structure and clear all bits by default */
+    RTL::BitMap::InitializeBitMap(&PfnBitMap, (PULONG_PTR)BitMap, (ULONG)HighestPhysicalPage + 1);
+    RTL::BitMap::ClearAllBits(&PfnBitMap);
+
+    /* Iterate through all contiguous physical memory runs to populate the availability map */
+    for(Index = 0; Index < PhysicalMemoryBlock->NumberOfRuns; Index++)
+    {
+        /* Ensure the current memory run contains at least one valid page frame */
+        if((&PhysicalMemoryBlock->Run[Index])->PageCount)
+        {
+            /* Set the corresponding bits to mark these physical pages as available for allocation */
+            RtlSetBits(&PfnBitMap,
+                       (ULONG)(&PhysicalMemoryBlock->Run[Index])->BasePage,
+                       (ULONG)(&PhysicalMemoryBlock->Run[Index])->PageCount);
+        }
+    }
+}
+
 /**
  * Links a physical page to the appropriate free lists.
  *
@@ -591,7 +658,7 @@ MM::Pfn::LinkPage(IN PMMPFNLIST ListHead,
            MM::Paging::GetPteSoftwareTransition(&PageFrame->OriginalPte))
         {
             /* Crash system due to corrupted PFN/PTE state */
-            KE::Crash::PanicEx(0x71, 0x8888, 0, 0, 0);
+            KE::Crash::Panic(0x1A, 0x8888, 0, 0, 0);
         }
     }
 
@@ -806,7 +873,7 @@ MM::Pfn::LinkPfn(IN PFN_NUMBER PageFrameIndex,
         if(Status != STATUS_SUCCESS)
         {
             /* Could not make the page table resident, crash system */
-            KE::Crash::PanicEx(0x1,
+            KE::Crash::Panic(0x1A,
                             (ULONG_PTR)0x61940,
                             (ULONG_PTR)PointerPte,
                             MM::Paging::GetPageFrameNumber(PointerPte),
@@ -856,7 +923,7 @@ MM::Pfn::LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex,
        (MM::Pte::AddressValid(EndAddress)) && (Pfn->u3.e1.PageLocation == ActiveAndValid))
     {
         /* Initialize the PFN entry for this page table page */
-        Pfn->OriginalPte = *PointerPte;
+        MM::Paging::SetPte(&Pfn->OriginalPte, MM::Paging::GetPte(PointerPte));
         Pfn->PteAddress = PointerPte;
         Pfn->u1.WsIndex = 0;
         Pfn->u2.ShareCount++;
@@ -1034,7 +1101,7 @@ MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
                                  IN PFN_NUMBER PageCount,
                                  IN LOADER_MEMORY_TYPE MemoryType)
 {
-    PVOID VirtualRangeStart, VirtualRangeEnd;
+    PVOID VirtualAddress, VirtualRangeStart, VirtualRangeEnd;
     PFN_NUMBER PageNumber;
     PMMPDE PointerPde;
     PMMPFN Pfn;
@@ -1087,8 +1154,12 @@ MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
                     /* Ensure that the page is not already in-use */
                     if(Pfn->u3.e2.ReferenceCount == 0)
                     {
+                        /* Calculate the virtual address for this page */
+                        VirtualAddress = (PVOID)(KSEG0_BASE + ((BasePage + PageNumber) << MM_PAGE_SHIFT));
+                        PointerPde = MM::Paging::GetPdeAddress(VirtualAddress);
+
                         /* Initialize the PFN entry to represent a ROM page */
-                        Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualRangeStart);
+                        Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualAddress);
                         Pfn->u1.Flink = 0;
                         Pfn->u2.ShareCount = 0;
                         Pfn->u3.e1.CacheAttribute = PfnCached;
@@ -1117,8 +1188,12 @@ MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage,
                     /* Ensure that the page is not already in-use */
                     if(Pfn->u3.e2.ReferenceCount == 0)
                     {
+                        /* Calculate the virtual address for this page */
+                        VirtualAddress = (PVOID)(KSEG0_BASE + ((BasePage + PageNumber) << MM_PAGE_SHIFT));
+                        PointerPde = MM::Paging::GetPdeAddress(VirtualAddress);
+
                         /* Initialize the PFN entry to represent an in-use page and prevent it from being allocated */
-                        Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualRangeStart);
+                        Pfn->PteAddress = MM::Paging::GetPteAddress(VirtualAddress);
                         Pfn->u2.ShareCount++;
                         Pfn->u3.e1.CacheAttribute = PfnCached;
                         Pfn->u3.e1.PageLocation = ActiveAndValid;
@@ -1213,7 +1288,7 @@ MM::Pfn::ScanMemoryDescriptors(VOID)
     if(!FreeDescriptor)
     {
         /* No free memory available to bootstrap the system */
-        KE::Crash::Panic(0);
+        KE::Crash::Panic(0x7D, LowestPhysicalPage, HighestPhysicalPage, FreePages, 0x1);
     }
 
     /* Save a copy of the original free descriptor before it gets modified */

xtoskrnl/mm/pool.cc

@@ -0,0 +1,736 @@
+/**
+ * PROJECT:         ExectOS
+ * COPYRIGHT:       See COPYING.md in the top level directory
+ * FILE:            xtoskrnl/mm/pool.cc
+ * DESCRIPTION:     Memory Manager pool manager
+ * DEVELOPERS:      Aiken Harris <harraiken91@gmail.com>
+ */
+
+#include <xtos.hh>
+
+
+/**
+ * Decodes an obfuscated doubly-linked pool list pointer.
+ *
+ * @param Link
+ *        Supplies the encoded list entry pointer to be decoded.
+ *
+ * @return This routine returns the valid, properly aligned list entry pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PLIST_ENTRY
+MM::Pool::DecodePoolLink(IN PLIST_ENTRY PoolLink)
+{
+    /* XOR the obfuscated pointer with the global pool cookie to reveal the true address */
+    return (PLIST_ENTRY)((ULONG_PTR)PoolLink ^ PoolSecureCookie);
+}
+
+/**
+ * Determines the pool type for a given memory address.
+ *
+ * @param VirtualAddress
+ *        Supplies a virtual address to determine the pool type for.
+ *
+ * @return This routine returns the determined pool type for the specified address.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+MMPOOL_TYPE
+MM::Pool::DeterminePoolType(IN PVOID VirtualAddress)
+{
+    PMMMEMORY_LAYOUT MemoryLayout;
+
+    /* Retrieve the memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Evaluate the virtual address against known pool boundaries */
+    if((VirtualAddress >= MemoryLayout->NonPagedPoolStart) &&
+       (VirtualAddress <= MemoryLayout->NonPagedPoolEnd))
+    {
+        /* Address belongs to the non-paged pool */
+        return NonPagedPool;
+    }
+    else if((VirtualAddress >= MemoryLayout->NonPagedExpansionPoolStart) &&
+            (VirtualAddress <= MemoryLayout->NonPagedExpansionPoolEnd))
+    {
+        /* Address belongs to the non-paged expansion pool */
+        return NonPagedPool;
+    }
+    else if((VirtualAddress >= MemoryLayout->PagedPoolStart) &&
+            (VirtualAddress <= MemoryLayout->PagedPoolEnd))
+    {
+        /* Address belongs to the paged pool */
+        return PagedPool;
+    }
+
+    /* Address does not belong to any known pool, kernel panic */
+    KE::Crash::Panic(0xC2, 0x42, (ULONG_PTR)VirtualAddress, 0, 0);
+
+    /* Return an invalid pool type to satisfy the compiler */
+    return (MMPOOL_TYPE)-1;
+}
+
+/**
+ * Encodes a doubly-linked pool list pointer to mitigate pool corruption.
+ *
+ * @param Link
+ *        Supplies the raw list entry pointer to be encoded.
+ *
+ * @return This routine returns the obfuscated list entry pointer.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PLIST_ENTRY
+MM::Pool::EncodePoolLink(IN PLIST_ENTRY PoolLink)
+{
+    /* XOR the raw pointer with the global pool cookie to securely obfuscate it */
+    return (PLIST_ENTRY)((ULONG_PTR)PoolLink ^ PoolSecureCookie);
+}
+
+/**
+ * Calculates the address of a pool block at a specific relative index.
+ *
+ * @param Header
+ *        Supplies a pointer to the base pool header.
+ *
+ * @param Index
+ *        Supplies the block index offset. This value can be negative to traverse backwards.
+ *
+ * @return This routine returns a pointer to the calculated pool header.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PPOOL_HEADER
+MM::Pool::GetPoolBlock(IN PPOOL_HEADER Header, IN SSIZE_T Index)
+{
+    /* The destination block is located by advancing the base address by the specified index */
+    return (PPOOL_HEADER)((ULONG_PTR)Header + (Index * MM_POOL_BLOCK_SIZE));
+}
+
+/**
+ * Retrieves the pool header associated with a given pool memory address.
+ *
+ * @param Memory
+ *        Supplies a pointer to the allocated memory region of a pool block.
+ *
+ * @return This routine returns a pointer to the originating pool header.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PPOOL_HEADER
+MM::Pool::GetPoolEntry(IN PVOID Memory)
+{
+    /* The structural header logically precedes the allocated memory region */
+    return (PPOOL_HEADER)((ULONG_PTR)Memory - sizeof(POOL_HEADER));
+}
+
+/**
+ * Resolves the list entry structure embedded within a free pool block.
+ *
+ * @param Header
+ *        Supplies a pointer to the pool header.
+ *
+ * @return This routine returns a pointer to the list entry directly following the header.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PLIST_ENTRY
+MM::Pool::GetPoolFreeBlock(IN PPOOL_HEADER Header)
+{
+    /* Return the list entry pointer */
+    return (PLIST_ENTRY)((ULONG_PTR)Header + sizeof(POOL_HEADER));
+}
+
+/**
+ * Retrieves a pointer to the adjacent contiguous pool block following the specified header.
+ *
+ * @param Header
+ *        Supplies a pointer to the current pool header.
+ *
+ * @return This routine returns a pointer to the next pool header in memory.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PPOOL_HEADER
+MM::Pool::GetPoolNextBlock(IN PPOOL_HEADER Header)
+{
+    /* The adjacent forward header is located exactly 'BlockSize' units ahead of the current block */
+    return (PPOOL_HEADER)((ULONG_PTR)Header + (Header->BlockSize * MM_POOL_BLOCK_SIZE));
+}
+
+/**
+ * Retrieves a pointer to the adjacent contiguous pool block preceding the specified header.
+ *
+ * @param Header
+ *        Supplies a pointer to the current pool header.
+ *
+ * @return This routine returns a pointer to the previous pool header in memory.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PPOOL_HEADER
+MM::Pool::GetPoolPreviousBlock(IN PPOOL_HEADER Header)
+{
+    /* The adjacent backward header is located exactly 'PreviousSize' units behind the current block */
+    return (PPOOL_HEADER)((ULONG_PTR)Header - (Header->PreviousSize * MM_POOL_BLOCK_SIZE));
+}
+
+/**
+ * Initializes the non-paged pool for memory allocator.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InitializeNonPagedPool(VOID)
+{
+    PMMFREE_POOL_ENTRY FreePage, SetupPage;
+    PMMMEMORY_LAYOUT MemoryLayout;
+    ULONG Index;
+
+    /* Retrieve memory layout */
+    MemoryLayout = MM::Manager::GetMemoryLayout();
+
+    /* Map PTEs for the non-paged pool */
+    MapNonPagedPool();
+
+    /* Iterate over the free page list heads */
+    for(Index = 0; Index < MM_MAX_FREE_PAGE_LIST_HEADS; Index++)
+    {
+        /* Initialize a free page list head */
+        RTL::LinkedList::InitializeListHead(&NonPagedPoolFreeList[Index]);
+    }
+
+    /* Take the first free page from the pool and set its size */
+    FreePage = (PMMFREE_POOL_ENTRY)MemoryLayout->NonPagedPoolStart;
+    FreePage->Size = MemoryLayout->NonPagedPoolSize;
+
+    /* Take number of pages in the pool */
+    Index = (ULONG)(MemoryLayout->NonPagedPoolSize - 1);
+    if(Index >= MM_MAX_FREE_PAGE_LIST_HEADS)
+    {
+        /* Number of pages exceeds the number of free page list heads */
+        Index = MM_MAX_FREE_PAGE_LIST_HEADS - 1;
+    }
+
+    /* Insert the first free page into the free page list */
+    RTL::LinkedList::InsertHeadList(&NonPagedPoolFreeList[Index], &FreePage->List);
+
+    /* Create a free page for each page in the pool */
+    SetupPage = FreePage;
+    for(Index = 0; Index < MemoryLayout->NonPagedPoolSize; Index++)
+    {
+        /* Initialize the owner for each page */
+        SetupPage->Owner = FreePage;
+        SetupPage = (PMMFREE_POOL_ENTRY)((ULONG_PTR)SetupPage + MM_PAGE_SIZE);
+    }
+
+    /* Store first and last allocated non-paged pool page */
+    NonPagedPoolFrameStart = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress(MemoryLayout->NonPagedPoolStart));
+    NonPagedPoolFrameEnd = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress(MemoryLayout->NonPagedPoolEnd));
+
+    /* Initialize system PTE pool for the non-paged expansion pool */
+    Pte::InitializeSystemPtePool(Paging::GetNextPte(Paging::GetPteAddress(MemoryLayout->NonPagedExpansionPoolStart)),
+                                                                          MemoryLayout->NonPagedExpansionPoolSize - 2,
+                                                                          NonPagedPoolExpansion);
+
+    /* Store non-paged pool descriptor in the pool vector and initialize it */
+    PoolVector[NonPagedPool] = &NonPagedPoolDescriptor;
+    InitializePoolDescriptor(PoolVector[NonPagedPool], NonPagedPool, 0, 0, NULLPTR);
+}
+
+/**
+ * Initializes the non-paged pool for memory allocator.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InitializePagedPool(VOID)
+{
+    UNIMPLEMENTED;
+}
+
+/**
+ * Initializes a pool descriptor used by the memory manager.
+ *
+ * @param Descriptor
+ *        Supplies a pointer to the pool descriptor structure to be initialized.
+ *
+ * @param PoolType
+ *        Specifies the type of memory pool that will be managed by the descriptor.
+ *
+ * @param Index
+ *        Supplies the zero-based index of the descriptor within the pool vector.
+ *
+ * @param Threshold
+ *        Specifies the allocation threshold that dictates when the pool should expand.
+ *
+ * @param LockAddress
+ *        Supplies a pointer to the synchronization primitive that will serialize access to this descriptor. *
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InitializePoolDescriptor(IN PPOOL_DESCRIPTOR Descriptor,
+                                   IN MMPOOL_TYPE PoolType,
+                                   IN ULONG Index,
+                                   IN ULONG Threshold,
+                                   IN PVOID LockAddress)
+{
+    PLIST_ENTRY LastEntry, ListEntry;
+
+    /* Populate the core attributes of the descriptor */
+    Descriptor->LockAddress = LockAddress;
+    Descriptor->PoolIndex = Index;
+    Descriptor->PoolType = PoolType;
+    Descriptor->Threshold = Threshold;
+
+    /* Clear the deferred free list */
+    Descriptor->PendingFrees = NULLPTR;
+    Descriptor->PendingFreeDepth = 0;
+
+    /* Zero out the runtime accounting and statistical tracking counters */
+    Descriptor->RunningAllocations = 0;
+    Descriptor->RunningFrees = 0;
+    Descriptor->TotalBigAllocations = 0;
+    Descriptor->TotalBytes = 0;
+    Descriptor->TotalPages = 0;
+
+    /* Establish the iteration boundaries */
+    ListEntry = Descriptor->ListHeads;
+    LastEntry = ListEntry + MM_POOL_LISTS_PER_PAGE;
+
+    /* Traverse and initialize all block list heads */
+    while(ListEntry < LastEntry)
+    {
+        /* Initialize the empty list head */
+        InitializePoolListHead(ListEntry);
+        ListEntry++;
+    }
+}
+
+/**
+ * Initializes a doubly-linked pool list head with encoded pointers.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the pool list head that is to be initialized.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InitializePoolListHead(IN PLIST_ENTRY ListHead)
+{
+    PLIST_ENTRY ListEntry;
+
+    /* Obfuscate the list head address and establish the empty circular linkage */
+    ListEntry = EncodePoolLink(ListHead);
+    ListHead->Flink = ListEntry;
+    ListHead->Blink = ListEntry;
+}
+
+/**
+ * Initializes the memory pool security mechanisms.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InitializePoolSecurity(VOID)
+{
+    UNIMPLEMENTED;
+
+    /* Initialize the global pool cookie using a hard-coded value */
+    PoolSecureCookie = 0xDEADC0DE;
+}
+
+/**
+ * Inserts a pool entry at the head of a doubly-linked pool list.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the head of the pool list.
+ *
+ * @param Entry
+ *        Supplies a pointer to the pool list entry to be inserted.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InsertPoolHeadList(IN PLIST_ENTRY ListHead,
+                             IN PLIST_ENTRY Entry)
+{
+    PLIST_ENTRY Flink;
+
+    /* Validate the pool list structure */
+    VerifyPoolLinks(ListHead);
+
+    /* Resolve the current forward link of the list head */
+    Flink = DecodePoolLink(ListHead->Flink);
+
+    /* Securely insert the new entry at the beginning of the pool list */
+    Entry->Blink = EncodePoolLink(ListHead);
+    Entry->Flink = EncodePoolLink(Flink);
+    Flink->Blink = EncodePoolLink(Entry);
+    ListHead->Flink = EncodePoolLink(Entry);
+
+    /* Re-validate the pool list structure */
+    VerifyPoolLinks(ListHead);
+}
+
+/**
+ * Inserts a pool entry at the tail of a doubly-linked pool list.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the head of the pool list.
+ *
+ * @param Entry
+ *        Supplies a pointer to the pool list entry to be inserted.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::InsertPoolTailList(IN PLIST_ENTRY ListHead,
+                             IN PLIST_ENTRY Entry)
+{
+    PLIST_ENTRY Blink;
+
+    /* Validate the pool list structure */
+    VerifyPoolLinks(ListHead);
+
+    /* Securely link the new entry at the end of the pool list */
+    Blink = DecodePoolLink(ListHead->Blink);
+    Blink->Flink = EncodePoolLink(Entry);
+    Entry->Blink = EncodePoolLink(Blink);
+    Entry->Flink = EncodePoolLink(ListHead);
+    ListHead->Blink = EncodePoolLink(Entry);
+
+    /* Re-validate the pool list structure */
+    VerifyPoolLinks(ListHead);
+}
+
+/**
+ * Determines whether a given doubly-linked pool list is empty.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the head of the pool list to be evaluated.
+ *
+ * @return This routine returns TRUE if the pool list is empty, or FALSE otherwise.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+BOOLEAN
+MM::Pool::PoolListEmpty(IN PLIST_ENTRY ListHead)
+{
+    /* Evaluate whether the pool list contains no valid entries */
+    return (DecodePoolLink(ListHead->Flink) == ListHead);
+}
+
+/**
+ * Removes a specific pool entry from a doubly-linked pool list.
+ *
+ * @param Entry
+ *        Supplies a pointer to the pool list entry to be removed.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::RemovePoolEntryList(IN PLIST_ENTRY Entry)
+{
+    PLIST_ENTRY Blink, Flink;
+
+    /* Resolve the adjacent forward and backward links */
+    Blink = DecodePoolLink(Entry->Blink);
+    Flink = DecodePoolLink(Entry->Flink);
+
+    /* Securely link the adjacent nodes together */
+    Blink->Flink = EncodePoolLink(Flink);
+    Flink->Blink = EncodePoolLink(Blink);
+}
+
+/**
+ * Removes the first entry from a doubly-linked pool list.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the head of the pool list.
+ *
+ * @return This routine returns a pointer to the removed pool list entry.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+PLIST_ENTRY
+MM::Pool::RemovePoolHeadList(IN PLIST_ENTRY ListHead)
+{
+    PLIST_ENTRY Entry, Flink;
+
+    /* Securely unlink the first entry from the pool list */
+    Entry = DecodePoolLink(ListHead->Flink);
+    Flink = DecodePoolLink(Entry->Flink);
+    Flink->Blink = EncodePoolLink(ListHead);
+    ListHead->Flink = EncodePoolLink(Flink);
+
+    /* Return the removed pool list entry */
+    return Entry;
+}
+
+/**
+ * Removes the last entry from a doubly-linked pool list.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the head of the pool list.
+ *
+ * @return This routine returns a pointer to the removed pool list entry.
+ *
+ * @since XT 1.0
+ */
+PLIST_ENTRY
+XTAPI
+MM::Pool::RemovePoolTailList(IN PLIST_ENTRY ListHead)
+{
+    PLIST_ENTRY Blink, Entry;
+
+    /* Securely unlink the last entry from the pool list */
+    Entry = DecodePoolLink(ListHead->Blink);
+    Blink = DecodePoolLink(Entry->Blink);
+    Blink->Flink = EncodePoolLink(ListHead);
+    ListHead->Blink = EncodePoolLink(Blink);
+
+    /* Return the removed pool list entry */
+    return Entry;
+}
+
+/**
+ * Verifies the structural integrity of all pool blocks residing on a specific page.
+ *
+ * @param Block
+ *        Supplies a pointer to the specific pool block.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::VerifyPoolBlocks(IN PVOID Block)
+{
+    PPOOL_HEADER Entry;
+    BOOLEAN FoundBlock;
+    SIZE_T Size;
+
+    /* Initialize tracking variables */
+    FoundBlock = FALSE;
+    Size = 0;
+
+    /* Resolve the first pool header */
+    Entry = (PPOOL_HEADER)PAGE_ALIGN(Block);
+
+    /* Iterate through all contiguous pool allocations */
+    do
+    {
+        /* Validate the current pool header */
+        VerifyPoolHeader(Entry);
+
+        /* Check if the current header corresponds to the target block */
+        if(Entry == Block)
+        {
+            /* Mark the block as found */
+            FoundBlock = TRUE;
+        }
+
+        /* Accumulate the total block size and advance to the next entry */
+        Size += Entry->BlockSize;
+        Entry = GetPoolNextBlock(Entry);
+    }
+    while((Size < (MM_PAGE_SIZE / MM_POOL_BLOCK_SIZE)) && (PAGE_ALIGN(Entry) != Entry));
+
+    /* Ensure the block was found and the total size is aligned with the page */
+    if(!FoundBlock || (PAGE_ALIGN(Entry) != Entry) || (Size != (MM_PAGE_SIZE / MM_POOL_BLOCK_SIZE)))
+    {
+        /* Pool blocks corruption detected, kernel panic */
+        KE::Crash::Panic(0x19, 10, (ULONG_PTR)Block, (ULONG_PTR)Entry, FoundBlock);
+    }
+}
+
+/**
+ * Verifies the structural and spatial invariants of a specific pool header.
+ *
+ * @param Entry
+ *        Supplies a pointer to the pool header to be verified.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::VerifyPoolHeader(IN PPOOL_HEADER Entry)
+{
+    PPOOL_HEADER PreviousEntry, NextEntry;
+
+    /* Verify that the current block header is valid */
+    if(!Entry->BlockSize)
+    {
+        /* Invalid block header size, kernel panic */
+        KE::Crash::Panic(0x19, 8, (ULONG_PTR)Entry->PreviousSize, Entry->BlockSize, (ULONG_PTR)Entry);
+    }
+
+    /* Verify that the previous block header is valid */
+    if(Entry->PreviousSize)
+    {
+        /* Resolve the previous block header */
+        PreviousEntry = GetPoolPreviousBlock(Entry);
+
+        /* Check if both adjacent blocks are within the same memory page */
+        if(PAGE_ALIGN(Entry) != PAGE_ALIGN(PreviousEntry))
+        {
+            /* Adjacent blocks are not on the same page, kernel panic */
+            KE::Crash::Panic(0x19, 6, (ULONG_PTR)PreviousEntry, (ULONG_PTR)PAGE_ALIGN(Entry), (ULONG_PTR)Entry);
+        }
+
+        /* Check the actual size of the previous block */
+        if(PreviousEntry->BlockSize != Entry->PreviousSize)
+        {
+            /* Block size mismatch, kernel panic */
+            KE::Crash::Panic(0x19, 5, (ULONG_PTR)PreviousEntry, (ULONG_PTR)Entry->PreviousSize, (ULONG_PTR)Entry);
+        }
+    }
+    else if(PAGE_ALIGN(Entry) != Entry)
+    {
+        /* Not aligned to a page boundary, kernel panic */
+        KE::Crash::Panic(0x19, 7, 0, (ULONG_PTR)PAGE_ALIGN(Entry), (ULONG_PTR)Entry);
+    }
+
+    /* Resolve the next block header */
+    NextEntry = GetPoolNextBlock(Entry);
+
+    /* Verify the next block header */
+    if(PAGE_ALIGN(NextEntry) != NextEntry)
+    {
+        /* Check if both adjacent blocks are within the same memory page */
+        if(PAGE_ALIGN(Entry) != PAGE_ALIGN(NextEntry))
+        {
+            /* Adjacent blocks are not on the same page, kernel panic */
+            KE::Crash::Panic(0x19, 9, (ULONG_PTR)NextEntry, (ULONG_PTR)PAGE_ALIGN(Entry), (ULONG_PTR)Entry);
+        }
+
+        /* Check the previous block size */
+        if(NextEntry->PreviousSize != Entry->BlockSize)
+        {
+            /* Block size mismatch, kernel panic */
+            KE::Crash::Panic(0x19, 5, (ULONG_PTR)NextEntry, NextEntry->PreviousSize, (ULONG_PTR)Entry);
+        }
+    }
+}
+
+/**
+ * Validates the structural integrity of a doubly-linked pool list.
+ *
+ * @param ListHead
+ *        Supplies a pointer to the pool list head that is to be validated.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::VerifyPoolLinks(IN PLIST_ENTRY ListHead)
+{
+    /* Validate the doubly-linked list invariants */
+    if((DecodePoolLink(DecodePoolLink(ListHead->Blink)->Flink) != ListHead) ||
+       (DecodePoolLink(DecodePoolLink(ListHead->Flink)->Blink) != ListHead))
+    {
+        /* Pool corruption detected, raise kernel panic */
+        KE::Crash::Panic(0x19,
+                         3,
+                         (ULONG_PTR)ListHead,
+                         (ULONG_PTR)DecodePoolLink(DecodePoolLink(ListHead->Blink)->Flink),
+                         (ULONG_PTR)DecodePoolLink(DecodePoolLink(ListHead->Flink)->Blink));
+    }
+}
+
+/**
+ * Validates the run level for the specified pool. If the run level is invalid, the kernel panics.
+ *
+ * @param PoolType
+ *        Supplies the pool type.
+ *
+ * @param Bytes
+ *        Supplies the size of the allocation.
+ *
+ * @param Entry
+ *        Supplies a pointer to the allocation entry.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Pool::VerifyRunLevel(IN MMPOOL_TYPE PoolType,
+                         IN SIZE_T Bytes,
+                         IN PVOID Entry)
+{
+    KRUNLEVEL RunLevel;
+
+    /* Get current run level */
+    RunLevel = KE::RunLevel::GetCurrentRunLevel();
+
+    /* Validate run level */
+    if((PoolType & MM_POOL_TYPE_MASK) == PagedPool)
+    {
+        /* Paged pool runs up to APC level */
+        if(RunLevel <= APC_LEVEL)
+        {
+            /* Run level is valid */
+            return;
+        }
+    }
+    else
+    {
+        /* Non-paged pool runs up to DISPATCH_LEVEL */
+        if(RunLevel <= DISPATCH_LEVEL)
+        {
+            /* Run level is valid */
+            return;
+        }
+    }
+
+    /* Invalid run level for specified pool, raise kernel panic */
+    KE::Crash::Panic(0xC2,
+                     (Entry ? MM_POOL_INVALID_FREE_RUNLEVEL : MM_POOL_INVALID_ALLOC_RUNLEVEL),
+                     RunLevel,
+                     PoolType,
+                     (Entry ? (ULONG_PTR)Entry : Bytes));
+}

xtoskrnl/rtl/amd64/exsup.cc

@@ -74,7 +74,7 @@ __CxxFrameHandler3(IN PEXCEPTION_RECORD ExceptionRecord,
 
     /* Disable interrupts and hang */
     AR::CpuFunc::ClearInterruptFlag();
-    KE::Crash::Panic(0); // CXX_FRAME_HANDLER_CALLED
+    KE::Crash::Panic(0);
 
     /* Continue search */
     return ExceptionContinueSearch;
@@ -129,5 +129,5 @@ _purecall(VOID)
 
     /* Disable interrupts and hang */
     AR::CpuFunc::ClearInterruptFlag();
-    KE::Crash::Panic(0); // PURE_VIRTUAL_FUNCTION_CALL
+    KE::Crash::Panic(0);
 }

xtoskrnl/rtl/i686/exsup.cc

@@ -74,7 +74,7 @@ __CxxFrameHandler3(IN PEXCEPTION_RECORD ExceptionRecord,
 
     /* Disable interrupts and hang */
     AR::CpuFunc::ClearInterruptFlag();
-    KE::Crash::Panic(0); // CXX_FRAME_HANDLER_CALLED
+    KE::Crash::Panic(0);
 
     /* Continue search */
     return ExceptionContinueSearch;
@@ -129,5 +129,5 @@ _purecall(VOID)
 
     /* Disable interrupts and hang */
     AR::CpuFunc::ClearInterruptFlag();
-    KE::Crash::Panic(0); // PURE_VIRTUAL_FUNCTION_CALL
+    KE::Crash::Panic(0);
 }

xtoskrnl/xtoskrnl.spec

@@ -42,6 +42,10 @@
 @ stdcall KeSetTimer(ptr long long long ptr)
 @ stdcall KeSignalCallDpcDone(ptr)
 @ stdcall KeSignalCallDpcSynchronize(ptr)
+@ stdcall MmAllocatePool(long long ptr)
+@ stdcall MmAllocatePoolWithTag(long long ptr long)
+@ stdcall MmFreePool(ptr)
+@ stdcall MmFreePoolWithTag(ptr long)
 @ stdcall RtlClearAllBits(ptr)
 @ stdcall RtlClearBit(ptr long)
 @ stdcall RtlClearBits(ptr long long)