From 4a00179af2328138f0b641f136eff014e4638ae7 Mon Sep 17 00:00:00 2001 From: Aiken Harris Date: Mon, 29 Dec 2025 18:31:16 +0100 Subject: [PATCH] Implement logic to link physical pages to PTEs --- xtoskrnl/includes/mm/pfn.hh | 25 +++- xtoskrnl/mm/data.cc | 29 ++++- xtoskrnl/mm/pfn.cc | 247 +++++++++++++++++++++++++++++++++++- 3 files changed, 286 insertions(+), 15 deletions(-) diff --git a/xtoskrnl/includes/mm/pfn.hh b/xtoskrnl/includes/mm/pfn.hh index 1832a78..cd4386e 100644 --- a/xtoskrnl/includes/mm/pfn.hh +++ b/xtoskrnl/includes/mm/pfn.hh @@ -19,22 +19,33 @@ namespace MM { private: STATIC PFN_NUMBER AvailablePages; + STATIC MMPFNLIST BadPagesList; STATIC PLOADER_MEMORY_DESCRIPTOR FreeDescriptor; STATIC MMPFNLIST FreePagesList; STATIC ULONG_PTR HighestPhysicalPage; + STATIC PVOID HighestUserAddress; STATIC ULONG_PTR LowestPhysicalPage; + STATIC MMPFNLIST ModifiedPagesList; + STATIC MMPFNLIST ModifiedReadOnlyPagesList; STATIC ULONGLONG NumberOfPhysicalPages; STATIC LOADER_MEMORY_DESCRIPTOR OriginalFreeDescriptor; + STATIC PMMPFNLIST PageLocationList[]; STATIC PFN_NUMBER PfnDatabaseSize; + STATIC MMPFNLIST StandbyPagesList; + STATIC MMPFNLIST ZeroedPagesList; public: STATIC XTAPI PFN_NUMBER AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages); + STATIC XTAPI PFN_NUMBER AllocatePhysicalPage(IN ULONG Color); STATIC XTAPI VOID ComputePfnDatabaseSize(VOID); STATIC XTAPI ULONG_PTR GetHighestPhysicalPage(VOID); STATIC XTAPI ULONGLONG GetNumberOfPhysicalPages(VOID); STATIC XTAPI PFN_NUMBER GetPfnDatabaseSize(VOID); STATIC XTAPI PMMPFN GetPfnEntry(IN PFN_NUMBER Pfn); STATIC XTAPI VOID InitializePfnDatabase(VOID); + STATIC VOID XTAPI LinkPfnToPte(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN BOOLEAN Modified); STATIC XTAPI VOID ScanMemoryDescriptors(VOID); private: @@ -42,13 +53,15 @@ namespace MM STATIC XTAPI VOID IncrementAvailablePages(VOID); STATIC XTAPI VOID InitializePageTablePfns(VOID); STATIC XTAPI VOID LinkFreePage(IN PFN_NUMBER PageFrameIndex); - STATIC XTAPI VOID LinkPfnForPageTable(PFN_NUMBER PageFrameIndex, - PMMPTE PointerPte); - STATIC XTAPI VOID ProcessMemoryDescriptor(PFN_NUMBER BasePage, - PFN_NUMBER PageCount, - LOADER_MEMORY_TYPE MemoryType); + STATIC XTAPI VOID LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte); + STATIC XTAPI VOID ProcessMemoryDescriptor(IN PFN_NUMBER BasePage, + IN PFN_NUMBER PageCount, + IN LOADER_MEMORY_TYPE MemoryType); STATIC XTAPI VOID ScanPageTable(IN PMMPTE PointerPte, - ULONG Level); + IN ULONG Level); + STATIC XTAPI PFN_NUMBER UnlinkFreePage(IN PFN_NUMBER PageFrameIndex, + IN ULONG Color); }; } diff --git a/xtoskrnl/mm/data.cc b/xtoskrnl/mm/data.cc index 886a78a..d775ba4 100644 --- a/xtoskrnl/mm/data.cc +++ b/xtoskrnl/mm/data.cc @@ -30,10 +30,10 @@ ULONG MM::HardwarePool::UsedHardwareAllocationDescriptors = 0; /* Processor structures data (THIS IS A TEMPORARY HACK) */ UCHAR MM::KernelPool::ProcessorStructuresData[MAXIMUM_PROCESSORS][KPROCESSOR_STRUCTURES_SIZE] = {{0}}; -/* Memory layout */ +/* Global structure describing the virtual memory layout of the system */ MMMEMORY_LAYOUT MM::Manager::MemoryLayout; -/* Number of system PTEs */ +/* Total number of PTEs reserved for system space mapping */ PFN_NUMBER MM::Manager::NumberOfSystemPtes; /* Instance of the page map routines for the current PML level */ @@ -42,6 +42,9 @@ MM::PPAGEMAP MM::Paging::PmlRoutines; /* Total number of physical pages available for allocation */ PFN_NUMBER MM::Pfn::AvailablePages; +/* Head of the list containing physical pages marked as defective */ +MMPFNLIST MM::Pfn::BadPagesList = {0, BadPageList, MAXULONG_PTR, MAXULONG_PTR}; + /* Biggest free memory descriptor */ PLOADER_MEMORY_DESCRIPTOR MM::Pfn::FreeDescriptor; @@ -54,15 +57,37 @@ ULONG_PTR MM::Pfn::HighestPhysicalPage; /* Lowest physical page number */ ULONG_PTR MM::Pfn::LowestPhysicalPage = -1; +/* List containing modified pages */ +MMPFNLIST MM::Pfn::ModifiedPagesList = {0, ModifiedPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* List containing modified pages mapped as read-only */ +MMPFNLIST MM::Pfn::ModifiedReadOnlyPagesList = {0, ModifiedReadOnlyPageList, MAXULONG_PTR, MAXULONG_PTR}; + /* Number of physical pages */ ULONGLONG MM::Pfn::NumberOfPhysicalPages; /* Old biggest free memory descriptor */ LOADER_MEMORY_DESCRIPTOR MM::Pfn::OriginalFreeDescriptor; +/* List containing standby pages (clean, can be reclaimed or repurposed) */ +MMPFNLIST MM::Pfn::StandbyPagesList = {0, StandbyPageList, MAXULONG_PTR, MAXULONG_PTR}; + +/* Array of pointers to PFN lists */ +PMMPFNLIST MM::Pfn::PageLocationList[] = {&ZeroedPagesList, + &FreePagesList, + &StandbyPagesList, + &ModifiedPagesList, + &ModifiedReadOnlyPagesList, + &BadPagesList, + NULLPTR, + NULLPTR}; + /* Size of the PFN database in pages */ PFN_NUMBER MM::Pfn::PfnDatabaseSize; +/* List containing free physical pages that have been zeroed out */ +MMPFNLIST MM::Pfn::ZeroedPagesList = {0, ZeroedPageList, MAXULONG_PTR, MAXULONG_PTR}; + /* Array of lists for available System PTEs, separated by pool type */ MMPTE MM::Pte::FirstSystemFreePte[MaximumPtePoolTypes]; diff --git a/xtoskrnl/mm/pfn.cc b/xtoskrnl/mm/pfn.cc index 9138ac1..93379f7 100644 --- a/xtoskrnl/mm/pfn.cc +++ b/xtoskrnl/mm/pfn.cc @@ -43,6 +43,59 @@ MM::Pfn::AllocateBootstrapPages(IN PFN_NUMBER NumberOfPages) return Pfn; } +/** + * Allocates a physical page frame (PFN) from one of the system's free page lists. + * + * @param Color + * The preferred page color, used to optimize CPU cache alignment and reduce cache contention. + * + * @return This routine returns the Page Frame Number (PFN) of the allocated page. + * + * @since XT 1.0 + */ +XTAPI +PFN_NUMBER +MM::Pfn::AllocatePhysicalPage(IN ULONG Color) +{ + PFN_NUMBER PageNumber; + ULONG PagingColorsMask; + + /* Check if any physical pages are available in the system */ + if(!AvailablePages) + { + /* No physical pages are available in the system, return 0 */ + return 0; + } + + /* Retrieve the bitmask used for calculating a page's color */ + PagingColorsMask = MM::Colors::GetPagingColorsMask(); + + /* Attempt to retrieve a page from the colored free page list */ + PageNumber = MM::Colors::GetFreePages(FreePageList, Color)->Flink; + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the colored free page list, check the colored zero page list */ + PageNumber = MM::Colors::GetFreePages(ZeroedPageList, Color)->Flink; + } + + /* Attempt to retrieve a page from the colored zero list */ + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the colored zero page list, check the global free page list */ + PageNumber = FreePagesList.Flink; + } + + /* Attempt to retrieve a page from the global free page list */ + if(PageNumber == MAXULONG_PTR) + { + /* No page was found in the global free page list, check the global zero page list */ + PageNumber = ZeroedPagesList.Flink; + } + + /* Remove the page from its list and return its PFN */ + return UnlinkFreePage(PageNumber, PageNumber & PagingColorsMask); +} + /** * Calculates the total number of pages required for the PFN database and its associated color tables. * @@ -85,7 +138,7 @@ MM::Pfn::DecrementAvailablePages(VOID) */ XTAPI ULONG_PTR -MM::Pfn::GetHighestPhysicalPage() +MM::Pfn::GetHighestPhysicalPage(VOID) { /* Return the highest physical page number */ return HighestPhysicalPage; @@ -288,8 +341,8 @@ MM::Pfn::LinkFreePage(IN PFN_NUMBER PageFrameIndex) */ XTAPI VOID -MM::Pfn::LinkPfnForPageTable(PFN_NUMBER PageFrameIndex, - PMMPTE PointerPte) +MM::Pfn::LinkPfnForPageTable(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte) { PMMPFN Pfn; PMMPDE PointerPde; @@ -306,12 +359,12 @@ MM::Pfn::LinkPfnForPageTable(PFN_NUMBER PageFrameIndex, (MM::Pte::AddressValid(EndAddress)) && (Pfn->u3.e1.PageLocation == ActiveAndValid)) { /* Initialize the PFN entry for this page table page */ + Pfn->OriginalPte = *PointerPte; + Pfn->PteAddress = PointerPte; Pfn->u1.WsIndex = 0; Pfn->u2.ShareCount++; - Pfn->PteAddress = PointerPte; - Pfn->OriginalPte = *PointerPte; - Pfn->u3.e1.PageLocation = ActiveAndValid; Pfn->u3.e1.CacheAttribute = PfnNonCached; + Pfn->u3.e1.PageLocation = ActiveAndValid; Pfn->u3.e2.ReferenceCount = 1; Pfn->u4.PteFrame = MM::Paging::GetPageFrameNumber(MM::Paging::GetPteAddress(PointerPte)); } @@ -322,6 +375,84 @@ MM::Pfn::LinkPfnForPageTable(PFN_NUMBER PageFrameIndex, Pfn->u2.ShareCount++; } +/** + * Links a PFN entry to its corresponding PTE and ensures the page table that contains the PTE is resident in memory. + * + * @param PageFrameIndex + * Supplies the index into the PFN database for the page being initialized. + * + * @param PointerPte + * Supplies the pointer to the PTE which maps the physical page. + * + * @param Modified + * Supplies a flag indicating if the page's initial state is modified. + * + * @return This routine does not return any value. + * + * @since XT 1.0 + */ +XTAPI +VOID +MM::Pfn::LinkPfnToPte(IN PFN_NUMBER PageFrameIndex, + IN PMMPTE PointerPte, + IN BOOLEAN Modified) +{ + PMMMEMORY_LAYOUT MemoryLayout; + XTSTATUS Status; + PMMPFN Pfn; + PMMPTE Pte; + + /* Get the memory layout */ + MemoryLayout = MM::Manager::GetMemoryLayout(); + + /* Point the PFN to its PTE */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabaseAddress)[PageFrameIndex]; + Pfn->PteAddress = PointerPte; + + /* Check if the page is already mapped and in use */ + if(MM::Paging::PteValid(PointerPte)) + { + /* Clear the original PTE information */ + MM::Paging::SetPte(&Pfn->OriginalPte, 0, MM_PTE_READWRITE | MM_PTE_CACHE_ENABLE); + } + else + { + /* Page is not resident, so save the PTE contents for later use */ + Pfn->OriginalPte = *PointerPte; + } + + /* Initialize the PFN database entry for this page */ + Pfn->u2.ShareCount = 1; + Pfn->u3.e1.Modified = Modified; + Pfn->u3.e1.PageLocation = ActiveAndValid; + Pfn->u3.e2.ReferenceCount = 1; + + /* Get the PDE that maps the page table containing this PTE */ + Pte = MM::Paging::GetPteAddress(PointerPte); + if(!MM::Paging::PteValid(Pte)) + { + /* Check if page table is resident */ + Status = MM::PageFault::CheckPdeForPagedPool(PointerPte); + if(Status != STATUS_SUCCESS) + { + /* Could not make the page table resident, crash system */ + KE::Crash::PanicEx(0x1, + (ULONG_PTR)0x61940, + (ULONG_PTR)PointerPte, + MM::Paging::GetPageFrameNumber(PointerPte), + (ULONG_PTR)MM::Paging::GetPteVirtualAddress(PointerPte)); + } + } + + /* Record the page frame of the page table itself */ + PageFrameIndex = MM::Paging::GetPageFrameNumber(Pte); + Pfn->u4.PteFrame = PageFrameIndex; + + /* Pin the page table in memory by incrementing its PFN share count */ + Pfn = &((PMMPFN)MemoryLayout->PfnDatabaseAddress)[PageFrameIndex]; + Pfn->u2.ShareCount++; +} + /** * Processes a memory descriptor and initializes the corresponding PFN database entries * @@ -385,8 +516,8 @@ MM::Pfn::ProcessMemoryDescriptor(IN PFN_NUMBER BasePage, Pfn->u3.e1.CacheAttribute = PfnNonCached; Pfn->u3.e1.PageLocation = 0; Pfn->u3.e1.PrototypePte = 1; - Pfn->u3.e2.ReferenceCount = 0; Pfn->u3.e1.Rom = 1; + Pfn->u3.e2.ReferenceCount = 0; Pfn->u4.InPageError = 0; Pfn->u4.PteFrame = 0; } @@ -490,3 +621,105 @@ MM::Pfn::ScanMemoryDescriptors(VOID) /* Save a copy of the original free descriptor before it gets modified */ RTL::Memory::CopyMemory(&OriginalFreeDescriptor, FreeDescriptor, sizeof(LOADER_MEMORY_DESCRIPTOR)); } + +/** + * Unlinks a physical page from its corresponding list. + * + * @param PageIndex + * The Page Frame Number (PFN) of the page to unlink. + * + * @param Color + * The color of the page, used to find the correct colored list. + * + * @return This routine returns the PFN of the page that was unlinked. + */ +XTAPI +PFN_NUMBER +MM::Pfn::UnlinkFreePage(IN PFN_NUMBER PageFrameIndex, + IN ULONG Color) +{ + PMMPFN Pfn; + PMMPFNLIST PfnList; + MMPAGELISTS PageList; + ULONG NodeColor; + PMMCOLOR_TABLES ColorTable; + PFN_NUMBER NextPage, PrevPage; + + /* Get the PFN database entry for the target page */ + Pfn = GetPfnEntry(PageFrameIndex); + + /* Identify which list the page belongs to (FreePageList or ZeroedPageList) */ + PfnList = PageLocationList[Pfn->u3.e1.PageLocation]; + PageList = PfnList->ListName; + + /* Update the forward link of the previous page */ + if(Pfn->u2.Blink != MAXULONG_PTR) + { + /* The page is not the head of the list; update the previous page's Flink */ + GetPfnEntry(Pfn->u2.Blink)->u1.Flink = Pfn->u1.Flink; + } + else + { + /* This is the first page in the list; update the list head's Flink */ + PfnList->Flink = Pfn->u1.Flink; + } + + /* Update the backward link of the next page */ + if(Pfn->u1.Flink != MAXULONG_PTR) + { + /* The page is not the tail of the list; update the next page's Blink */ + GetPfnEntry(Pfn->u1.Flink)->u2.Blink = Pfn->u2.Blink; + } + else + { + /* This is the last page in the list; update the list head's Blink */ + PfnList->Blink = Pfn->u2.Blink; + } + + /* Get the first page on the color list */ + ColorTable = MM::Colors::GetFreePages(PageList, Color); + NodeColor = Pfn->u3.e1.PageColor; + PrevPage = Pfn->u4.PteFrame; + NextPage = MM::Paging::GetPte(&Pfn->OriginalPte); + + /* Decrement the count of pages for this specific color and total page count for this list */ + ColorTable->Count--; + PfnList->Total--; + + /* Update the forward link of the previous colored page */ + if(PrevPage != MM_PFN_PTE_FRAME) + { + /* This is not the first page; update the previous page's Flink */ + MM::Paging::SetPte(&GetPfnEntry(PrevPage)->OriginalPte, NextPage); + } + else + { + /* This was the first page; update the color table's Flink */ + ColorTable->Flink = NextPage; + } + + /* Update the backward link of the next colored page */ + if (NextPage != MAXULONG_PTR) + { + /* This is not the last page; update the next page's Blink */ + GetPfnEntry(NextPage)->u4.PteFrame = PrevPage; + } + else + { + /* This was the last page; update the color table's Blink */ + ColorTable->Blink = (PVOID)PrevPage; + } + + /* Clear the list pointers and flags, but preserve the color and cache attributes */ + Pfn->u1.Flink = 0; + Pfn->u2.Blink = 0; + Pfn->u3.e1.CacheAttribute = PfnNotMapped; + Pfn->u3.e1.PageColor = NodeColor; + Pfn->u3.e2.ShortFlags = 0; + + /* Decrement the global count of available pages */ + DecrementAvailablePages(); + + /* Return the page that was just unlinked */ + return PageFrameIndex; +}