Overhaul kernel memory layout initialization and pool sizing

xtoskrnl/includes/mm/mmgr.hh

@@ -32,6 +32,12 @@ namespace MM
 
         private:
             STATIC XTAPI VOID ComputeBootImageSize(OUT PPFN_NUMBER BootImageSize);
+            STATIC XTAPI VOID ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize);
+            STATIC XTAPI VOID ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize);
+            STATIC XTAPI VOID ComputeSystemPteSize(OUT PPFN_NUMBER PteSize);
+            STATIC XTAPI VOID DumpMemoryLayout(VOID);
     };
 }
 

xtoskrnl/mm/amd64/mmgr.cc

@@ -9,6 +9,249 @@
 #include <xtos.hh>
 
 
+/**
+ * Calculates the maximum possible size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR MaximumNonPagedPoolSize;
+
+    /* Start with the 1MiB and add 400KiB for each MiB above 16MiB */
+    MaximumNonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 409600);
+
+    /* Check if non-paged pool does not exceed 128GiB */
+    if(MaximumNonPagedPoolSize > 137438953472)
+    {
+        /* Limit non-paged pool size to 128GiB */
+        MaximumNonPagedPoolSize = 137438953472;
+    }
+
+    /* Return non-paged pool size */
+    *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize);
+}
+
+/**
+ * Calculates the size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR NonPagedPoolSize;
+
+    /* Start with 1MiB and add 32KiB for each MiB above 16MiB */
+    NonPagedPoolSize = 1048576 + (((MM::Pfn::GetNumberOfPhysicalPages() - 4096) / 256) * 32768);
+
+    /* Check if non-paged pool does not exceed 128GiB */
+    if(NonPagedPoolSize > 137438953472)
+    {
+        /* Limit non-paged pool size to 128GiB */
+        NonPagedPoolSize = 137438953472;
+    }
+
+    /* Return non-paged pool size in pages, aligned up to page size boundary */
+    *PoolSize = SIZE_TO_PAGES(ROUND_UP(NonPagedPoolSize, MM_PAGE_SIZE));
+}
+
+/**
+ * Calculates the size of the paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONGLONG PagedPoolSize, PteCount;
+    ULONG PtesPerPage;
+
+    /* Start with 4x maximum non-paged pool size and at least 48MiB */
+    ComputeMaximumNonPagedPoolSize(&PagedPoolSize);
+    PagedPoolSize *= 4 * MM_PAGE_SIZE;
+
+    /* Ensure that paged pool size is at least 48MiB */
+    if(PagedPoolSize < 50331648)
+    {
+        /* Increase paged pool size to at least 48MiB */
+        PagedPoolSize = 50331648;
+    }
+
+    /* Check if paged pool does not overlap non-paged pool */
+    if(PagedPoolSize > (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart)
+    {
+        /* Limit paged pool size to maximum possible */
+        PagedPoolSize = (ULONGLONG)MemoryLayout.NonPagedSystemPoolStart - (ULONGLONG)MemoryLayout.PagedPoolStart;
+    }
+
+    /* Check if paged pool does not exceed 128GiB */
+    if(PagedPoolSize > 137438953472)
+    {
+        /* Limit paged pool size to 128GiB */
+        PagedPoolSize = 137438953472;
+    }
+
+    /* Get the number of PTEs per page and calculate size of paged pool */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+    PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage);
+
+    /* Return paged pool size */
+    *PoolSize = PteCount * PtesPerPage;
+}
+
+/**
+ * Calculates the size of the session space.
+ *
+ * @param SpaceSize
+ *        A pointer to a variable that will receive the number of pages available by the session space.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize)
+{
+    PFN_NUMBER SessionSpaceSize;
+
+    /* Session Pool, Session View, Session Image, Session Working Set and System View takes 1120MiB */
+    SessionSpaceSize = 1174405120;
+
+    /* Return number of pages used by the session space */
+    *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE;
+}
+
+/**
+ * Calculates the size of the system PTEs.
+ *
+ * @param PteSize
+ *        A pointer to a variable that will receive the number of system PTEs.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize)
+{
+    PFN_NUMBER SystemPteSize;
+
+    /* Check if system has less than 24MiB of physical memory */
+    if(MM::Pfn::GetNumberOfPhysicalPages() < 6144)
+    {
+        /* Set minimal system PTE size */
+        SystemPteSize = 7000;
+    }
+    else
+    {
+        /* Use standard system PTE size */
+        SystemPteSize = 11000;
+
+        /* Check if system has more than 32MiB of physical memory */
+        if(MM::Pfn::GetNumberOfPhysicalPages() > 8192)
+        {
+            /* Double system PTE size */
+            SystemPteSize *= 2;
+
+            /* Check if system has more than 256MiB of physical memory */
+            if(MM::Pfn::GetNumberOfPhysicalPages() > 65536)
+            {
+                /* Double system PTE size */
+                SystemPteSize *= 2;
+            }
+        }
+    }
+
+    /* Return system PTE size */
+    *PteSize = SystemPteSize;
+}
+
+/**
+ * Dumps the kernel's memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::DumpMemoryLayout(VOID)
+{
+    /* Dump memory layout */
+    DebugPrint(L"System with %zu MiB of installed memory:\n"
+               L"User Space:               %.16P - %.16P\n"
+               L"Non-Canonical:            %.16P - %.16P\n"
+               L"Reserved System Pool:     %.16P - %.16P\n"
+               L"PTE Space:                %.16P - %.16P\n"
+               L"Hyper Space:              %.16P - %.16P\n"
+               L"Shared System Page:       %.16P - %.16P\n"
+               L"System Working Set:       %.16P - %.16P\n"
+               L"Loader Mappings:          %.16P - %.16P\n"
+               L"Non-Paged System Pool:    %.16P - %.16P\n"
+               L"Paged Pool:               %.16P - %.16P\n"
+               L"Session Space:            %.16P - %.16P\n"
+               L"System Cache:             %.16P - %.16P\n"
+               L"PFN Database:             %.16P - %.16P\n"
+               L"Non-Paged Pool:           %.16P - %.16P\n"
+               L"Non-Paged Expansion Pool: %.16P - %.16P\n"
+               L"Hardware Pool:            %.16P - %.16P\n",
+               GetInstalledMemorySize(),
+               MemoryLayout.UserSpaceStart,
+               MemoryLayout.UserSpaceEnd,
+               MemoryLayout.NonCanonicalStart,
+               MemoryLayout.NonCanonicalEnd,
+               MemoryLayout.ReservedSystemPoolStart,
+               MemoryLayout.ReservedSystemPoolEnd,
+               MemoryLayout.PteSpaceStart,
+               MemoryLayout.PteSpaceEnd,
+               MemoryLayout.HyperSpaceStart,
+               MemoryLayout.HyperSpaceEnd,
+               MemoryLayout.SharedSystemPageStart,
+               MemoryLayout.SharedSystemPageEnd,
+               MemoryLayout.SystemWorkingSetStart,
+               MemoryLayout.SystemWorkingSetEnd,
+               MemoryLayout.LoaderMappingsStart,
+               MemoryLayout.LoaderMappingsEnd,
+               MemoryLayout.NonPagedSystemPoolStart,
+               MemoryLayout.NonPagedSystemPoolEnd,
+               MemoryLayout.PagedPoolStart,
+               MemoryLayout.PagedPoolEnd,
+               MemoryLayout.SessionSpaceStart,
+               MemoryLayout.SessionSpaceEnd,
+               MemoryLayout.SystemCacheStart,
+               MemoryLayout.SystemCacheEnd,
+               MemoryLayout.PfnDatabase,
+               (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE),
+               MemoryLayout.NonPagedPoolStart,
+               MemoryLayout.NonPagedPoolEnd,
+               MemoryLayout.NonPagedExpansionPoolStart,
+               MemoryLayout.NonPagedExpansionPoolEnd,
+               MemoryLayout.HardwarePoolStart,
+               MemoryLayout.HardwarePoolEnd);
+}
+
 /**
  * Initializes the kernel's virtual memory layout.
  *
@@ -20,53 +263,131 @@ XTAPI
 VOID
 MM::Manager::InitializeMemoryLayout(VOID)
 {
-    ULONG_PTR PagedPoolSize, PteCount;
+    PFN_NUMBER MaximumNonPagedPoolSize;
-    ULONG PtesPerPage;
+    ULONG_PTR PfnDatabaseEnd;
-
-    /* Get the number of PTEs per page and calculate size of paged pool (at least 32MiB) */
-    PtesPerPage = MM::Pte::GetPtesPerPage();
-    PteCount = ((SIZE_TO_PAGES(33554432) + (PtesPerPage - 1)) / PtesPerPage);
-    PagedPoolSize = PteCount * PtesPerPage * MM_PAGE_SIZE;
-
-    /* Retrieve the PFN database size */
-    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
-
-    /* Define the number of system PTEs */
-    NumberOfSystemPtes = MM_DEFAULT_NUMBER_SYSTEM_PTES;
 
     /* Check if 5-level paging (LA57) is enabled */
     if(MM::Paging::GetXpaStatus())
     {
-        /* Configure memory layout for 5-level paging, using 57bit address space and providing a 128 PB address space */
+        /* Set PML5 base address */
-        MemoryLayout.PfnDatabase = (PMMPFN)0xFFFEFA8000000000ULL;
         MemoryLayout.SelfMapAddress = (PVOID)MM_P5E_LA57_BASE;
 
-        /* Define the non-paged and paged pool regions */
+        /* Define memory layout for 5-level paging */
-        MemoryLayout.NonPagedPoolStart = (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+        MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000;
-        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFEFFFFFFBFFFFFULL;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x00FFFFFFFFFEFFFF;
-        MemoryLayout.PagedPoolStart = (PVOID)0xFFFEF8A000000000ULL;
+        MemoryLayout.NonCanonicalStart = (PVOID)0x0080000000000000;
-        MemoryLayout.PagedPoolEnd = (PVOID)(((ULONG_PTR)MemoryLayout.PagedPoolStart + PagedPoolSize) - 1);
+        MemoryLayout.NonCanonicalEnd = (PVOID)0xFEFFFFFFFFFFFFFF;
-
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFF00000000000000;
-        /* Define hyperspace, system PTE space, and the user space limit */
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFECFFFFFFFFFFFF;
-        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFEF70000000000ULL;
+        MemoryLayout.PteSpaceStart = (PVOID)0xFFED000000000000;
-        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFEF77FFFFFFFFFULL;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xFFEDFFFFFFFFFFFF;
-        MemoryLayout.UserSpaceEnd = (PVOID)0x07FFFFFFFFFFFFFULL;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF;
     }
     else
     {
-        /* Configure memory layout for 4-level paging, using 48bit address space and providing a 128 TB address space */
+        /* Set PML4 base address */
-        MemoryLayout.PfnDatabase = (PMMPFN)0xFFFFFA8000000000ULL;
         MemoryLayout.SelfMapAddress = (PVOID)MM_PXE_BASE;
 
-        /* Define the non-paged and paged pool regions */
+        /* Define memory layout for 4-level paging */
-        MemoryLayout.NonPagedPoolStart = (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+        MemoryLayout.UserSpaceStart = (PVOID)0x0000000000010000;
-        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFFULL;
+        MemoryLayout.UserSpaceEnd = (PVOID)0x000007FFFFFEFFFF;
-        MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000ULL;
+        MemoryLayout.NonCanonicalStart = (PVOID)0x0000800000000000;
-        MemoryLayout.PagedPoolEnd = (PVOID)(((ULONG_PTR)MemoryLayout.PagedPoolStart + PagedPoolSize) - 1);
+        MemoryLayout.NonCanonicalEnd = (PVOID)0xFFFF7FFFFFFFFFFF;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0xFFFF800000000000;
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0xFFFFF67FFFFFFFFF;
+        MemoryLayout.PteSpaceStart = (PVOID)0xFFFFF68000000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xFFFFF6FFFFFFFFFF;
+        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000;
+        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFF;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFFFF78000000000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFFFF78000000FFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xFFFFF78000001000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xFFFFF7FFFFFFFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0xFFFFF80000000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0xFFFFF87FFFFFFFFF;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xFFFFF88000000000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xFFFFF89FFFFFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xFFFFF8A000000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xFFFFF8BFFFFFFFFF;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xFFFFF90000000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheStart = (PVOID)0xFFFFF98000000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xFFFFFA7FFFFFFFFF;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0xFFFFFA8000000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFFFFFFFFFBFFFFF;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFFFFFFFFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFFFFFFFFFF;
+    }
 
-        /* Define hyperspace, system PTE space, and the user space limit */
+    /* Compute allocation size for the PFN database */
-        MemoryLayout.HyperSpaceStart = (PVOID)0xFFFFF70000000000ULL;
+    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
-        MemoryLayout.HyperSpaceEnd = (PVOID)0xFFFFF77FFFFFFFFFULL;
+
-        MemoryLayout.UserSpaceEnd = (PVOID)0x000007FFFFFEFFFFULL;
+    /* Compute boot image size */
-    }
+    ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize);
+
+    /* Compute session space size */
+    ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize);
+
+    /* Update loader mappings space end address */
+    MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONGLONG)MemoryLayout.LoaderMappingsStart +
+                                             MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE);
+
+    /* Update session space start address */
+    MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd -
+                                             MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE);
+
+    /* Compute system PTE size */
+    ComputeSystemPteSize(&NumberOfSystemPtes);
+
+    /* Compute non-paged pool size */
+    ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize);
+    ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize);
+
+    /* Compute paged pool size */
+    ComputePagedPoolSize(&MemoryLayout.PagedPoolSize);
+
+    /* Insert the PFN database at the beginning of the non-paged pool */
+    MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.NonPagedPoolStart;
+
+    /* Compute the PFN database page-aligned end address */
+    PfnDatabaseEnd = (ULONGLONG)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+    PfnDatabaseEnd = ROUND_UP(PfnDatabaseEnd, MM_PAGE_SIZE);
+
+    /* Shrink the non-paged pool to fit the PFN database */
+    MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd;
+
+    /* Assign the rest of the non-paged pool to the expansion pool */
+    MemoryLayout.NonPagedExpansionPoolStart = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart +
+                                                      MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE);
+    MemoryLayout.NonPagedPoolEnd = MemoryLayout.NonPagedExpansionPoolStart;
+    MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONGLONG)MemoryLayout.NonPagedPoolStart +
+                                                    MaximumNonPagedPoolSize * MM_PAGE_SIZE);
+    MemoryLayout.NonPagedExpansionPoolSize = ((ULONGLONG)MemoryLayout.NonPagedExpansionPoolEnd -
+                                              (ULONGLONG)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE;
+
+    /* Update paged pool end address */
+    MemoryLayout.PagedPoolEnd = (PVOID)(((ULONGLONG)MemoryLayout.PagedPoolStart +
+                                         MemoryLayout.PagedPoolSize * MM_PAGE_SIZE) - 1);
+
+    /* Dump memory layout */
+    DumpMemoryLayout();
 }

xtoskrnl/mm/i686/mmgr.cc

@@ -9,6 +9,283 @@
 #include <xtos.hh>
 
 
+/**
+ * Calculates the maximum possible size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeMaximumNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR MaximumNonPagedPoolSize;
+    ULONGLONG PhysicalPages;
+
+    /* Get number of physical pages */
+    PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages();
+
+    /* Start with 1MiB and reserve space for PFN database */
+    MaximumNonPagedPoolSize = 1048576;
+
+    /* Check if system has at least 512MiB of physical memory */
+    if(PhysicalPages >= 126976)
+    {
+        /* Add 200KiB for each MiB above 4MiB */
+        MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 204800;
+
+        /* Check if non-paged pool has at least 128MiB */
+        if(MaximumNonPagedPoolSize < 134217728)
+        {
+            /* Expand non-paged pool size to 128MiB */
+            MaximumNonPagedPoolSize = 134217728;
+        }
+    }
+    else
+    {
+        /* Add 400KiB for each MiB above 4MiB */
+        MaximumNonPagedPoolSize += ((PhysicalPages - 1024)/256) * 409600;
+    }
+
+    /* Check if non-paged pool does not exceed 256MiB */
+    if(MaximumNonPagedPoolSize > 268435456)
+    {
+        /* Limit non-paged pool size to 256MiB */
+        MaximumNonPagedPoolSize = 268435456;
+    }
+
+    /* Return maximum non-paged pool size in pages */
+    *PoolSize = SIZE_TO_PAGES(MaximumNonPagedPoolSize);
+}
+
+/**
+ * Calculates the size of the non-paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the non-paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeNonPagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR NonPagedPoolSize;
+    ULONGLONG PhysicalPages;
+
+    /* Get number of physical pages */
+    PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages();
+
+    /* Verify if system has less than 256MiB of physical memory */
+    if(PhysicalPages <= 65536)
+    {
+        /* Reduce initial non-paged pool size to 2MiB to save memory */
+        NonPagedPoolSize = 2097152;
+    }
+    else
+    {
+        /* Start with 256KiB and add 32KiB for each MiB above 4MiB */
+        NonPagedPoolSize = 262144 + (((PhysicalPages - 1024) / 256) * 32768);
+
+        if(NonPagedPoolSize > 134217728)
+        {
+            /* Limit non-paged pool size to 128MiB */
+            NonPagedPoolSize = 134217728;
+        }
+    }
+
+    /* Return non-paged pool size in pages, aligned down to page size boundary */
+    *PoolSize = SIZE_TO_PAGES(ROUND_DOWN(NonPagedPoolSize, MM_PAGE_SIZE));
+}
+
+/**
+ * Calculates the size of the paged pool.
+ *
+ * @param PoolSize
+ *        A pointer to a variable that will receive the number of pages available for the paged pool.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputePagedPoolSize(OUT PPFN_NUMBER PoolSize)
+{
+    ULONG_PTR PagedPoolSize, PteCount;
+    ULONG PtesPerPage;
+
+    /* Start with maximum non-paged pool size */
+    ComputeMaximumNonPagedPoolSize(&PagedPoolSize);
+    PagedPoolSize *= MM_PAGE_SIZE;
+
+    /* Check XPA status */
+    if(MM::Paging::GetXpaStatus())
+    {
+        /* Four times the non-paged pool size on PAE-enabled systems */
+        PagedPoolSize *= 4;
+    }
+    else
+    {
+        /* Double the non-paged pool size on PAE-disabled systems */
+        PagedPoolSize *= 2;
+    }
+
+    /* Check if paged pool does not overlap non-paged pool */
+    if(PagedPoolSize > (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart)
+    {
+        /* Limit paged pool size to maximum possible */
+        PagedPoolSize = (ULONG_PTR)MemoryLayout.NonPagedSystemPoolStart - (ULONG_PTR)MemoryLayout.PagedPoolStart;
+    }
+
+    /* Ensure that paged pool size is at least 32MiB */
+    if(PagedPoolSize < 33554432)
+    {
+        /* Increase paged pool size to at least 32MiB */
+        PagedPoolSize = 33554432;
+    }
+
+    /* Get the number of PTEs per page and calculate size of paged pool */
+    PtesPerPage = MM::Pte::GetPtesPerPage();
+    PteCount = ((SIZE_TO_PAGES(PagedPoolSize) + (PtesPerPage - 1)) / PtesPerPage);
+
+    /* Return paged pool size */
+    *PoolSize = (PFN_NUMBER)(PteCount * PtesPerPage);
+}
+
+/**
+ * Calculates the size of the session space.
+ *
+ * @param SpaceSize
+ *        A pointer to a variable that will receive the number of pages available by the session space.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSessionSpaceSize(OUT PPFN_NUMBER SpaceSize)
+{
+    PFN_NUMBER SessionSpaceSize;
+
+    /* Session Pool, Session View, Session Image, Session Working Set and System View takes 108MiB */
+    SessionSpaceSize = 113246208;
+
+    /* Return number of pages used by the session space */
+    *SpaceSize = SessionSpaceSize / MM_PAGE_SIZE;
+}
+
+/**
+ * Calculates the size of the system PTEs.
+ *
+ * @param PteSize
+ *        A pointer to a variable that will receive the number of system PTEs.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::ComputeSystemPteSize(OUT PPFN_NUMBER PteSize)
+{
+    PFN_NUMBER SystemPteSize;
+
+    /* Check if system has less than 19MiB of physical memory */
+    if(MM::Pfn::GetNumberOfPhysicalPages() < 4864)
+    {
+        /* Set minimal system PTE size */
+        SystemPteSize = 7000;
+    }
+    else
+    {
+        /* Use standard system PTE size */
+        SystemPteSize = 11000;
+
+        /* Check if system has more than 32MiB of physical memory */
+        if(MM::Pfn::GetNumberOfPhysicalPages() > 8192)
+        {
+            /* Double system PTE size */
+            SystemPteSize *= 2;
+
+            /* Check if system has more than 256MiB of physical memory */
+            if(MM::Pfn::GetNumberOfPhysicalPages() > 65536)
+            {
+                /* Double system PTE size */
+                SystemPteSize *= 2;
+            }
+        }
+    }
+
+    /* Return system PTE size */
+    *PteSize = SystemPteSize;
+}
+
+/**
+ * Dumps the kernel's memory layout.
+ *
+ * @return This routine does not return any value.
+ *
+ * @since XT 1.0
+ */
+XTAPI
+VOID
+MM::Manager::DumpMemoryLayout(VOID)
+{
+    /* Dump memory layout */
+    DebugPrint(L"System with %zu MiB of installed memory:\n"
+               L"User Space:               %.8P - %.8P\n"
+               L"Loader Mappings:          %.8P - %.8P\n"
+               L"PFN Database:             %.8P - %.8P\n"
+               L"Non-Paged Pool:           %.8P - %.8P\n"
+               L"Session Space:            %.8P - %.8P\n"
+               L"PTE Space:                %.8P - %.8P\n"
+               L"Hyper Space:              %.8P - %.8P\n"
+               L"System Working Set:       %.8P - %.8P\n"
+               L"System Cache:             %.8P - %.8P\n"
+               L"Paged Pool:               %.8P - %.8P\n"
+               L"Non-Paged System Pool:    %.8P - %.8P\n"
+               L"Non-Paged Expansion Pool: %.8P - %.8P\n"
+               L"Shared System Page:       %.8P - %.8P\n"
+               L"Hardware Pool:            %.8P - %.8P\n",
+               GetInstalledMemorySize(),
+               MemoryLayout.UserSpaceStart,
+               MemoryLayout.UserSpaceEnd,
+               MemoryLayout.LoaderMappingsStart,
+               MemoryLayout.LoaderMappingsEnd,
+               MemoryLayout.PfnDatabase,
+               (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + (ULONG_PTR)MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE),
+               MemoryLayout.NonPagedPoolStart,
+               MemoryLayout.NonPagedPoolEnd,
+               MemoryLayout.SessionSpaceStart,
+               MemoryLayout.SessionSpaceEnd,
+               MemoryLayout.PteSpaceStart,
+               MemoryLayout.PteSpaceEnd,
+               MemoryLayout.HyperSpaceStart,
+               MemoryLayout.HyperSpaceEnd,
+               MemoryLayout.SystemWorkingSetStart,
+               MemoryLayout.SystemWorkingSetEnd,
+               MemoryLayout.SystemCacheStart,
+               MemoryLayout.SystemCacheEnd,
+               MemoryLayout.PagedPoolStart,
+               MemoryLayout.PagedPoolEnd,
+               MemoryLayout.NonPagedSystemPoolStart,
+               MemoryLayout.NonPagedSystemPoolEnd,
+               MemoryLayout.NonPagedExpansionPoolStart,
+               MemoryLayout.NonPagedExpansionPoolEnd,
+               MemoryLayout.SharedSystemPageStart,
+               MemoryLayout.SharedSystemPageEnd,
+               MemoryLayout.HardwarePoolStart,
+               MemoryLayout.HardwarePoolEnd);
+}
+
 /**
  * Initializes the kernel's virtual memory layout.
  *
@@ -20,69 +297,153 @@ XTAPI
 VOID
 MM::Manager::InitializeMemoryLayout(VOID)
 {
-    ULONG_PTR PagedPoolSize, PteCount;
+    PFN_NUMBER MaximumNonPagedPoolSize;
-    PFN_NUMBER PhysicalPages;
+    ULONG_PTR PfnDatabaseEnd;
-    ULONG PtesPerPage;
-
-    /* Define the number of system PTEs */
-    NumberOfSystemPtes = MM_DEFAULT_NUMBER_SYSTEM_PTES;
-
-    /* Retrieve the number of physical pages */
-    PhysicalPages = MM::Pfn::GetNumberOfPhysicalPages();
-
-    /* Verify the number of physical pages */
-    if(PhysicalPages < 8192)
-    {
-        /* Less than 32MiB of physical memory (8192 pages), use the minimum number of system PTEs */
-        NumberOfSystemPtes = MM_MINIMUM_NUMBER_SYSTEM_PTES;
-    }
-    else if(PhysicalPages > 32768)
-    {
-        /* More than 128MiB of physical memory (32768 pages), use the maximum number of system PTEs */
-        NumberOfSystemPtes = MM_MAXIMUM_NUMBER_SYSTEM_PTES;
-    }
-
-    /* Calculate size of paged pool */
-    PtesPerPage = MM::Pte::GetPtesPerPage();
-    PteCount = (SIZE_TO_PAGES(33554432) + (PtesPerPage - 1)) / PtesPerPage;
-    PagedPoolSize = PteCount * PtesPerPage * MM_PAGE_SIZE;
-
-    /* Retrieve the PFN database size */
-    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
 
     /* Check if 3-level paging (PAE) is enabled */
     if(MM::Paging::GetXpaStatus())
     {
-        /* Configure memory layout for 3-level paging, using 36bit address space and providing a 64 GB address space */
+        /* Set PML3 base address */
-        MemoryLayout.PfnDatabase = (PMMPFN)0xB0000000;
         MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE;
 
-        /* Define the non-paged and paged pool regions */
+        /* Define memory layout for 3-level paging */
-        MemoryLayout.NonPagedPoolStart = (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+        MemoryLayout.NonCanonicalStart = (PVOID)0x00000000;
-        MemoryLayout.NonPagedPoolEnd = (PVOID)0xEEFFFFFF;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000;
-        MemoryLayout.PagedPoolStart = (PVOID)0xE2000000;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000;
-        MemoryLayout.PagedPoolEnd = (PVOID)(((ULONG_PTR)MemoryLayout.PagedPoolStart + PagedPoolSize) - 1);
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000;
-
+        MemoryLayout.UserSpaceStart = (PVOID)0x00010000;
-        /* Define hyperspace, system PTE space, and the user space limit */
+        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceStart = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xC07FFFFF;
         MemoryLayout.HyperSpaceStart = (PVOID)0xC0800000;
         MemoryLayout.HyperSpaceEnd = (PVOID)0xC0BFFFFF;
-        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF;
+        MemoryLayout.SystemCacheStart = (PVOID)0xC1000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xE1000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF;
     }
     else
     {
-        /* Configure memory layout for 2-level paging, using 32bit address space and providing a 4 GB address space */
+        /* Set PML2 base address */
-        MemoryLayout.PfnDatabase = (PMMPFN)0xB0000000;
         MemoryLayout.SelfMapAddress = (PVOID)MM_PTE_BASE;
 
-        /* Define the non-paged and paged pool regions */
+        /* Define memory layout for 2-level paging */
-        MemoryLayout.NonPagedPoolStart = (PVOID)((ULONG_PTR)MemoryLayout.PfnDatabase + MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+        MemoryLayout.NonCanonicalStart = (PVOID)0x00000000;
-        MemoryLayout.NonPagedPoolEnd = (PVOID)0xFFBE0000;
+        MemoryLayout.NonCanonicalEnd = (PVOID)0x00000000;
-        MemoryLayout.PagedPoolStart = (PVOID)0xE1000000;
+        MemoryLayout.ReservedSystemPoolStart = (PVOID)0x00000000;
-        MemoryLayout.PagedPoolEnd = (PVOID)(((ULONG_PTR)MemoryLayout.PagedPoolStart + PagedPoolSize) - 1);
+        MemoryLayout.ReservedSystemPoolEnd = (PVOID)0x00000000;
-
+        MemoryLayout.UserSpaceStart = (PVOID)0x00010000;
-        /* Define hyperspace, system PTE space, and the user space limit */
+        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.LoaderMappingsStart = (PVOID)0x80000000;
+        MemoryLayout.LoaderMappingsEnd = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolStart = (PVOID)0x90000000;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceStart = (PVOID)0xB0000000;
+        MemoryLayout.SessionSpaceEnd = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceStart = (PVOID)0xC0000000;
+        MemoryLayout.PteSpaceEnd = (PVOID)0xC03FFFFF;
         MemoryLayout.HyperSpaceStart = (PVOID)0xC0400000;
         MemoryLayout.HyperSpaceEnd = (PVOID)0xC07FFFFF;
-        MemoryLayout.UserSpaceEnd = (PVOID)0x7FFEFFFF;
+        MemoryLayout.SystemWorkingSetStart = (PVOID)0xC0C00000;
+        MemoryLayout.SystemWorkingSetEnd = (PVOID)0xC0FFFFFF;
+        MemoryLayout.SystemCacheStart = (PVOID)0xC1000000;
+        MemoryLayout.SystemCacheEnd = (PVOID)0xE0FFFFFF;
+        MemoryLayout.PagedPoolStart = (PVOID)0xE1000000;
+        MemoryLayout.PagedPoolEnd = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolStart = (PVOID)0xECC00000;
+        MemoryLayout.NonPagedSystemPoolEnd = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolStart = (PVOID)0xF7BE0000;
+        MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageStart = (PVOID)0xFFBFF000;
+        MemoryLayout.SharedSystemPageEnd = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolStart = (PVOID)0xFFC00000;
+        MemoryLayout.HardwarePoolEnd = (PVOID)0xFFFFFFFF;
+    }
+
+    /* Compute allocation size for the PFN database */
+    MM::Pfn::ComputePfnDatabaseSize(&MemoryLayout.PfnDatabaseSize);
+
+    /* Compute boot image size */
+    ComputeBootImageSize(&MemoryLayout.LoaderMappingsSize);
+
+    /* Compute session space size */
+    ComputeSessionSpaceSize(&MemoryLayout.SessionSpaceSize);
+
+    /* Update loader mappings space end address */
+    MemoryLayout.LoaderMappingsEnd = (PVOID)((ULONG_PTR)MemoryLayout.LoaderMappingsStart +
+                                             MemoryLayout.LoaderMappingsSize * MM_PAGE_SIZE);
+
+    /* Update session space start address */
+    MemoryLayout.SessionSpaceStart = (PVOID)((ULONGLONG)MemoryLayout.SessionSpaceEnd -
+                                             MemoryLayout.SessionSpaceSize * MM_PAGE_SIZE);
+
+    /* Compute system PTE size */
+    ComputeSystemPteSize(&NumberOfSystemPtes);
+
+    /* Compute non-paged pool size */
+    ComputeNonPagedPoolSize(&MemoryLayout.NonPagedPoolSize);
+    ComputeMaximumNonPagedPoolSize(&MaximumNonPagedPoolSize);
+
+    /* Compute paged pool size */
+    ComputePagedPoolSize(&MemoryLayout.PagedPoolSize);
+
+    /* Insert the PFN database right after the loader mappings */
+    MemoryLayout.PfnDatabase = (PMMPFN)MemoryLayout.LoaderMappingsEnd;
+
+    /* Compute the PFN database page-aligned end address */
+    PfnDatabaseEnd = (ULONG_PTR)MemoryLayout.PfnDatabase + (MemoryLayout.PfnDatabaseSize * MM_PAGE_SIZE);
+    PfnDatabaseEnd = ROUND_UP(PfnDatabaseEnd, MM_PAGE_SIZE);
+
+    /* Check in non-paged pool fits before session space */
+    if(MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE <= ((ULONG_PTR)MemoryLayout.SessionSpaceStart - PfnDatabaseEnd))
+    {
+        /* Set non-paged pool start and end addresses */
+        MemoryLayout.NonPagedPoolStart = (PVOID)PfnDatabaseEnd;
+        MemoryLayout.NonPagedPoolEnd = (PVOID)(PfnDatabaseEnd + MemoryLayout.NonPagedPoolSize * MM_PAGE_SIZE);
+
+        /* Check if non-paged expansion pool overflows */
+        if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize *
+           MM_PAGE_SIZE >= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart)
+        {
+            /* Check if non-paged expansion pool fits */
+            if((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart + MaximumNonPagedPoolSize *
+               MM_PAGE_SIZE <= (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd)
+            {
+                /* Set non-paged expansion pool end address */
+                MemoryLayout.NonPagedExpansionPoolEnd = (PVOID)((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart +
+                                                                MaximumNonPagedPoolSize * MM_PAGE_SIZE);
             }
         }
+
+        /* Compute non-paged expansion pool size */
+        MemoryLayout.NonPagedExpansionPoolSize = ((ULONG_PTR)MemoryLayout.NonPagedExpansionPoolEnd -
+                                                  (ULONG_PTR)MemoryLayout.NonPagedExpansionPoolStart) / MM_PAGE_SIZE;
+    }
+    else
+    {
+        /* Unfortunally non-paged pool does not fit before session space. What can we do? */
+    }
+
+    /* Update paged pool end address */
+    MemoryLayout.PagedPoolEnd = (PVOID)(((ULONG_PTR)MemoryLayout.PagedPoolStart +
+                                         MemoryLayout.PagedPoolSize * MM_PAGE_SIZE) - 1);
+
+    /* Dump memory layout */
+    DumpMemoryLayout();
+}