From 49302f5b416507faf64a83ac36eba8c957fd6b36 Mon Sep 17 00:00:00 2001 From: Dibyamartanda Samanta Date: Tue, 21 May 2024 14:30:43 +0200 Subject: [PATCH] [NTOSKRNL:CC] Implementation of Lazy Writer Lazy Writer is complete --- NTOSKRNL/CC/cclazywriter.cpp | 433 ++++++++++++++++++++++++++++++++++- 1 file changed, 432 insertions(+), 1 deletion(-) diff --git a/NTOSKRNL/CC/cclazywriter.cpp b/NTOSKRNL/CC/cclazywriter.cpp index 70086df..bdcb581 100644 --- a/NTOSKRNL/CC/cclazywriter.cpp +++ b/NTOSKRNL/CC/cclazywriter.cpp @@ -9,12 +9,63 @@ #define NTDEBUG #include #include "ccinternal.hpp" -#include "cclazywriter.hpp" + extern "C" /*Internal Function*/ + + +VOID +NTAPI +CcComputeNextScanTime(PLARGE_INTEGER OldestTICKTIMEForMetadata, PLARGE_INTEGER NextScanDelay) +{ + NextScanDelay- = 0; + LARGE_INTEGER CurrentTickCount = {0}; + LARGE_INTEGER TICKTIME = {0}; + LARGE_INTEGER WRITE_DELAY = {0}; + LARGE_INTEGER TICK_ELAPSED = {0}; + + if (CcIsWriteBehindThreadpoolAtLowPriority()) + { + + + KeQueryTickCount(&CurrentTickCount); + + // Calculate Tick Time based on the current tick count and the oldest scan time + TICKTIME.QuadPart = 160000000 / KeMaximumIncrement; + WRITE_DELAY.QuadPart = (OldestTICKTIMEForMetadata->QuadPart - CurrentTickCount.QuadPart) / KeMaximumIncrement; + + // Increment the consecutive workless lazy scan count + ++CcConsecutiveWorklessLazyScanCount; + + // Check if the oldest scan time is not the maximum and the calculated delay is greater than the current tick + // count + if (OldestTICKTIMEForMetadata->QuadPart != -1 && OldestTICKTIMEForMetadata->QuadPart != 0x7FFFFFFFFFFFFFFF && + (TICKTIME.QuadPart + OldestTICKTIMEForMetadata->QuadPart) > CurrentTickCount.QuadPart) + { + + TICK_ELAPSED.QuadPart = OldestTICKTIMEForMetadata->QuadPart - CurrentTickCount.QuadPart; + + // Calculate the next scan delay + NextScanDelay->QuadPart = TICKTIME.QuadPart + TICK_ELAPSED.QuadPart; + + // Reset the consecutive workless lazy scan count + CcConsecutiveWorklessLazyScanCount = 0; + } + + // Check if the number of consecutive workless lazy scans has reached the maximum + if (CcConsecutiveWorklessLazyScanCount >= CcMaxWorklessLazywriteScans) + { + // Disable the scan by setting the next scan delay to the maximum values + NextScanDelay->QuadPart = -1; + CcConsecutiveWorklessLazyScanCount = 0; + NextScanDelay->HighPart = 0x7FFFFFFF; + } + } +} + VOID VECTORCALL CcPostWorkQueue(IN PWORK_QUEUE_ENTRY WorkItem, @@ -85,6 +136,76 @@ CcScheduleLazyWriteScan(IN BOOLEAN NoDelay) { return CcScheduleLazyWriteScanEx(NoDelay, False); } +VOID VECTORCALL CcScanDpc(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SysArg0, IN PVOID SysArg1) +{ + + PLIST_ENTRY WorkQueue; + PGENERAL_LOOKASIDE LookasideList; + KIRQL CurrentIrql; + + /* GET Current PRCB and Assign work item*/ + PKPRCB Prcb = KeGetCurrentPrcb(); + LookasideList = Prcb->PPLookasideList[5].P; + InterlockedIncrement(&LookasideList->TotalAllocates); + PWORK_QUEUE_ENTRY WorkItem = static_cast(InterlockedPopEntrySList(&LookasideList->ListHead)); + + InterlockedIncrement(&LookasideList->AllocateMisses); + LookasideList = Prcb->PPLookasideList[5].L; + InterlockedIncrement(&LookasideList->TotalAllocates); + + WorkItem = static_cast(InterlockedPopEntrySList(&LookasideList->ListHead); + + /* Assingning Work Item if it is null*/ + if (!WorkItem) + { + InterlockedIncrement(&LookasideList->AllocateMisses); + WorkItem = static_cast( + (LookasideList->Allocate(LookasideList->Type, LookasideList->Size, LookasideList->Tag))); + + if (WorkItem != nullptr) + { + DBGPRINT("CcScanDpc: WorkQueue is NULL, SECOND Assingment in Progress\n"); + InterlockedIncrement(&LookasideList->AllocateMisses); + WorkItem = static_cast( + LookasideList->Allocate(LookasideList->Type, LookasideList->Size, LookasideList->Tag)); + } + } + + /* Release SpinLock if WOrk Item Queue is not Assigned*/ + if (!WorkItem) + { + DBGRINT("CcScanDpc: WorkQueue is not assigned.!\n"); + CurrentIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + /* Set Lazy Writer Scan to False */ + LazyWriter.ScanActive = FALSE; + DBGRINT("CcScanDpc: Lazy Writer Scan is Disabled!\n"); + KeReleaseQueuedSpinLock(LockQueueMasterLock, CurrentIrql); + return; + } + + + WorkItem->Function = LazyWriteScan; + CurrentIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + /* Check for Lazy Writer Teardown Status*/ + + if (LazyWriter.PendingTeardown) + { + /* Assign Worker Type*/ + WorkQueue = &CcFastTeardownWorkQueue; + /*If Pending Teardown is active in the Queue , disable it now since Queue type is assigned*/ + LazyWriter.PendingTeardown = false; + } + + else + { + WorkQueue = &CcRegularWorkQueue; + } + + /* Release the Spinlock and Post the Lazy Write */ + KeReleaseQueuedSpinLock(LockQueueMasterLock, CurrentIrql); + CcPostWorkQueue(WorkItem, WorkQueue); +} LONG VECTORCALL @@ -860,3 +981,313 @@ NTAPI CcLazyWriteScan() CcScheduleLazyWriteScan(FALSE); } } +NTSTATUS CcWaitForCurrentLazyWriterActivity() +{ + NTSTATUS result; + PWORK_QUEUE_ENTRY WorkQueueEntry; + KEVENT Event; + KIRQL irql; + + result = CcAllocateWorkQueueEntry(&WorkQueueEntry); + if (NT_SUCCESS(result)) + { + WorkQueueEntry->Function = SetDone; + KeInitializeEvent(&Event, NotificationEvent, FALSE); + WorkQueueEntry->Parameters.Notification.Reason = (ULONG_PTR)&Event; + + if ((PerfGlobalGroupMask.Masks[4] & 0x20000) != 0) + CcPerfLogWorkItemEnqueue(&CcPostTickWorkQueue, WorkQueueEntry, 0, 0); + + irql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + WorkQueueEntry->WorkQueueLinks.Flink = &CcPostTickWorkQueue; + WorkQueueEntry->WorkQueueLinks.Blink = CcPostTickWorkQueue.Blink; + CcPostTickWorkQueue.Blink->Flink = &WorkQueueEntry->WorkQueueLinks; + CcPostTickWorkQueue.Blink = &WorkQueueEntry->WorkQueueLinks; + + LazyWriter.OtherWork = 1; + _InterlockedIncrement(&CcPostTickWorkItemCount); + + CcScheduleLazyWriteScan(1, 1); + + KeReleaseQueuedSpinLock(LockQueueMasterLock, irql); + + result = KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + + _InterlockedDecrement(&CcPostTickWorkItemCount); + } + + return result; +} + +VOID +NTAPI +CcWorkerThread(PVOID Parameter) +{ + PWORK_QUEUE_ITEM WorkItem = static_cast(Parameter); + PGENERAL_LOOKASIDE LookasideList = nullptr; + PSHARED_CACHE_MAP SharedMap = nullptr; + PWORK_QUEUE_ENTRY WorkEntry = nullptr; + PLIST_ENTRY Entry = nullptr; + PKPRCB Prcb = nullptr; + IO_STATUS_BLOCK IoStatus = {}; + KIRQL OldIrql = PASSIVE_LEVEL; + BOOLEAN DropThrottle = FALSE; + BOOLEAN WritePerformed = FALSE; + + DPRINT("CcWorkerThread: WorkItem"); + + IoStatus.Status = STATUS_SUCCESS; + IoStatus.Information = 0; + + /* Loop till we have jobs */ + while (TRUE) + { + /* Lock queues */ + OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock); + + /* If we have to touch throttle, reset it now! */ + if (DropThrottle) + { + CcQueueThrottle = FALSE; + DropThrottle = FALSE; + } + + if (IoStatus.Information == 0x8A5E) + { + ASSERT(Entry); + + if (WorkEntry->Function == WriteBehind) + { + SharedMap = WorkEntry->Parameters.Write.SharedCacheMap; + ASSERT(Entry != &CcFastTeardownWorkQueue); + SharedMap->WriteBehindWorkQueueEntry = WorkEntry; + } + + InsertTailList(Entry, &WorkEntry->WorkQueueLinks); + IoStatus.Information = 0; + } + + /* Check if we have write to do */ + if (!IsListEmpty(&CcFastTeardownWorkQueue)) + { + Entry = &CcFastTeardownWorkQueue; + WorkEntry = CONTAINING_RECORD(Entry->Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); + + ASSERT((WorkEntry->Function == LazyWriteScan) || (WorkEntry->Function == WriteBehind)); + } + /* If not, check read queues */ + else if (!IsListEmpty(&CcExpressWorkQueue)) + { + Entry = &CcExpressWorkQueue; + } + else if (!IsListEmpty(&CcRegularWorkQueue)) + { + Entry = &CcRegularWorkQueue; + } + else + { + break; + } + + /* Get our work item, if someone is waiting for us to finish + and we're not the only thread in queue then, quit running to let the others do + and throttle so that noone starts till current activity is over + */ + WorkEntry = CONTAINING_RECORD(Entry->Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); + + if (WorkEntry->Function == SetDone && CcNumberActiveWorkerThreads > 1) + { + CcQueueThrottle = TRUE; + break; + } + + if (WorkEntry->Function == WriteBehind) + WorkEntry->Parameters.Write.SharedCacheMap->WriteBehindWorkQueueEntry = NULL; + + /* Remove current entry */ + RemoveHeadList(Entry); + + /* Unlock queues */ + KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); + + /* And handle it */ + __try + { + switch (WorkEntry->Function) + { + case ReadAhead: { + CcPerformReadAhead(WorkEntry->Parameters.Read.FileObject); + break; + } + case WriteBehind: { + WritePerformed = TRUE; + PsGetCurrentThread()->MemoryMaker = 1; + + CcWriteBehind(WorkEntry->Parameters.Write.SharedCacheMap, &IoStatus); + + if (!NT_SUCCESS(IoStatus.Status)) + WritePerformed = FALSE; + + PsGetCurrentThread()->MemoryMaker = 0; + break; + } + case LazyWriteScan: { + CcLazyWriteScan(); + break; + } + case SetDone: { + KeSetEvent(WorkEntry->Parameters.Event.Event, IO_NO_INCREMENT, FALSE); + DropThrottle = TRUE; + break; + } + } + } + __except (CcExceptionFilter(GetExceptionCode())) + { + if (WorkEntry->Function == WriteBehind) + PsGetCurrentThread()->MemoryMaker = 0; + } + + /* Handle for WriteBehind */ + if (IoStatus.Information == 0x8A5E) + continue; + + /* Release the current element and continue */ + + LookasideList = Prcb->PPLookasideList[5].P; + InterlockedIncrement(&LookasideList->TotalFrees); // Use interlocked increment + + if (LookasideList->ListHead.Depth < LookasideList->Depth) + { + InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry); + continue; + } + + if (LookasideList->ListHead.Depth < LookasideList->Depth) + { + InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry); + continue; + } + InterlockedIncrement(&LookasideList->FreeMisses); // Use interlocked increment + + LookasideList = Prcb->PPLookasideList[5].L; + InterlockedIncrement(&LookasideList->TotalFrees); + + if (LookasideList->ListHead.Depth < LookasideList->Depth) + { + InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry); + continue; + } + + InterlockedIncrement(&LookasideList->FreeMisses); + LookasideList->Free(WorkEntry); + } + /* Our thread is available again */ + InsertTailList(&CcIdleWorkerThreadList, &WorkItem->List); + + /* One less worker */ + CcNumberActiveWorkerThreads--; + + /* Unlock queues */ + KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); + + /* If there are pending write openations and we have at least 20 dirty pages */ + if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20) + { + /* And if we performed a write operation previously, + then stress the system a bit and reschedule a scan to find stuff to write + */ + if (WritePerformed) + CcLazyWriteScan(); + } +} + +VECTORCALL +CcAllocateWorkQueueEntry(PWORK_QUEUE_ENTRY &workQueueEntry) +{ + PKPRCB Prcb = KeGetCurrentPrcb(); + PGENERAL_LOOKASIDE LookasideList = nullptr; + PWORK_QUEUE_ENTRY WorkItem = nullptr; + KEVENT Event = NULL; + KIRQL OldIrql = NULL; + NTSTATUS Status = NULL; + + /* Allocate a work item */ + LookasideList = Prcb->PPLookasideList[6].P; + _InterlockedIncrement(&LookasideList->TotalAllocates); + + WorkItem = (PWORK_QUEUE_ENTRY)InterlockedPopEntrySList(&LookasideList->ListHead); + if (!WorkItem) + { + LookasideList->AllocateMisses++; + LookasideList = Prcb->PPLookasideList[5].L; + LookasideList->TotalAllocates++; + + WorkItem = (PWORK_QUEUE_ENTRY)InterlockedPopEntrySList(&LookasideList->ListHead); + if (!WorkItem) + { + LookasideList->AllocateMisses++; + WorkItem = (PWORK_QUEUE_ENTRY)LookasideList->Allocate(LookasideList->Type, LookasideList->Size, + LookasideList->Tag); + WorkItem->WorkQueueLinks.Flink = Prcb->Number; + } + } + + if (!WorkItem) + { + DBGPRINT("CcAllocateWorkQueueEntry: STATUS_INSUFFICIENT_RESOURCES\n"); + return STATUS_INSUFFICIENT_RESOURCES; + } + + workQueueEntry = WorkItem; +} +/* Exported Function */ + +NTSTATUS CcWaitForCurrentLazyWriterActivity() +{ + NTSTATUS status; // Status of the operation + PWORK_QUEUE_ENTRY workQueueEntry; // Work queue entry + PLIST_ENTRY blink; + KIRQL irql; + KEVENT event; // Event object + + // Allocate a work queue entry + status = CcAllocateWorkQueueEntry(&workQueueEntry); + if (NT_SUCCESS(status)) // Check if the status is a success + { + /* Set the function of the work queue entry*/ + workQueueEntry->Function = SetDone; + /* Initialize the event object*/ + KeInitializeEvent(&event, NotificationEvent, 0); + /*Set the reason for the notification*/ + workQueueEntry->Parameters.Notification.Reason = &event; + + /* Acquire the queued spin lock*/ + irql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + blink = CcPostTickWorkQueue.Blink; + + /*Enqueue the work item*/ + workQueueEntry->WorkQueueLinks.Flink = &CcPostTickWorkQueue; + workQueueEntry->WorkQueueLinks.Blink = blink; + blink->Flink = &workQueueEntry->WorkQueueLinks; + CcPostTickWorkQueue.Blink = &workQueueEntry->WorkQueueLinks; + + /*Set the other work flag*/ + LazyWriter.OtherWork = 1; + /*Increment the work item count*/ + _InterlockedIncrement(&CcPostTickWorkItemCount); + /* Schedule Lazy Write Scan */ + CcScheduleLazyWriteScan(true); + /*Release the queued spin lock*/ + KeReleaseQueuedSpinLock(LockQueueMasterLock, irql); + + /*Wait for the single object*/ + status = KeWaitForSingleObject(&event, Executive, NULL, NULL, NULL); + // Decrement the work item count + _InterlockedDecrement(&CcPostTickWorkItemCount); + } + + /*Return the status of the operation*/ + return status; +}