[NTOSKRNL:CC] Implementation of Lazy Writer

Lazy Writer is complete
This commit is contained in:
Dibyamartanda Samanta 2024-05-21 14:30:43 +02:00
parent 3ff9824c8b
commit 49302f5b41

View File

@ -9,12 +9,63 @@
#define NTDEBUG #define NTDEBUG
#include <debug.h> #include <debug.h>
#include "ccinternal.hpp" #include "ccinternal.hpp"
#include "cclazywriter.hpp"
extern "C" extern "C"
/*Internal Function*/ /*Internal Function*/
VOID
NTAPI
CcComputeNextScanTime(PLARGE_INTEGER OldestTICKTIMEForMetadata, PLARGE_INTEGER NextScanDelay)
{
NextScanDelay- = 0;
LARGE_INTEGER CurrentTickCount = {0};
LARGE_INTEGER TICKTIME = {0};
LARGE_INTEGER WRITE_DELAY = {0};
LARGE_INTEGER TICK_ELAPSED = {0};
if (CcIsWriteBehindThreadpoolAtLowPriority())
{
KeQueryTickCount(&CurrentTickCount);
// Calculate Tick Time based on the current tick count and the oldest scan time
TICKTIME.QuadPart = 160000000 / KeMaximumIncrement;
WRITE_DELAY.QuadPart = (OldestTICKTIMEForMetadata->QuadPart - CurrentTickCount.QuadPart) / KeMaximumIncrement;
// Increment the consecutive workless lazy scan count
++CcConsecutiveWorklessLazyScanCount;
// Check if the oldest scan time is not the maximum and the calculated delay is greater than the current tick
// count
if (OldestTICKTIMEForMetadata->QuadPart != -1 && OldestTICKTIMEForMetadata->QuadPart != 0x7FFFFFFFFFFFFFFF &&
(TICKTIME.QuadPart + OldestTICKTIMEForMetadata->QuadPart) > CurrentTickCount.QuadPart)
{
TICK_ELAPSED.QuadPart = OldestTICKTIMEForMetadata->QuadPart - CurrentTickCount.QuadPart;
// Calculate the next scan delay
NextScanDelay->QuadPart = TICKTIME.QuadPart + TICK_ELAPSED.QuadPart;
// Reset the consecutive workless lazy scan count
CcConsecutiveWorklessLazyScanCount = 0;
}
// Check if the number of consecutive workless lazy scans has reached the maximum
if (CcConsecutiveWorklessLazyScanCount >= CcMaxWorklessLazywriteScans)
{
// Disable the scan by setting the next scan delay to the maximum values
NextScanDelay->QuadPart = -1;
CcConsecutiveWorklessLazyScanCount = 0;
NextScanDelay->HighPart = 0x7FFFFFFF;
}
}
}
VOID VOID
VECTORCALL VECTORCALL
CcPostWorkQueue(IN PWORK_QUEUE_ENTRY WorkItem, CcPostWorkQueue(IN PWORK_QUEUE_ENTRY WorkItem,
@ -85,6 +136,76 @@ CcScheduleLazyWriteScan(IN BOOLEAN NoDelay)
{ {
return CcScheduleLazyWriteScanEx(NoDelay, False); return CcScheduleLazyWriteScanEx(NoDelay, False);
} }
VOID VECTORCALL CcScanDpc(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SysArg0, IN PVOID SysArg1)
{
PLIST_ENTRY WorkQueue;
PGENERAL_LOOKASIDE LookasideList;
KIRQL CurrentIrql;
/* GET Current PRCB and Assign work item*/
PKPRCB Prcb = KeGetCurrentPrcb();
LookasideList = Prcb->PPLookasideList[5].P;
InterlockedIncrement(&LookasideList->TotalAllocates);
PWORK_QUEUE_ENTRY WorkItem = static_cast<PWORK_QUEUE_ENTRY>(InterlockedPopEntrySList(&LookasideList->ListHead));
InterlockedIncrement(&LookasideList->AllocateMisses);
LookasideList = Prcb->PPLookasideList[5].L;
InterlockedIncrement(&LookasideList->TotalAllocates);
WorkItem = static_cast<PWORK_QUEUE_ENTRY>(InterlockedPopEntrySList(&LookasideList->ListHead);
/* Assingning Work Item if it is null*/
if (!WorkItem)
{
InterlockedIncrement(&LookasideList->AllocateMisses);
WorkItem = static_cast<PWORK_QUEUE_ENTRY>(
(LookasideList->Allocate(LookasideList->Type, LookasideList->Size, LookasideList->Tag)));
if (WorkItem != nullptr)
{
DBGPRINT("CcScanDpc: WorkQueue is NULL, SECOND Assingment in Progress\n");
InterlockedIncrement(&LookasideList->AllocateMisses);
WorkItem = static_cast<PWORK_QUEUE_ENTRY>(
LookasideList->Allocate(LookasideList->Type, LookasideList->Size, LookasideList->Tag));
}
}
/* Release SpinLock if WOrk Item Queue is not Assigned*/
if (!WorkItem)
{
DBGRINT("CcScanDpc: WorkQueue is not assigned.!\n");
CurrentIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
/* Set Lazy Writer Scan to False */
LazyWriter.ScanActive = FALSE;
DBGRINT("CcScanDpc: Lazy Writer Scan is Disabled!\n");
KeReleaseQueuedSpinLock(LockQueueMasterLock, CurrentIrql);
return;
}
WorkItem->Function = LazyWriteScan;
CurrentIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
/* Check for Lazy Writer Teardown Status*/
if (LazyWriter.PendingTeardown)
{
/* Assign Worker Type*/
WorkQueue = &CcFastTeardownWorkQueue;
/*If Pending Teardown is active in the Queue , disable it now since Queue type is assigned*/
LazyWriter.PendingTeardown = false;
}
else
{
WorkQueue = &CcRegularWorkQueue;
}
/* Release the Spinlock and Post the Lazy Write */
KeReleaseQueuedSpinLock(LockQueueMasterLock, CurrentIrql);
CcPostWorkQueue(WorkItem, WorkQueue);
}
LONG LONG
VECTORCALL VECTORCALL
@ -860,3 +981,313 @@ NTAPI CcLazyWriteScan()
CcScheduleLazyWriteScan(FALSE); CcScheduleLazyWriteScan(FALSE);
} }
} }
NTSTATUS CcWaitForCurrentLazyWriterActivity()
{
NTSTATUS result;
PWORK_QUEUE_ENTRY WorkQueueEntry;
KEVENT Event;
KIRQL irql;
result = CcAllocateWorkQueueEntry(&WorkQueueEntry);
if (NT_SUCCESS(result))
{
WorkQueueEntry->Function = SetDone;
KeInitializeEvent(&Event, NotificationEvent, FALSE);
WorkQueueEntry->Parameters.Notification.Reason = (ULONG_PTR)&Event;
if ((PerfGlobalGroupMask.Masks[4] & 0x20000) != 0)
CcPerfLogWorkItemEnqueue(&CcPostTickWorkQueue, WorkQueueEntry, 0, 0);
irql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
WorkQueueEntry->WorkQueueLinks.Flink = &CcPostTickWorkQueue;
WorkQueueEntry->WorkQueueLinks.Blink = CcPostTickWorkQueue.Blink;
CcPostTickWorkQueue.Blink->Flink = &WorkQueueEntry->WorkQueueLinks;
CcPostTickWorkQueue.Blink = &WorkQueueEntry->WorkQueueLinks;
LazyWriter.OtherWork = 1;
_InterlockedIncrement(&CcPostTickWorkItemCount);
CcScheduleLazyWriteScan(1, 1);
KeReleaseQueuedSpinLock(LockQueueMasterLock, irql);
result = KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
_InterlockedDecrement(&CcPostTickWorkItemCount);
}
return result;
}
VOID
NTAPI
CcWorkerThread(PVOID Parameter)
{
PWORK_QUEUE_ITEM WorkItem = static_cast<PWORK_QUEUE_ITEM>(Parameter);
PGENERAL_LOOKASIDE LookasideList = nullptr;
PSHARED_CACHE_MAP SharedMap = nullptr;
PWORK_QUEUE_ENTRY WorkEntry = nullptr;
PLIST_ENTRY Entry = nullptr;
PKPRCB Prcb = nullptr;
IO_STATUS_BLOCK IoStatus = {};
KIRQL OldIrql = PASSIVE_LEVEL;
BOOLEAN DropThrottle = FALSE;
BOOLEAN WritePerformed = FALSE;
DPRINT("CcWorkerThread: WorkItem");
IoStatus.Status = STATUS_SUCCESS;
IoStatus.Information = 0;
/* Loop till we have jobs */
while (TRUE)
{
/* Lock queues */
OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
/* If we have to touch throttle, reset it now! */
if (DropThrottle)
{
CcQueueThrottle = FALSE;
DropThrottle = FALSE;
}
if (IoStatus.Information == 0x8A5E)
{
ASSERT(Entry);
if (WorkEntry->Function == WriteBehind)
{
SharedMap = WorkEntry->Parameters.Write.SharedCacheMap;
ASSERT(Entry != &CcFastTeardownWorkQueue);
SharedMap->WriteBehindWorkQueueEntry = WorkEntry;
}
InsertTailList(Entry, &WorkEntry->WorkQueueLinks);
IoStatus.Information = 0;
}
/* Check if we have write to do */
if (!IsListEmpty(&CcFastTeardownWorkQueue))
{
Entry = &CcFastTeardownWorkQueue;
WorkEntry = CONTAINING_RECORD(Entry->Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
ASSERT((WorkEntry->Function == LazyWriteScan) || (WorkEntry->Function == WriteBehind));
}
/* If not, check read queues */
else if (!IsListEmpty(&CcExpressWorkQueue))
{
Entry = &CcExpressWorkQueue;
}
else if (!IsListEmpty(&CcRegularWorkQueue))
{
Entry = &CcRegularWorkQueue;
}
else
{
break;
}
/* Get our work item, if someone is waiting for us to finish
and we're not the only thread in queue then, quit running to let the others do
and throttle so that noone starts till current activity is over
*/
WorkEntry = CONTAINING_RECORD(Entry->Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
if (WorkEntry->Function == SetDone && CcNumberActiveWorkerThreads > 1)
{
CcQueueThrottle = TRUE;
break;
}
if (WorkEntry->Function == WriteBehind)
WorkEntry->Parameters.Write.SharedCacheMap->WriteBehindWorkQueueEntry = NULL;
/* Remove current entry */
RemoveHeadList(Entry);
/* Unlock queues */
KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
/* And handle it */
__try
{
switch (WorkEntry->Function)
{
case ReadAhead: {
CcPerformReadAhead(WorkEntry->Parameters.Read.FileObject);
break;
}
case WriteBehind: {
WritePerformed = TRUE;
PsGetCurrentThread()->MemoryMaker = 1;
CcWriteBehind(WorkEntry->Parameters.Write.SharedCacheMap, &IoStatus);
if (!NT_SUCCESS(IoStatus.Status))
WritePerformed = FALSE;
PsGetCurrentThread()->MemoryMaker = 0;
break;
}
case LazyWriteScan: {
CcLazyWriteScan();
break;
}
case SetDone: {
KeSetEvent(WorkEntry->Parameters.Event.Event, IO_NO_INCREMENT, FALSE);
DropThrottle = TRUE;
break;
}
}
}
__except (CcExceptionFilter(GetExceptionCode()))
{
if (WorkEntry->Function == WriteBehind)
PsGetCurrentThread()->MemoryMaker = 0;
}
/* Handle for WriteBehind */
if (IoStatus.Information == 0x8A5E)
continue;
/* Release the current element and continue */
LookasideList = Prcb->PPLookasideList[5].P;
InterlockedIncrement(&LookasideList->TotalFrees); // Use interlocked increment
if (LookasideList->ListHead.Depth < LookasideList->Depth)
{
InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry);
continue;
}
if (LookasideList->ListHead.Depth < LookasideList->Depth)
{
InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry);
continue;
}
InterlockedIncrement(&LookasideList->FreeMisses); // Use interlocked increment
LookasideList = Prcb->PPLookasideList[5].L;
InterlockedIncrement(&LookasideList->TotalFrees);
if (LookasideList->ListHead.Depth < LookasideList->Depth)
{
InterlockedPushEntrySList(&LookasideList->ListHead, (PSINGLE_LIST_ENTRY)WorkEntry);
continue;
}
InterlockedIncrement(&LookasideList->FreeMisses);
LookasideList->Free(WorkEntry);
}
/* Our thread is available again */
InsertTailList(&CcIdleWorkerThreadList, &WorkItem->List);
/* One less worker */
CcNumberActiveWorkerThreads--;
/* Unlock queues */
KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
/* If there are pending write openations and we have at least 20 dirty pages */
if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20)
{
/* And if we performed a write operation previously,
then stress the system a bit and reschedule a scan to find stuff to write
*/
if (WritePerformed)
CcLazyWriteScan();
}
}
VECTORCALL
CcAllocateWorkQueueEntry(PWORK_QUEUE_ENTRY &workQueueEntry)
{
PKPRCB Prcb = KeGetCurrentPrcb();
PGENERAL_LOOKASIDE LookasideList = nullptr;
PWORK_QUEUE_ENTRY WorkItem = nullptr;
KEVENT Event = NULL;
KIRQL OldIrql = NULL;
NTSTATUS Status = NULL;
/* Allocate a work item */
LookasideList = Prcb->PPLookasideList[6].P;
_InterlockedIncrement(&LookasideList->TotalAllocates);
WorkItem = (PWORK_QUEUE_ENTRY)InterlockedPopEntrySList(&LookasideList->ListHead);
if (!WorkItem)
{
LookasideList->AllocateMisses++;
LookasideList = Prcb->PPLookasideList[5].L;
LookasideList->TotalAllocates++;
WorkItem = (PWORK_QUEUE_ENTRY)InterlockedPopEntrySList(&LookasideList->ListHead);
if (!WorkItem)
{
LookasideList->AllocateMisses++;
WorkItem = (PWORK_QUEUE_ENTRY)LookasideList->Allocate(LookasideList->Type, LookasideList->Size,
LookasideList->Tag);
WorkItem->WorkQueueLinks.Flink = Prcb->Number;
}
}
if (!WorkItem)
{
DBGPRINT("CcAllocateWorkQueueEntry: STATUS_INSUFFICIENT_RESOURCES\n");
return STATUS_INSUFFICIENT_RESOURCES;
}
workQueueEntry = WorkItem;
}
/* Exported Function */
NTSTATUS CcWaitForCurrentLazyWriterActivity()
{
NTSTATUS status; // Status of the operation
PWORK_QUEUE_ENTRY workQueueEntry; // Work queue entry
PLIST_ENTRY blink;
KIRQL irql;
KEVENT event; // Event object
// Allocate a work queue entry
status = CcAllocateWorkQueueEntry(&workQueueEntry);
if (NT_SUCCESS(status)) // Check if the status is a success
{
/* Set the function of the work queue entry*/
workQueueEntry->Function = SetDone;
/* Initialize the event object*/
KeInitializeEvent(&event, NotificationEvent, 0);
/*Set the reason for the notification*/
workQueueEntry->Parameters.Notification.Reason = &event;
/* Acquire the queued spin lock*/
irql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
blink = CcPostTickWorkQueue.Blink;
/*Enqueue the work item*/
workQueueEntry->WorkQueueLinks.Flink = &CcPostTickWorkQueue;
workQueueEntry->WorkQueueLinks.Blink = blink;
blink->Flink = &workQueueEntry->WorkQueueLinks;
CcPostTickWorkQueue.Blink = &workQueueEntry->WorkQueueLinks;
/*Set the other work flag*/
LazyWriter.OtherWork = 1;
/*Increment the work item count*/
_InterlockedIncrement(&CcPostTickWorkItemCount);
/* Schedule Lazy Write Scan */
CcScheduleLazyWriteScan(true);
/*Release the queued spin lock*/
KeReleaseQueuedSpinLock(LockQueueMasterLock, irql);
/*Wait for the single object*/
status = KeWaitForSingleObject(&event, Executive, NULL, NULL, NULL);
// Decrement the work item count
_InterlockedDecrement(&CcPostTickWorkItemCount);
}
/*Return the status of the operation*/
return status;
}