Something went wrong. Try again.
Reactos
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248/* * COPYRIGHT: See COPYING in the top level directory * PROJECT: ReactOS Kernel * FILE: ntoskrnl/ex/pushlock.c * PURPOSE: Pushlock and Cache-Aware Pushlock Implementation * PROGRAMMER: Alex Ionescu (alex.ionescu@reactos.com) */
/* INCLUDES *****************************************************************/
#include <ntoskrnl.h>#define NDEBUG#include <debug.h>
/* DATA **********************************************************************/
ULONG ExPushLockSpinCount = 0;
#undef EX_PUSH_LOCK#undef PEX_PUSH_LOCK
/* PRIVATE FUNCTIONS *********************************************************/
#ifdef _WIN64#define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)#else#define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)#endif
/*++ * @name ExpInitializePushLocks * * The ExpInitializePushLocks routine initialized Pushlock support. * * @param None. * * @return None. * * @remarks The ExpInitializePushLocks routine sets up the spin on SMP machines. * *--*/CODE_SEG("INIT")VOIDNTAPIExpInitializePushLocks(VOID){#ifdef CONFIG_SMP /* Initialize an internal 1024-iteration spin for MP CPUs */ if (KeNumberProcessors > 1) ExPushLockSpinCount = 1024;#endif}
/*++ * @name ExfWakePushLock * * The ExfWakePushLock routine wakes a Pushlock that is in the waiting * state. * * @param PushLock * Pointer to a pushlock that is waiting. * * @param OldValue * Last known value of the pushlock before this routine was called. * * @return None. * * @remarks This is an internal routine; do not call it manually. Only the system * can properly know if the pushlock is ready to be awakened or not. * External callers should use ExfTrytoWakePushLock. * *--*/VOIDFASTCALLExfWakePushLock(PEX_PUSH_LOCK PushLock, EX_PUSH_LOCK OldValue){ EX_PUSH_LOCK NewValue; PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock, FirstWaitBlock, LastWaitBlock; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock; KIRQL OldIrql;
/* Start main wake loop */ for (;;) { /* Sanity checks */ ASSERT(!OldValue.MultipleShared);
/* Check if it's locked */ while (OldValue.Locked) { /* It's not waking anymore */ NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */ ASSERT(!NewValue.Waking); ASSERT(NewValue.Locked); ASSERT(NewValue.Waiting);
/* Write the New Value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) return;
/* Someone changed the value behind our back, update it*/ OldValue = NewValue; }
/* Save the First Block */ FirstWaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS); WaitBlock = FirstWaitBlock;
/* Try to find the last block */ while (TRUE) { /* Get the last wait block */ LastWaitBlock = WaitBlock->Last;
/* Check if we found it */ if (LastWaitBlock) { /* Use it */ WaitBlock = LastWaitBlock; break; }
/* Save the previous block */ PreviousWaitBlock = WaitBlock;
/* Move to next block */ WaitBlock = WaitBlock->Next;
/* Save the previous block */ WaitBlock->Previous = PreviousWaitBlock; }
/* Check if the last Wait Block is not Exclusive or if it's the only one */ PreviousWaitBlock = WaitBlock->Previous; if (!(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE) || !(PreviousWaitBlock)) { /* Destroy the pushlock */ NewValue.Value = 0; ASSERT(!NewValue.Waking);
/* Write the New Value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) break;
/* Someone changed the value behind our back, update it*/ OldValue = NewValue; } else { /* Link the wait blocks */ FirstWaitBlock->Last = PreviousWaitBlock; WaitBlock->Previous = NULL;
/* Sanity checks */ ASSERT(FirstWaitBlock != WaitBlock); ASSERT(PushLock->Waiting);
/* Remove waking bit from pushlock */ InterlockedAndPointer(&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
/* Leave the loop */ break; } }
/* Check if there's a previous block */ OldIrql = DISPATCH_LEVEL; if (WaitBlock->Previous) { /* Raise to Dispatch */ KeRaiseIrql(DISPATCH_LEVEL, &OldIrql); }
/* Signaling loop */ for (;;) { /* Get the previous Wait block */ PreviousWaitBlock = WaitBlock->Previous;
/* Sanity check */ ASSERT(!WaitBlock->Signaled);
#if DBG /* We are about to get signaled */ WaitBlock->Signaled = TRUE;#endif
/* Set the Wait Bit in the Wait Block */ if (!InterlockedBitTestAndReset(&WaitBlock->Flags, 1)) { /* Nobody signaled us, so do it */ KeSignalGateBoostPriority(&WaitBlock->WakeGate); }
/* Set the wait block and check if there still is one to loop*/ WaitBlock = PreviousWaitBlock; if (!WaitBlock) break; }
/* Check if we have to lower back the IRQL */ if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);}
/*++ * @name ExpOptimizePushLockList * * The ExpOptimizePushLockList routine optimizes the list of waiters * associated to a pushlock's wait block. * * @param PushLock * Pointer to a pushlock whose waiter list needs to be optimized. * * @param OldValue * Last known value of the pushlock before this routine was called. * * @return None. * * @remarks At the end of the optimization, the pushlock will also be wakened. * *--*/VOIDFASTCALLExpOptimizePushLockList(PEX_PUSH_LOCK PushLock, EX_PUSH_LOCK OldValue){ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock, PreviousWaitBlock, FirstWaitBlock; EX_PUSH_LOCK NewValue;
/* Start main loop */ for (;;) { /* Check if we've been unlocked */ if (!OldValue.Locked) { /* Wake us up and leave */ ExfWakePushLock(PushLock, OldValue); break; }
/* Get the wait block */ WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
/* Loop the blocks */ FirstWaitBlock = WaitBlock; while (TRUE) { /* Get the last wait block */ LastWaitBlock = WaitBlock->Last; if (LastWaitBlock) { /* Set this as the new last block, we're done */ FirstWaitBlock->Last = LastWaitBlock; break; }
/* Save the block */ PreviousWaitBlock = WaitBlock;
/* Get the next block */ WaitBlock = WaitBlock->Next;
/* Save the previous */ WaitBlock->Previous = PreviousWaitBlock; }
/* Remove the wake bit */ NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */ ASSERT(NewValue.Locked); ASSERT(!NewValue.Waking);
/* Update the value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr);
/* If we updated correctly, leave */ if (NewValue.Value == OldValue.Value) break;
/* Update value */ OldValue = NewValue; }}
/*++ * @name ExTimedWaitForUnblockPushLock * * The ExTimedWaitForUnblockPushLock routine waits for a pushlock * to be unblocked, for a specified internal. * * @param PushLock * Pointer to a pushlock whose waiter list needs to be optimized. * * @param WaitBlock * Pointer to the pushlock's wait block. * * @param Timeout * Amount of time to wait for this pushlock to be unblocked. * * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error * code returned by KeWaitForSingleObject. * * @remarks If the wait fails, then a manual unblock is attempted. * *--*/NTSTATUSFASTCALLExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock, IN PVOID WaitBlock, IN PLARGE_INTEGER Timeout){ NTSTATUS Status;
/* Initialize the wait event */ KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->WakeEvent, SynchronizationEvent, FALSE);
#ifdef CONFIG_SMP /* Spin on the push lock if necessary */ if (ExPushLockSpinCount) { ULONG i = ExPushLockSpinCount;
do { /* Check if we got lucky and can leave early */ if (!(*(volatile LONG *)&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags & EX_PUSH_LOCK_WAITING)) return STATUS_SUCCESS;
YieldProcessor(); } while (--i); }#endif
/* Now try to remove the wait bit */ if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags, EX_PUSH_LOCK_FLAGS_WAIT_V)) { /* Nobody removed it already, let's do a full wait */ Status = KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)-> WakeEvent, WrPushLock, KernelMode, FALSE, Timeout); /* Check if the wait was satisfied */ if (Status != STATUS_SUCCESS) { /* Try unblocking the pushlock if it was not */ ExfUnblockPushLock(PushLock, WaitBlock); } } else { /* Someone beat us to it, no need to wait */ Status = STATUS_SUCCESS; }
/* Return status */ return Status;}
/*++ * @name ExWaitForUnblockPushLock * * The ExWaitForUnblockPushLock routine waits for a pushlock * to be unblocked, for a specified internal. * * @param PushLock * Pointer to a pushlock whose waiter list needs to be optimized. * * @param WaitBlock * Pointer to the pushlock's wait block. * * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error * code returned by KeWaitForSingleObject. * * @remarks If the wait fails, then a manual unblock is attempted. * *--*/VOIDFASTCALLExWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock, IN PVOID WaitBlock){ /* Call the timed function with no timeout */ ExTimedWaitForUnblockPushLock(PushLock, WaitBlock, NULL);}
/*++ * @name ExBlockPushLock * * The ExBlockPushLock routine blocks a pushlock. * * @param PushLock * Pointer to a pushlock whose waiter list needs to be optimized. * * @param WaitBlock * Pointer to the pushlock's wait block. * * @return None. * * @remarks None. * *--*/VOIDFASTCALLExBlockPushLock(PEX_PUSH_LOCK PushLock, PVOID pWaitBlock){ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = pWaitBlock; EX_PUSH_LOCK NewValue, OldValue;
/* Detect invalid wait block alignment */ ASSERT(((ULONG_PTR)pWaitBlock & 0xF) == 0);
/* Set the waiting bit */ WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT;
/* Get the old value */ OldValue = *PushLock;
/* Start block loop */ for (;;) { /* Link the wait blocks */ WaitBlock->Next = OldValue.Ptr;
/* Set the new wait block value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, WaitBlock, OldValue.Ptr); if (OldValue.Ptr == NewValue.Ptr) break;
/* Try again with the new value */ OldValue = NewValue; }}
/* PUBLIC FUNCTIONS **********************************************************/
/*++ * @name ExAcquirePushLockExclusive * @implemented NT5.1 * * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. * * @params PushLock * Pointer to the pushlock which is to be acquired. * * @return None. * * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. * This macro should usually be paired up with KeAcquireCriticalRegion. * *--*/VOIDFASTCALLExfAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK OldValue = *PushLock, NewValue, TempValue; BOOLEAN NeedWake; EX_PUSH_LOCK_WAIT_BLOCK Block; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */ for (;;) { /* Check if it's unlocked */ if (!OldValue.Locked) { /* Lock it */ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK; ASSERT(NewValue.Locked);
/* Set the new value */ if (InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr) != OldValue.Ptr) { /* Retry */ OldValue = *PushLock; continue; }
/* Break out of the loop */ break; } else { /* We'll have to create a Waitblock */ WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_EXCLUSIVE | EX_PUSH_LOCK_FLAGS_WAIT; WaitBlock->Previous = NULL; NeedWake = FALSE;
/* Check if there is already a waiter */ if (OldValue.Waiting) { /* Nobody is the last waiter yet */ WaitBlock->Last = NULL;
/* We are an exclusive waiter */ WaitBlock->ShareCount = 0;
/* Set the current Wait Block pointer */ WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)( OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Point to ours */ NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_MULTIPLE_SHARED) | EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_WAKING | EX_PUSH_LOCK_WAITING | (ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */ if (!OldValue.Waking) NeedWake = TRUE; } else { /* We are the first waiter, so loop the wait block */ WaitBlock->Last = WaitBlock;
/* Set the share count */ WaitBlock->ShareCount = (LONG)OldValue.Shared;
/* Check if someone is sharing this pushlock */ if (OldValue.Shared > 1) { /* Point to our wait block */ NewValue.Value = EX_PUSH_LOCK_MULTIPLE_SHARED | EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_WAITING | (ULONG_PTR)WaitBlock; } else { /* No shared count */ WaitBlock->ShareCount = 0;
/* Point to our wait block */ NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_WAITING | (ULONG_PTR)WaitBlock; } }
#if DBG /* Setup the Debug Wait Block */ WaitBlock->Signaled = 0; WaitBlock->OldValue = OldValue; WaitBlock->NewValue = NewValue; WaitBlock->PushLock = PushLock;#endif
/* Sanity check */ ASSERT(NewValue.Waiting); ASSERT(NewValue.Locked);
/* Write the new value */ TempValue = NewValue; NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value != OldValue.Value) { /* Retry */ OldValue = *PushLock; continue; }
/* Check if the pushlock needed waking */ if (NeedWake) { /* Scan the Waiters and Wake PushLocks */ ExpOptimizePushLockList(PushLock, TempValue); }
/* Set up the Wait Gate */ KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP /* Now spin on the push lock if necessary */ if (ExPushLockSpinCount) { ULONG i = ExPushLockSpinCount;
do { if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING)) break;
YieldProcessor(); } while (--i); }#endif
/* Now try to remove the wait bit */ if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1)) { /* Nobody removed it already, let's do a full wait */ KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode); ASSERT(WaitBlock->Signaled); }
/* We shouldn't be shared anymore */ ASSERT((WaitBlock->ShareCount == 0));
/* Loop again */ OldValue = NewValue; } }}
/*++ * @name ExAcquirePushLockShared * @implemented NT5.1 * * The ExAcquirePushLockShared routine acquires a shared PushLock. * * @params PushLock * Pointer to the pushlock which is to be acquired. * * @return None. * * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. * This macro should usually be paired up with KeAcquireCriticalRegion. * *--*/VOIDFASTCALLExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK OldValue = *PushLock, NewValue; BOOLEAN NeedWake; EX_PUSH_LOCK_WAIT_BLOCK Block; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */ for (;;) { /* Check if it's unlocked or if it's waiting without any sharers */ if (!(OldValue.Locked) || (!(OldValue.Waiting) && (OldValue.Shared > 0))) { /* Check if anyone is waiting on it */ if (!OldValue.Waiting) { /* Increase the share count and lock it */ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK; NewValue.Shared++; } else { /* Simply set the lock bit */ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK; }
/* Sanity check */ ASSERT(NewValue.Locked);
/* Set the new value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value != OldValue.Value) { /* Retry */ OldValue = *PushLock; continue; }
/* Break out of the loop */ break; } else { /* We'll have to create a Waitblock */ WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT; WaitBlock->ShareCount = 0; NeedWake = FALSE; WaitBlock->Previous = NULL;
/* Check if there is already a waiter */ if (OldValue.Waiting) { /* Set the current Wait Block pointer */ WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)( OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Nobody is the last waiter yet */ WaitBlock->Last = NULL;
/* Point to ours */ NewValue.Value = (OldValue.Value & (EX_PUSH_LOCK_MULTIPLE_SHARED | EX_PUSH_LOCK_LOCK)) | EX_PUSH_LOCK_WAKING | EX_PUSH_LOCK_WAITING | (ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */ if (!OldValue.Waking) NeedWake = TRUE; } else { /* We are the first waiter, so loop the wait block */ WaitBlock->Last = WaitBlock;
/* Point to our wait block */ NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_PTR_BITS) | EX_PUSH_LOCK_WAITING | (ULONG_PTR)WaitBlock; }
/* Sanity check */ ASSERT(NewValue.Waiting);
#if DBG /* Setup the Debug Wait Block */ WaitBlock->Signaled = 0; WaitBlock->OldValue = OldValue; WaitBlock->NewValue = NewValue; WaitBlock->PushLock = PushLock;#endif
/* Write the new value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Ptr != OldValue.Ptr) { /* Retry */ OldValue = *PushLock; continue; }
/* Update the value now */ OldValue = NewValue;
/* Check if the pushlock needed waking */ if (NeedWake) { /* Scan the Waiters and Wake PushLocks */ ExpOptimizePushLockList(PushLock, OldValue); }
/* Set up the Wait Gate */ KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP /* Now spin on the push lock if necessary */ if (ExPushLockSpinCount) { ULONG i = ExPushLockSpinCount;
do { if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING)) break;
YieldProcessor(); } while (--i); }#endif
/* Now try to remove the wait bit */ if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1)) { /* Fast-path did not work, we need to do a full wait */ KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode); ASSERT(WaitBlock->Signaled); }
/* We shouldn't be shared anymore */ ASSERT((WaitBlock->ShareCount == 0)); } }}
/*++ * @name ExfReleasePushLock * @implemented NT5.1 * * The ExReleasePushLock routine releases a previously acquired PushLock. * * * @params PushLock * Pointer to a previously acquired pushlock. * * @return None. * * @remarks Callers of ExfReleasePushLock must be running at IRQL <= APC_LEVEL. * This macro should usually be paired up with KeLeaveCriticalRegion. * *--*/VOIDFASTCALLExfReleasePushLock(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK OldValue = *PushLock, NewValue, WakeValue; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock;
/* Sanity check */ ASSERT(OldValue.Locked);
/* Start main loop */ while (TRUE) { /* Check if someone is waiting on the lock */ if (!OldValue.Waiting) { /* Check if it's shared */ if (OldValue.Shared > 1) { /* Write the Old Value but decrease share count */ NewValue = OldValue; NewValue.Shared--; } else { /* Simply clear the lock */ NewValue.Value = 0; }
/* Write the New Value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) return;
/* Did it enter a wait state? */ OldValue = NewValue; } else { /* Ok, we do know someone is waiting on it. Are there more then one? */ if (OldValue.MultipleShared) { /* Get the wait block */ WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
/* Loop until we find the last wait block */ while (TRUE) { /* Get the last wait block */ LastWaitBlock = WaitBlock->Last;
/* Did it exist? */ if (LastWaitBlock) { /* Choose it */ WaitBlock = LastWaitBlock; break; }
/* Keep searching */ WaitBlock = WaitBlock->Next; }
/* Make sure the Share Count is above 0 */ if (WaitBlock->ShareCount > 0) { /* This shouldn't be an exclusive wait block */ ASSERT(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
/* Do the decrease and check if the lock isn't shared anymore */ if (InterlockedDecrement(&WaitBlock->ShareCount) > 0) return; } }
/* * If nobody was waiting on the block, then we possibly reduced the number * of times the pushlock was shared, and we unlocked it. * If someone was waiting, and more then one person is waiting, then we * reduced the number of times the pushlock is shared in the wait block. * Therefore, at this point, we can now 'satisfy' the wait. */ for (;;) { /* Now we need to see if it's waking */ if (OldValue.Waking) { /* Remove the lock and multiple shared bits */ NewValue.Value = OldValue.Value; NewValue.MultipleShared = FALSE; NewValue.Locked = FALSE;
/* Sanity check */ ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) return; } else { /* Remove the lock and multiple shared bits */ NewValue.Value = OldValue.Value; NewValue.MultipleShared = FALSE; NewValue.Locked = FALSE;
/* It's not already waking, so add the wake bit */ NewValue.Waking = TRUE;
/* Sanity check */ ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */ WakeValue = NewValue; NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value != OldValue.Value) continue;
/* The write was successful. The pushlock is Unlocked and Waking */ ExfWakePushLock(PushLock, WakeValue); return; } } } }}
/*++ * @name ExfReleasePushLockShared * @implemented NT5.2 * * The ExfReleasePushLockShared macro releases a previously acquired PushLock. * * @params PushLock * Pointer to a previously acquired pushlock. * * @return None. * * @remarks Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. * This macro should usually be paired up with KeLeaveCriticalRegion. * *--*/VOIDFASTCALLExfReleasePushLockShared(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK OldValue = *PushLock, NewValue, WakeValue; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock;
/* Check if someone is waiting on the lock */ while (!OldValue.Waiting) { /* Check if it's shared */ if (OldValue.Shared > 1) { /* Write the Old Value but decrease share count */ NewValue = OldValue; NewValue.Shared--; } else { /* Simply clear the lock */ NewValue.Value = 0; }
/* Write the New Value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) return;
/* Did it enter a wait state? */ OldValue = NewValue; }
/* Ok, we do know someone is waiting on it. Are there more then one? */ if (OldValue.MultipleShared) { /* Get the wait block */ WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
/* Loop until we find the last wait block */ while (TRUE) { /* Get the last wait block */ LastWaitBlock = WaitBlock->Last;
/* Did it exist? */ if (LastWaitBlock) { /* Choose it */ WaitBlock = LastWaitBlock; break; }
/* Keep searching */ WaitBlock = WaitBlock->Next; }
/* Sanity checks */ ASSERT(WaitBlock->ShareCount > 0); ASSERT(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
/* Do the decrease and check if the lock isn't shared anymore */ if (InterlockedDecrement(&WaitBlock->ShareCount) > 0) return; }
/* * If nobody was waiting on the block, then we possibly reduced the number * of times the pushlock was shared, and we unlocked it. * If someone was waiting, and more then one person is waiting, then we * reduced the number of times the pushlock is shared in the wait block. * Therefore, at this point, we can now 'satisfy' the wait. */ for (;;) { /* Now we need to see if it's waking */ if (OldValue.Waking) { /* Remove the lock and multiple shared bits */ NewValue.Value = OldValue.Value; NewValue.MultipleShared = FALSE; NewValue.Locked = FALSE;
/* Sanity check */ ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value == OldValue.Value) return; } else { /* Remove the lock and multiple shared bits */ NewValue.Value = OldValue.Value; NewValue.MultipleShared = FALSE; NewValue.Locked = FALSE;
/* It's not already waking, so add the wake bit */ NewValue.Waking = TRUE;
/* Sanity check */ ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */ WakeValue = NewValue; NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr); if (NewValue.Value != OldValue.Value) continue;
/* The write was successful. The pushlock is Unlocked and Waking */ ExfWakePushLock(PushLock, WakeValue); return; } }}
/*++ * ExfReleasePushLockExclusive * @implemented NT5.2 * * The ExfReleasePushLockExclusive routine releases a previously * exclusively acquired PushLock. * * @params PushLock * Pointer to a previously acquired pushlock. * * @return None. * * @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. * This macro should usually be paired up with KeLeaveCriticalRegion. * *--*/VOIDFASTCALLExfReleasePushLockExclusive(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK NewValue, WakeValue; EX_PUSH_LOCK OldValue = *PushLock;
/* Loop until we can change */ for (;;) { /* Sanity checks */ ASSERT(OldValue.Locked); ASSERT(OldValue.Waiting || OldValue.Shared == 0);
/* Check if it's waiting and not yet waking */ if ((OldValue.Waiting) && !(OldValue.Waking)) { /* Remove the lock bit, and add the wake bit */ NewValue.Value = (OldValue.Value &~ EX_PUSH_LOCK_LOCK) | EX_PUSH_LOCK_WAKING;
/* Sanity check */ ASSERT(NewValue.Waking && !NewValue.Locked);
/* Write the New Value. Save our original value for waking */ WakeValue = NewValue; NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr);
/* Check if the value changed behind our back */ if (NewValue.Value == OldValue.Value) { /* Wake the Pushlock */ ExfWakePushLock(PushLock, WakeValue); break; } } else { /* A simple unlock */ NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_LOCK;
/* Sanity check */ ASSERT(NewValue.Waking || !NewValue.Waiting);
/* Write the New Value */ NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr);
/* Check if the value changed behind our back */ if (NewValue.Value == OldValue.Value) break; }
/* Loop again */ OldValue = NewValue; }}
/*++ * @name ExfTryToWakePushLock * @implemented NT5.2 * * The ExfTryToWakePushLock attempts to wake a waiting pushlock. * * @param PushLock * Pointer to a PushLock which is in the wait state. * * @return None. * * @remarks The pushlock must be in a wait state and must not be already waking. * *--*/VOIDFASTCALLExfTryToWakePushLock(PEX_PUSH_LOCK PushLock){ EX_PUSH_LOCK OldValue = *PushLock, NewValue;
/* * If the Pushlock is not waiting on anything, or if it's already waking up * and locked, don't do anything */ if ((OldValue.Waking) || (OldValue.Locked) || !(OldValue.Waiting)) return;
/* Make it Waking */ NewValue = OldValue; NewValue.Waking = TRUE;
/* Write the New Value */ if (InterlockedCompareExchangePointer(&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr) == OldValue.Ptr) { /* Wake the Pushlock */ ExfWakePushLock(PushLock, NewValue); }}
/*++ * @name ExfUnblockPushLock * @implemented NT5.1 * * The ExfUnblockPushLock routine unblocks a previously blocked PushLock. * * @param PushLock * Pointer to a previously blocked PushLock. * * @return None. * * @remarks Callers of ExfUnblockPushLock can be running at any IRQL. * *--*/VOIDFASTCALLExfUnblockPushLock(PEX_PUSH_LOCK PushLock, PVOID CurrentWaitBlock){ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock; KIRQL OldIrql = DISPATCH_LEVEL;
/* Get the wait block and erase the previous one */ WaitBlock = InterlockedExchangePointer(&PushLock->Ptr, NULL); if (WaitBlock) { /* Check if there is a linked pushlock and raise IRQL appropriately */ if (WaitBlock->Next) KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
/* Start block loop */ while (WaitBlock) { /* Get the next block */ NextWaitBlock = WaitBlock->Next;
/* Remove the wait flag from the Wait block */ if (!InterlockedBitTestAndReset(&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_WAIT_V)) { /* Nobody removed the flag before us, so signal the event */ KeSetEventBoostPriority(&WaitBlock->WakeEvent, NULL); }
/* Try the next one */ WaitBlock = NextWaitBlock; }
/* Lower IRQL if needed */ if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql); }
/* Check if we got a wait block that's pending */ if ((CurrentWaitBlock) && (((PEX_PUSH_LOCK_WAIT_BLOCK)CurrentWaitBlock)->Flags & EX_PUSH_LOCK_FLAGS_WAIT)) { /* Wait for the pushlock to be unblocked */ ExWaitForUnblockPushLock(PushLock, CurrentWaitBlock); }}