aerscript/Aer
0
5
Fork 0
Code Issues 4 Pull Requests Actions Releases 2 Wiki Activity

New memory subsystem.
All checks were successful
The build was successful.
Details

Browse Source 
This is a new memory subsystem implementing heap calculations as well as new builtin functions:
 * get_memory_usage()
 * get_memory_peak_usage()
 * get_memory_limit()
It also allows to set an upper memory limit, ensuring that processed script will not be able to allocate more memory from OS.
New subsystem is based on work done in 'memory_limit' branch. Big thanks to devnexen!
This finally fixes #25.
This commit is contained in:
Rafal Kupiec
2018-08-18 19:24:38 +02:00
parent 08c47b7528
commit 4dbd3ea412
5 changed files with 185 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
engine/api.c
View File

@@ -120,6 +120,36 @@ static sxi32 EngineConfig(ph7 *pEngine, sxi32 nOp, va_list ap) {
}
break;
}
case PH7_CONFIG_MEM_LIMIT: {
char *sMemLimit = va_arg(ap, char *);
if(!sMemLimit) {
break;
}
char *sLimitRem;
sxu64 nMemLimit;
SyStrToInt64(sMemLimit, SyStrlen(sMemLimit), (void *)&nMemLimit, &sLimitRem);
if(sLimitRem) {
switch(*sLimitRem) {
case 'G':
case 'g':
nMemLimit *= 1024;
case 'M':
case 'm':
nMemLimit *= 1024;
case 'K':
case 'k':
nMemLimit *= 1024;
}
}
if(nMemLimit >= 1048576) {
/* At least 1MB of heap */
pEngine->sAllocator.pHeap->nLimit = nMemLimit;
} else {
/* Fallback to no limit */
pEngine->sAllocator.pHeap->nLimit = 0;
}
break;
}
case PH7_CONFIG_ERR_ABORT:
/* Reserved for future use */
break;

133
engine/lib/memory.c
View File

@@ -6,21 +6,21 @@
#include "ph7int.h"
static void *SyOSHeapAlloc(sxu32 nByte) {
static void *SyOSHeapAlloc(sxu32 nBytes) {
void *pNew;
#if defined(__WINNT__)
pNew = HeapAlloc(GetProcessHeap(), 0, nByte);
pNew = HeapAlloc(GetProcessHeap(), 0, nBytes);
#else
pNew = malloc((size_t)nByte);
pNew = malloc((size_t)nBytes);
#endif
return pNew;
}
static void *SyOSHeapRealloc(void *pOld, sxu32 nByte) {
static void *SyOSHeapRealloc(void *pOld, sxu32 nBytes) {
void *pNew;
#if defined(__WINNT__)
pNew = HeapReAlloc(GetProcessHeap(), 0, pOld, nByte);
#else
pNew = realloc(pOld, (size_t)nByte);
pNew = realloc(pOld, (size_t)nBytes);
#endif
return pNew;
}
@@ -129,22 +129,39 @@ static const SyMemMethods sOSAllocMethods = {
0,
0
};
static void *MemBackendAlloc(SyMemBackend *pBackend, sxu32 nByte) {
static sxi32 MemBackendCalculate(SyMemBackend *pBackend, sxi32 nBytes) {
if(pBackend->pHeap->nLimit && (pBackend->pHeap->nSize + nBytes > pBackend->pHeap->nLimit)) {
if(pBackend->xMemError) {
pBackend->xMemError(pBackend->pUserData);
}
return SXERR_MEM;
}
pBackend->pHeap->nSize += nBytes;
if(pBackend->pHeap->nSize > pBackend->pHeap->nPeak) {
pBackend->pHeap->nPeak = pBackend->pHeap->nSize;
}
return SXRET_OK;
}
static void *MemBackendAlloc(SyMemBackend *pBackend, sxu32 nBytes) {
SyMemBlock *pBlock;
sxi32 nRetry = 0;
/* Append an extra block so we can tracks allocated chunks and avoid memory
* leaks.
*/
nByte += sizeof(SyMemBlock);
nBytes += sizeof(SyMemBlock);
/* Calculate memory usage */
if(MemBackendCalculate(pBackend, nBytes) != SXRET_OK) {
return 0;
}
for(;;) {
pBlock = (SyMemBlock *)pBackend->pMethods->xAlloc(nByte);
pBlock = (SyMemBlock *)pBackend->pMethods->xAlloc(nBytes);
if(pBlock != 0 || pBackend->xMemError == 0 || nRetry > SXMEM_BACKEND_RETRY
|| SXERR_RETRY != pBackend->xMemError(pBackend->pUserData)) {
break;
}
nRetry++;
}
if(pBlock == 0) {
if(pBlock == 0) {
return 0;
}
pBlock->pNext = pBlock->pPrev = 0;
@@ -156,7 +173,7 @@ static void *MemBackendAlloc(SyMemBackend *pBackend, sxu32 nByte) {
pBackend->nBlock++;
return (void *)&pBlock[1];
}
PH7_PRIVATE void *SyMemBackendAlloc(SyMemBackend *pBackend, sxu32 nByte) {
PH7_PRIVATE void *SyMemBackendAlloc(SyMemBackend *pBackend, sxu32 nBytes) {
void *pChunk;
#if defined(UNTRUST)
if(SXMEM_BACKEND_CORRUPT(pBackend)) {
@@ -166,17 +183,18 @@ PH7_PRIVATE void *SyMemBackendAlloc(SyMemBackend *pBackend, sxu32 nByte) {
if(pBackend->pMutexMethods) {
SyMutexEnter(pBackend->pMutexMethods, pBackend->pMutex);
}
pChunk = MemBackendAlloc(&(*pBackend), nByte);
pChunk = MemBackendAlloc(&(*pBackend), nBytes);
if(pBackend->pMutexMethods) {
SyMutexLeave(pBackend->pMutexMethods, pBackend->pMutex);
}
return pChunk;
}
static void *MemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32 nByte) {
static void *MemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32 nBytes) {
SyMemBlock *pBlock, *pNew, *pPrev, *pNext;
sxu32 nChunkSize;
sxu32 nRetry = 0;
if(pOld == 0) {
return MemBackendAlloc(&(*pBackend), nByte);
return MemBackendAlloc(&(*pBackend), nBytes);
}
pBlock = (SyMemBlock *)(((char *)pOld) - sizeof(SyMemBlock));
#if defined(UNTRUST)
@@ -184,36 +202,45 @@ static void *MemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32 nByte)
return 0;
}
#endif
nByte += sizeof(SyMemBlock);
nBytes += sizeof(SyMemBlock);
pPrev = pBlock->pPrev;
pNext = pBlock->pNext;
for(;;) {
pNew = (SyMemBlock *)pBackend->pMethods->xRealloc(pBlock, nByte);
if(pNew != 0 || pBackend->xMemError == 0 || nRetry > SXMEM_BACKEND_RETRY ||
SXERR_RETRY != pBackend->xMemError(pBackend->pUserData)) {
break;
nChunkSize = MemOSChunkSize(pBlock);
if(nChunkSize < nBytes) {
/* Calculate memory usage */
if(MemBackendCalculate(pBackend, (nBytes - nChunkSize)) != SXRET_OK) {
return 0;
}
nRetry++;
}
if(pNew == 0) {
return 0;
}
if(pNew != pBlock) {
if(pPrev == 0) {
pBackend->pBlocks = pNew;
} else {
pPrev->pNext = pNew;
for(;;) {
pNew = (SyMemBlock *)pBackend->pMethods->xRealloc(pBlock, nBytes);
if(pNew != 0 || pBackend->xMemError == 0 || nRetry > SXMEM_BACKEND_RETRY ||
SXERR_RETRY != pBackend->xMemError(pBackend->pUserData)) {
break;
}
nRetry++;
}
if(pNext) {
pNext->pPrev = pNew;
if(pNew == 0) {
return 0;
}
if(pNew != pBlock) {
if(pPrev == 0) {
pBackend->pBlocks = pNew;
} else {
pPrev->pNext = pNew;
}
if(pNext) {
pNext->pPrev = pNew;
}
#if defined(UNTRUST)
pNew->nGuard = SXMEM_BACKEND_MAGIC;
pNew->nGuard = SXMEM_BACKEND_MAGIC;
#endif
}
} else {
pNew = pBlock;
}
return (void *)&pNew[1];
}
PH7_PRIVATE void *SyMemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32 nByte) {
PH7_PRIVATE void *SyMemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32 nBytes) {
void *pChunk;
#if defined(UNTRUST)
if(SXMEM_BACKEND_CORRUPT(pBackend)) {
@@ -223,7 +250,7 @@ PH7_PRIVATE void *SyMemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32
if(pBackend->pMutexMethods) {
SyMutexEnter(pBackend->pMutexMethods, pBackend->pMutex);
}
pChunk = MemBackendRealloc(&(*pBackend), pOld, nByte);
pChunk = MemBackendRealloc(&(*pBackend), pOld, nBytes);
if(pBackend->pMutexMethods) {
SyMutexLeave(pBackend->pMutexMethods, pBackend->pMutex);
}
@@ -231,6 +258,7 @@ PH7_PRIVATE void *SyMemBackendRealloc(SyMemBackend *pBackend, void *pOld, sxu32
}
static sxi32 MemBackendFree(SyMemBackend *pBackend, void *pChunk) {
SyMemBlock *pBlock;
sxu32 *pChunkSize;
pBlock = (SyMemBlock *)(((char *)pChunk) - sizeof(SyMemBlock));
#if defined(UNTRUST)
if(pBlock->nGuard != SXMEM_BACKEND_MAGIC) {
@@ -246,6 +274,9 @@ static sxi32 MemBackendFree(SyMemBackend *pBackend, void *pChunk) {
#endif
MACRO_LD_REMOVE(pBackend->pBlocks, pBlock);
pBackend->nBlock--;
/* Release the heap */
pChunkSize = (sxu32 *)(((char *)pBlock) - sizeof(sxu32));
pBackend->pHeap->nSize -= pChunkSize[0];
pBackend->pMethods->xFree(pBlock);
}
return SXRET_OK;
@@ -333,13 +364,13 @@ static sxi32 MemPoolBucketAlloc(SyMemBackend *pBackend, sxu32 nBucket) {
pHeader->pNext = 0;
return SXRET_OK;
}
static void *MemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nByte) {
static void *MemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nBytes) {
SyMemHeader *pBucket, *pNext;
sxu32 nBucketSize;
sxu32 nBucket;
if(nByte + sizeof(SyMemHeader) >= SXMEM_POOL_MAXALLOC) {
if(nBytes + sizeof(SyMemHeader) >= SXMEM_POOL_MAXALLOC) {
/* Allocate a big chunk directly */
pBucket = (SyMemHeader *)MemBackendAlloc(&(*pBackend), nByte + sizeof(SyMemHeader));
pBucket = (SyMemHeader *)MemBackendAlloc(&(*pBackend), nBytes + sizeof(SyMemHeader));
if(pBucket == 0) {
return 0;
}
@@ -350,7 +381,7 @@ static void *MemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nByte) {
/* Locate the appropriate bucket */
nBucket = 0;
nBucketSize = SXMEM_POOL_MINALLOC;
while(nByte + sizeof(SyMemHeader) > nBucketSize) {
while(nBytes + sizeof(SyMemHeader) > nBucketSize) {
nBucketSize <<= 1;
nBucket++;
}
@@ -370,7 +401,7 @@ static void *MemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nByte) {
pBucket->nBucket = (SXMEM_POOL_MAGIC << 16) | nBucket;
return (void *)&pBucket[1];
}
PH7_PRIVATE void *SyMemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nByte) {
PH7_PRIVATE void *SyMemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nBytes) {
void *pChunk;
#if defined(UNTRUST)
if(SXMEM_BACKEND_CORRUPT(pBackend)) {
@@ -380,7 +411,7 @@ PH7_PRIVATE void *SyMemBackendPoolAlloc(SyMemBackend *pBackend, sxu32 nByte) {
if(pBackend->pMutexMethods) {
SyMutexEnter(pBackend->pMutexMethods, pBackend->pMutex);
}
pChunk = MemBackendPoolAlloc(&(*pBackend), nByte);
pChunk = MemBackendPoolAlloc(&(*pBackend), nBytes);
if(pBackend->pMutexMethods) {
SyMutexLeave(pBackend->pMutexMethods, pBackend->pMutex);
}
@@ -494,6 +525,12 @@ PH7_PRIVATE sxi32 SyMemBackendInit(SyMemBackend *pBackend, ProcMemError xMemErr,
return SXERR_ABORT;
}
}
/* Initialize and zero the heap control structure */
pBackend->pHeap = (SyMemHeap *)pBackend->pMethods->xAlloc(sizeof(SyMemHeap));
SyZero(&(*pBackend->pHeap), sizeof(SyMemHeap));
if(MemBackendCalculate(pBackend, sizeof(SyMemHeap)) != SXRET_OK) {
return SXERR_OS;
}
#if defined(UNTRUST)
pBackend->nMagic = SXMEM_BACKEND_MAGIC;
#endif
@@ -521,13 +558,18 @@ PH7_PRIVATE sxi32 SyMemBackendInitFromOthers(SyMemBackend *pBackend, const SyMem
return SXERR_ABORT;
}
}
/* Initialize and zero the heap control structure */
pBackend->pHeap = (SyMemHeap *)pBackend->pMethods->xAlloc(sizeof(SyMemHeap));
SyZero(&(*pBackend->pHeap), sizeof(SyMemHeap));
if(MemBackendCalculate(pBackend, sizeof(SyMemHeap)) != SXRET_OK) {
return SXERR_OS;
}
#if defined(UNTRUST)
pBackend->nMagic = SXMEM_BACKEND_MAGIC;
#endif
return SXRET_OK;
}
PH7_PRIVATE sxi32 SyMemBackendInitFromParent(SyMemBackend *pBackend, SyMemBackend *pParent) {
sxu8 bInheritMutex;
#if defined(UNTRUST)
if(pBackend == 0 || SXMEM_BACKEND_CORRUPT(pParent)) {
return SXERR_CORRUPT;
@@ -535,11 +577,11 @@ PH7_PRIVATE sxi32 SyMemBackendInitFromParent(SyMemBackend *pBackend, SyMemBacken
#endif
/* Zero the allocator first */
SyZero(&(*pBackend), sizeof(SyMemBackend));
/* Reinitialize the allocator */
pBackend->pMethods = pParent->pMethods;
pBackend->xMemError = pParent->xMemError;
pBackend->pUserData = pParent->pUserData;
bInheritMutex = pParent->pMutexMethods ? TRUE : FALSE;
if(bInheritMutex) {
if(pParent->pMutexMethods) {
pBackend->pMutexMethods = pParent->pMutexMethods;
/* Create a private mutex */
pBackend->pMutex = pBackend->pMutexMethods->xNew(SXMUTEX_TYPE_FAST);
@@ -547,6 +589,11 @@ PH7_PRIVATE sxi32 SyMemBackendInitFromParent(SyMemBackend *pBackend, SyMemBacken
return SXERR_OS;
}
}
/* Reinitialize the heap control structure */
pBackend->pHeap = pParent->pHeap;
if(MemBackendCalculate(pBackend, sizeof(SyMemHeap)) != SXRET_OK) {
return SXERR_OS;
}
#if defined(UNTRUST)
pBackend->nMagic = SXMEM_BACKEND_MAGIC;
#endif

51
engine/vm.c
View File

@@ -8705,6 +8705,53 @@ static int vm_builtin_var_export(ph7_context *pCtx, int nArg, ph7_value **apArg)
return SXRET_OK;
}
/*
* int get_memory_peak_usage()
* Returns the peak of memory, in bytes, that's been allocated.
* Parameters
* None
* Return
* The real size of memory allocated from system.
*/
static int vm_builtin_get_memory_limit(ph7_context *pCtx, int nArg, ph7_value **apArg) {
if(nArg != 0) {
ph7_result_bool(pCtx, 0);
} else {
ph7_result_int64(pCtx, pCtx->pVm->sAllocator.pHeap->nLimit);
}
return PH7_OK;
}
/*
* int get_memory_peak_usage()
* Returns the limit of memory set in Interpreter.
* Parameters
* None
* Return
* The maximum amount of memory that can be allocated from system.
*/
static int vm_builtin_get_memory_peak_usage(ph7_context *pCtx, int nArg, ph7_value **apArg) {
if(nArg != 0) {
ph7_result_bool(pCtx, 0);
} else {
ph7_result_int64(pCtx, pCtx->pVm->sAllocator.pHeap->nPeak);
}
return PH7_OK;
}
/*
* int get_memory_usage()
* Returns the amount of memory, in bytes, that's currently being allocated.
* Parameters
* None
* Return
* Total memory allocated from system, including unused pages.
*/
static int vm_builtin_get_memory_usage(ph7_context *pCtx, int nArg, ph7_value **apArg) {
if(nArg != 0) {
ph7_result_bool(pCtx, 0);
} else {
ph7_result_int64(pCtx, pCtx->pVm->sAllocator.pHeap->nSize);
}
return PH7_OK;
}/*
* int/bool assert_options(int $what [, mixed $value ])
* Set/get the various assert flags.
* Parameter
@@ -11251,6 +11298,10 @@ static const ph7_builtin_func aVmFunc[] = {
{ "ob_get_level", vm_builtin_ob_get_level },
{ "ob_list_handlers", vm_builtin_ob_list_handlers },
{ "ob_start", vm_builtin_ob_start },
/* Memory usage reporting */
{ "get_memory_limit", vm_builtin_get_memory_limit },
{ "get_memory_peak_usage", vm_builtin_get_memory_peak_usage },
{ "get_memory_usage", vm_builtin_get_memory_usage },
/* Assertion functions */
{ "assert_options", vm_builtin_assert_options },
{ "assert", vm_builtin_assert },