diff --git a/mps/code/cbs.c b/mps/code/cbs.c index db4fa5efb46..a3c4f69e042 100644 --- a/mps/code/cbs.c +++ b/mps/code/cbs.c @@ -10,6 +10,11 @@ * collections of memory blocks. * * .sources: . + * + * .critical: In manual-allocation-bound programs using MVFF, many of + * these functions are on the critical paths via mps_alloc (and then + * PoolAlloc, MVFFAlloc, failoverFind*, cbsFind*) and mps_free (and + * then MVFFFree, failoverInsert, cbsInsert). */ #include "cbs.h" @@ -121,11 +126,11 @@ static Bool cbsTestNode(SplayTree splay, Tree tree, void *closure) CBSBlock block; Size *sizeP = closure; - AVERT(SplayTree, splay); - AVERT(Tree, tree); - AVER(sizeP != NULL); - AVER(*sizeP > 0); - AVER(IsA(CBSFast, cbsOfSplay(splay))); + AVERT_CRITICAL(SplayTree, splay); + AVERT_CRITICAL(Tree, tree); + AVER_CRITICAL(sizeP != NULL); + AVER_CRITICAL(*sizeP > 0); + AVER_CRITICAL(IsA(CBSFast, cbsOfSplay(splay))); block = cbsBlockOfTree(tree); @@ -138,11 +143,11 @@ static Bool cbsTestTree(SplayTree splay, Tree tree, CBSFastBlock block; Size *sizeP = closure; - AVERT(SplayTree, splay); - AVERT(Tree, tree); - AVER(sizeP != NULL); - AVER(*sizeP > 0); - AVER(IsA(CBSFast, cbsOfSplay(splay))); + AVERT_CRITICAL(SplayTree, splay); + AVERT_CRITICAL(Tree, tree); + AVER_CRITICAL(sizeP != NULL); + AVER_CRITICAL(*sizeP > 0); + AVER_CRITICAL(IsA(CBSFast, cbsOfSplay(splay))); block = cbsFastBlockOfTree(tree); @@ -313,7 +318,7 @@ static void cbsFinish(Inst inst) static Size cbsSize(Land land) { - CBS cbs = MustBeA(CBS, land); + CBS cbs = MustBeA_CRITICAL(CBS, land); return cbs->size; } @@ -425,12 +430,12 @@ static void cbsBlockInsert(CBS cbs, CBSBlock block) { Bool b; - AVERT(CBS, cbs); - AVERT(CBSBlock, block); + AVERT_CRITICAL(CBS, cbs); + AVERT_CRITICAL(CBSBlock, block); METER_ACC(cbs->treeSearch, cbs->treeSize); b = SplayTreeInsert(cbsSplay(cbs), cbsBlockTree(block)); - AVER(b); + AVER_CRITICAL(b); STATISTIC(++cbs->treeSize); cbs->size += CBSBlockSize(block); } @@ -442,14 +447,11 @@ static void cbsBlockInsert(CBS cbs, CBSBlock block) * * .insert.alloc: Will only allocate a block if the range does not * abut an existing range. - * - * .insert.critical: In manual-allocation-bound programs using MVFF - * this is on the critical path. */ static Res cbsInsert(Range rangeReturn, Land land, Range range) { - CBS cbs = MustBeA(CBS, land); + CBS cbs = MustBeA_CRITICAL(CBS, land); Bool b; Res res; Addr base, limit, newBase, newLimit; @@ -486,7 +488,7 @@ static Res cbsInsert(Range rangeReturn, Land land, Range range) leftMerge = FALSE; } else { leftCBS = cbsBlockOfTree(leftSplay); - AVER(leftCBS->limit <= base); + AVER_CRITICAL(leftCBS->limit <= base); leftMerge = leftCBS->limit == base; } @@ -876,15 +878,15 @@ static void cbsFindDeleteRange(Range rangeReturn, Range oldRangeReturn, static Bool cbsFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - CBS cbs = MustBeA(CBS, land); + CBS cbs = MustBeA_CRITICAL(CBS, land); Bool found; Tree tree; - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVER(size > 0); - AVER(SizeIsAligned(size, LandAlignment(land))); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVER_CRITICAL(size > 0); + AVER_CRITICAL(SizeIsAligned(size, LandAlignment(land))); + AVERT_CRITICAL(FindDelete, findDelete); METER_ACC(cbs->treeSearch, cbs->treeSize); found = SplayFindFirst(&tree, cbsSplay(cbs), &cbsTestNode, @@ -893,9 +895,9 @@ static Bool cbsFindFirst(Range rangeReturn, Range oldRangeReturn, CBSBlock block; RangeStruct range; block = cbsBlockOfTree(tree); - AVER(CBSBlockSize(block) >= size); + AVER_CRITICAL(CBSBlockSize(block) >= size); RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block)); - AVER(RangeSize(&range) >= size); + AVER_CRITICAL(RangeSize(&range) >= size); cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range, size, findDelete); } @@ -952,15 +954,15 @@ static Bool cbsTestTreeInZones(SplayTree splay, Tree tree, static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - CBS cbs = MustBeA(CBSFast, land); + CBS cbs = MustBeA_CRITICAL(CBSFast, land); Bool found; Tree tree; - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVER(size > 0); - AVER(SizeIsAligned(size, LandAlignment(land))); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVER_CRITICAL(size > 0); + AVER_CRITICAL(SizeIsAligned(size, LandAlignment(land))); + AVERT_CRITICAL(FindDelete, findDelete); METER_ACC(cbs->treeSearch, cbs->treeSize); found = SplayFindLast(&tree, cbsSplay(cbs), &cbsTestNode, @@ -969,9 +971,9 @@ static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn, CBSBlock block; RangeStruct range; block = cbsBlockOfTree(tree); - AVER(CBSBlockSize(block) >= size); + AVER_CRITICAL(CBSBlockSize(block) >= size); RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block)); - AVER(RangeSize(&range) >= size); + AVER_CRITICAL(RangeSize(&range) >= size); cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range, size, findDelete); } @@ -985,13 +987,13 @@ static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn, static Bool cbsFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - CBS cbs = MustBeA(CBSFast, land); + CBS cbs = MustBeA_CRITICAL(CBSFast, land); Bool found = FALSE; - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVER(size > 0); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVER_CRITICAL(size > 0); + AVERT_CRITICAL(FindDelete, findDelete); if (!SplayTreeIsEmpty(cbsSplay(cbs))) { RangeStruct range; @@ -1004,11 +1006,11 @@ static Bool cbsFindLargest(Range rangeReturn, Range oldRangeReturn, METER_ACC(cbs->treeSearch, cbs->treeSize); found = SplayFindFirst(&tree, cbsSplay(cbs), &cbsTestNode, &cbsTestTree, &maxSize); - AVER(found); /* maxSize is exact, so we will find it. */ + AVER_CRITICAL(found); /* maxSize is exact, so we will find it. */ block = cbsBlockOfTree(tree); - AVER(CBSBlockSize(block) >= maxSize); + AVER_CRITICAL(CBSBlockSize(block) >= maxSize); RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block)); - AVER(RangeSize(&range) >= maxSize); + AVER_CRITICAL(RangeSize(&range) >= maxSize); cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range, size, findDelete); } @@ -1022,7 +1024,7 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high) { - CBS cbs = MustBeA(CBSZoned, land); + CBS cbs = MustBeA_CRITICAL(CBSZoned, land); CBSBlock block; Tree tree; cbsTestNodeInZonesClosureStruct closure; @@ -1031,11 +1033,11 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn, SplayFindFunction splayFind; RangeStruct rangeStruct, oldRangeStruct; - AVER(foundReturn != NULL); - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - /* AVERT(ZoneSet, zoneSet); */ - AVERT(Bool, high); + AVER_CRITICAL(foundReturn != NULL); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + /* AVERT_CRITICAL(ZoneSet, zoneSet); */ + AVERT_CRITICAL(Bool, high); landFind = high ? cbsFindLast : cbsFindFirst; splayFind = high ? SplayFindLast : SplayFindFirst; @@ -1064,10 +1066,10 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn, block = cbsBlockOfTree(tree); - AVER(CBSBlockBase(block) <= closure.base); - AVER(AddrOffset(closure.base, closure.limit) >= size); - AVER(ZoneSetSub(ZoneSetOfRange(LandArena(land), closure.base, closure.limit), zoneSet)); - AVER(closure.limit <= CBSBlockLimit(block)); + AVER_CRITICAL(CBSBlockBase(block) <= closure.base); + AVER_CRITICAL(AddrOffset(closure.base, closure.limit) >= size); + AVER_CRITICAL(ZoneSetSub(ZoneSetOfRange(LandArena(land), closure.base, closure.limit), zoneSet)); + AVER_CRITICAL(closure.limit <= CBSBlockLimit(block)); if (!high) RangeInit(&rangeStruct, closure.base, AddrAdd(closure.base, size)); diff --git a/mps/code/failover.c b/mps/code/failover.c index d4139e2fe8f..86d1b193411 100644 --- a/mps/code/failover.c +++ b/mps/code/failover.c @@ -4,6 +4,11 @@ * Copyright (c) 2014 Ravenbrook Limited. See end of file for license. * * .design: + * + * .critical: In manual-allocation-bound programs using MVFF, many of + * these functions are on the critical paths via mps_alloc (and then + * PoolAlloc, MVFFAlloc, failoverFind*) and mps_free (and then + * MVFFFree, failoverInsert). */ #include "failover.h" @@ -63,18 +68,18 @@ static void failoverFinish(Inst inst) static Size failoverSize(Land land) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); return LandSize(fo->primary) + LandSize(fo->secondary); } static Res failoverInsert(Range rangeReturn, Land land, Range range) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); Res res; - AVER(rangeReturn != NULL); - AVERT(Range, range); + AVER_CRITICAL(rangeReturn != NULL); + AVERT_CRITICAL(Range, range); /* Provide more opportunities for coalescence. See * . @@ -150,7 +155,7 @@ static Res failoverDelete(Range rangeReturn, Land land, Range range) } } if (res == ResOK) { - AVER(RangesNest(&oldRange, range)); + AVER_CRITICAL(RangesNest(&oldRange, range)); RangeCopy(rangeReturn, &oldRange); } return res; @@ -170,11 +175,11 @@ static Bool failoverIterate(Land land, LandVisitor visitor, void *closure) static Bool failoverFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVERT_CRITICAL(FindDelete, findDelete); /* See . */ (void)LandFlush(fo->primary, fo->secondary); @@ -186,11 +191,11 @@ static Bool failoverFindFirst(Range rangeReturn, Range oldRangeReturn, Land land static Bool failoverFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVERT_CRITICAL(FindDelete, findDelete); /* See . */ (void)LandFlush(fo->primary, fo->secondary); @@ -202,11 +207,11 @@ static Bool failoverFindLast(Range rangeReturn, Range oldRangeReturn, Land land, static Bool failoverFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + AVERT_CRITICAL(FindDelete, findDelete); /* See . */ (void)LandFlush(fo->primary, fo->secondary); @@ -218,16 +223,16 @@ static Bool failoverFindLargest(Range rangeReturn, Range oldRangeReturn, Land la static Bool failoverFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high) { - Failover fo = MustBeA(Failover, land); + Failover fo = MustBeA_CRITICAL(Failover, land); Bool found = FALSE; Res res; - AVER(FALSE); /* TODO: this code is completely untested! */ - AVER(foundReturn != NULL); - AVER(rangeReturn != NULL); - AVER(oldRangeReturn != NULL); - /* AVERT(ZoneSet, zoneSet); */ - AVERT(Bool, high); + AVER_CRITICAL(FALSE); /* TODO: this code is completely untested! */ + AVER_CRITICAL(foundReturn != NULL); + AVER_CRITICAL(rangeReturn != NULL); + AVER_CRITICAL(oldRangeReturn != NULL); + /* AVERT_CRITICAL(ZoneSet, zoneSet); */ + AVERT_CRITICAL(Bool, high); /* See . */ (void)LandFlush(fo->primary, fo->secondary); diff --git a/mps/code/land.c b/mps/code/land.c index d5f7993a7f6..5e7e5452a56 100644 --- a/mps/code/land.c +++ b/mps/code/land.c @@ -148,10 +148,10 @@ void LandFinish(Land land) * is on the critical path. */ -Size LandSize(Land land) +Size (LandSize)(Land land) { /* .enter-leave.simple */ - AVERC_CRITICAL(Land, land); + AVERC(Land, land); return Method(Land, land, sizeMethod)(land); } @@ -165,15 +165,15 @@ Size LandSize(Land land) * this is on the critical path. */ -Res LandInsert(Range rangeReturn, Land land, Range range) +Res (LandInsert)(Range rangeReturn, Land land, Range range) { Res res; - AVER_CRITICAL(rangeReturn != NULL); - AVERC_CRITICAL(Land, land); - AVERT_CRITICAL(Range, range); - AVER_CRITICAL(RangeIsAligned(range, land->alignment)); - AVER_CRITICAL(!RangeIsEmpty(range)); + AVER(rangeReturn != NULL); + AVERC(Land, land); + AVERT(Range, range); + AVER(RangeIsAligned(range, land->alignment)); + AVER(!RangeIsEmpty(range)); landEnter(land); res = Method(Land, land, insert)(rangeReturn, land, range); @@ -188,7 +188,7 @@ Res LandInsert(Range rangeReturn, Land land, Range range) * See */ -Res LandDelete(Range rangeReturn, Land land, Range range) +Res (LandDelete)(Range rangeReturn, Land land, Range range) { Res res; @@ -213,11 +213,11 @@ Res LandDelete(Range rangeReturn, Land land, Range range) * this is on the critical path. */ -Bool LandIterate(Land land, LandVisitor visitor, void *closure) +Bool (LandIterate)(Land land, LandVisitor visitor, void *closure) { Bool b; - AVERC_CRITICAL(Land, land); - AVER_CRITICAL(FUNCHECK(visitor)); + AVERC(Land, land); + AVER(FUNCHECK(visitor)); landEnter(land); b = Method(Land, land, iterate)(land, visitor, closure); @@ -233,11 +233,11 @@ Bool LandIterate(Land land, LandVisitor visitor, void *closure) * See */ -Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure) +Bool (LandIterateAndDelete)(Land land, LandDeleteVisitor visitor, void *closure) { Bool b; - AVERC_CRITICAL(Land, land); - AVER_CRITICAL(FUNCHECK(visitor)); + AVERC(Land, land); + AVER(FUNCHECK(visitor)); landEnter(land); b = Method(Land, land, iterateAndDelete)(land, visitor, closure); @@ -252,7 +252,7 @@ Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure) * See */ -Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) +Bool (LandFindFirst)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { Bool b; @@ -276,7 +276,7 @@ Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size * See */ -Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) +Bool (LandFindLast)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { Bool b; @@ -300,7 +300,7 @@ Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, * See */ -Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) +Bool (LandFindLargest)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete) { Bool b; @@ -324,7 +324,7 @@ Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size si * See */ -Res LandFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high) +Res (LandFindInZones)(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high) { Res res; @@ -360,20 +360,25 @@ Res LandDescribe(Land land, mps_lib_FILE *stream, Count depth) * * closure argument is the destination Land. Attempt to insert the * range into the destination. + * + * .flush.critical: In manual-allocation-bound programs using MVFF + * this is on the critical paths via mps_alloc (and then PoolAlloc, + * MVFFAlloc, failoverFind*, LandFlush) and mps_free (and then + * MVFFFree, failoverInsert, LandFlush). */ -static Bool landFlushVisitor(Bool *deleteReturn, Land land, Range range, - void *closure) +Bool LandFlushVisitor(Bool *deleteReturn, Land land, Range range, + void *closure) { Res res; RangeStruct newRange; Land dest; - AVER(deleteReturn != NULL); - AVERC(Land, land); - AVERT(Range, range); - AVER(closure != NULL); + AVER_CRITICAL(deleteReturn != NULL); + AVERC_CRITICAL(Land, land); + AVERT_CRITICAL(Range, range); + AVER_CRITICAL(closure != NULL); - dest = closure; + dest = MustBeA_CRITICAL(Land, closure); res = LandInsert(&newRange, dest, range); if (res == ResOK) { *deleteReturn = TRUE; @@ -388,17 +393,14 @@ static Bool landFlushVisitor(Bool *deleteReturn, Land land, Range range, /* LandFlush -- move ranges from src to dest * * See - * - * .flush.critical: In manual-allocation-bound programs using MVFF - * this is on the critical path. */ -Bool LandFlush(Land dest, Land src) +Bool (LandFlush)(Land dest, Land src) { - AVERC_CRITICAL(Land, dest); - AVERC_CRITICAL(Land, src); + AVERC(Land, dest); + AVERC(Land, src); - return LandIterateAndDelete(src, landFlushVisitor, dest); + return LandIterateAndDelete(src, LandFlushVisitor, dest); } diff --git a/mps/code/mpm.h b/mps/code/mpm.h index 01f1785e402..c8139ba0e7b 100644 --- a/mps/code/mpm.h +++ b/mps/code/mpm.h @@ -224,7 +224,7 @@ extern Res PoolCreate(Pool *poolReturn, Arena arena, PoolClass klass, extern void PoolDestroy(Pool pool); extern BufferClass PoolDefaultBufferClass(Pool pool); extern Res PoolAlloc(Addr *pReturn, Pool pool, Size size); -extern void PoolFree(Pool pool, Addr old, Size size); +extern void (PoolFree)(Pool pool, Addr old, Size size); extern PoolGen PoolSegPoolGen(Pool pool, Seg seg); extern Res PoolTraceBegin(Pool pool, Trace trace); extern void PoolFreeWalk(Pool pool, FreeBlockVisitor f, void *p); @@ -263,6 +263,9 @@ extern PoolDebugMixin PoolNoDebugMixin(Pool pool); extern BufferClass PoolNoBufferClass(void); extern Size PoolNoSize(Pool pool); +#if !defined(AVER_AND_CHECK_ALL) +#define PoolFree(pool, old, size) Method(Pool, pool, free)(pool, old, size) +#endif /* !defined(AVER_AND_CHECK_ALL) */ /* Abstract Pool Classes Interface -- see */ extern void PoolClassMixInBuffer(PoolClass klass); @@ -962,22 +965,36 @@ extern Res RootsIterate(Globals arena, RootIterateFn f, void *p); extern Bool LandCheck(Land land); #define LandArena(land) ((land)->arena) #define LandAlignment(land) ((land)->alignment) -extern Size LandSize(Land land); +extern Size (LandSize)(Land land); extern Res LandInit(Land land, LandClass klass, Arena arena, Align alignment, void *owner, ArgList args); extern void LandFinish(Land land); -extern Res LandInsert(Range rangeReturn, Land land, Range range); -extern Res LandDelete(Range rangeReturn, Land land, Range range); -extern Bool LandIterate(Land land, LandVisitor visitor, void *closure); -extern Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure); -extern Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); -extern Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); -extern Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); -extern Res LandFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high); +extern Res (LandInsert)(Range rangeReturn, Land land, Range range); +extern Res (LandDelete)(Range rangeReturn, Land land, Range range); +extern Bool (LandIterate)(Land land, LandVisitor visitor, void *closure); +extern Bool (LandIterateAndDelete)(Land land, LandDeleteVisitor visitor, void *closure); +extern Bool (LandFindFirst)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); +extern Bool (LandFindLast)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); +extern Bool (LandFindLargest)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete); +extern Res (LandFindInZones)(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high); extern Res LandDescribe(Land land, mps_lib_FILE *stream, Count depth); -extern Bool LandFlush(Land dest, Land src); - +extern Bool LandFlushVisitor(Bool *deleteReturn, Land land, Range range, void *closure); +extern Bool (LandFlush)(Land dest, Land src); extern Size LandSlowSize(Land land); extern Bool LandClassCheck(LandClass klass); + +#if !defined(AVER_AND_CHECK_ALL) +#define LandSize(land) Method(Land, land, sizeMethod)(land) +#define LandInsert(rangeReturn, land, range) Method(Land, land, insert)(rangeReturn, land, range) +#define LandDelete(rangeReturn, land, range) Method(Land, land, delete)(rangeReturn, land, range) +#define LandIterate(land, visitor, closure) Method(Land, land, iterate)(land, visitor, closure) +#define LandIterateAndDelete(land, visitor, closure) Method(Land, land, iterateAndDelete)(land, visitor, closure) +#define LandFindFirst(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findFirst)(rangeReturn, oldRangeReturn, land, size, findDelete) +#define LandFindLast(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findLast)(rangeReturn, oldRangeReturn, land, size, findDelete) +#define LandFindLargest(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findLargest)(rangeReturn, oldRangeReturn, land, size, findDelete) +#define LandFindInZones(foundReturn, rangeReturn, oldRangeReturn, land, size, zoneSet, high) Method(Land, land, findInZones)(foundReturn, rangeReturn, oldRangeReturn, land, size, zoneSet, high) +#define LandFlush(dest, src) LandIterateAndDelete(src, LandFlushVisitor, dest) +#endif /* !defined(AVER_AND_CHECK_ALL) */ + DECLARE_CLASS(Inst, LandClass, InstClass); DECLARE_CLASS(Land, Land, Inst); diff --git a/mps/code/pool.c b/mps/code/pool.c index 8f46528030b..ad2f0338ab0 100644 --- a/mps/code/pool.c +++ b/mps/code/pool.c @@ -210,7 +210,7 @@ BufferClass PoolDefaultBufferClass(Pool pool) /* PoolAlloc -- allocate a block of memory from a pool * * .alloc.critical: In manual-allocation-bound programs this is on the - * critical path. + * critical path via mps_alloc. */ Res PoolAlloc(Addr *pReturn, Pool pool, Size size) @@ -239,20 +239,16 @@ Res PoolAlloc(Addr *pReturn, Pool pool, Size size) } -/* PoolFree -- deallocate a block of memory allocated from the pool - * - * .free.critical: In manual-allocation-bound programs this is on the - * critical path. - */ +/* PoolFree -- deallocate a block of memory allocated from the pool */ -void PoolFree(Pool pool, Addr old, Size size) +void (PoolFree)(Pool pool, Addr old, Size size) { - AVERT_CRITICAL(Pool, pool); - AVER_CRITICAL(old != NULL); + AVERT(Pool, pool); + AVER(old != NULL); /* The pool methods should check that old is in pool. */ - AVER_CRITICAL(size > 0); - AVER_CRITICAL(AddrIsAligned(old, pool->alignment)); - AVER_CRITICAL(PoolHasRange(pool, old, AddrAdd(old, size))); + AVER(size > 0); + AVER(AddrIsAligned(old, pool->alignment)); + AVER(PoolHasRange(pool, old, AddrAdd(old, size))); Method(Pool, pool, free)(pool, old, size); diff --git a/mps/code/poolmvff.c b/mps/code/poolmvff.c index 6970e3e9d89..b45c0b1965e 100644 --- a/mps/code/poolmvff.c +++ b/mps/code/poolmvff.c @@ -10,12 +10,9 @@ * * .design: * - * NOTE - * - * There's potential for up to 4% speed improvement by calling Land - * methods statically instead of indirectly via the Land abstraction - * (thus, cbsInsert instead of LandInsert, and so on). See - * + * .critical: In manual-allocation-bound programs using MVFF, many of + * these functions are on the critical paths via mps_alloc (and then + * PoolAlloc, MVFFAlloc) and mps_free (and then PoolFree, MVFFFree). */ #include "cbs.h" @@ -104,17 +101,9 @@ static void MVFFReduce(MVFF mvff) RangeStruct freeRange, oldFreeRange; Align grainSize; - AVERT(MVFF, mvff); + AVERT_CRITICAL(MVFF, mvff); arena = PoolArena(MVFFPool(mvff)); - /* NOTE: Memory is returned to the arena in the smallest units - possible (arena grains). There's a possibility that this could - lead to fragmentation in the arena (because allocation is in - multiples of mvff->extendBy). If so, try setting grainSize = - mvff->extendBy here. */ - - grainSize = ArenaGrainSize(arena); - /* Try to return memory when the amount of free memory exceeds a threshold fraction of the total memory. */ @@ -123,6 +112,14 @@ static void MVFFReduce(MVFF mvff) if (freeSize < freeLimit) return; + /* NOTE: Memory is returned to the arena in the smallest units + possible (arena grains). There's a possibility that this could + lead to fragmentation in the arena (because allocation is in + multiples of mvff->extendBy). If so, try setting grainSize = + mvff->extendBy here. */ + + grainSize = ArenaGrainSize(arena); + /* For hysteresis, return only a proportion of the free memory. */ targetFree = freeLimit / 2; @@ -269,12 +266,12 @@ static Res mvffFindFree(Range rangeReturn, MVFF mvff, Size size, RangeStruct oldRange; Land land; - AVER(rangeReturn != NULL); - AVERT(MVFF, mvff); - AVER(size > 0); - AVER(SizeIsAligned(size, PoolAlignment(MVFFPool(mvff)))); - AVER(FUNCHECK(findMethod)); - AVERT(FindDelete, findDelete); + AVER_CRITICAL(rangeReturn != NULL); + AVERT_CRITICAL(MVFF, mvff); + AVER_CRITICAL(size > 0); + AVER_CRITICAL(SizeIsAligned(size, PoolAlignment(MVFFPool(mvff)))); + AVER_CRITICAL(FUNCHECK(findMethod)); + AVERT_CRITICAL(FindDelete, findDelete); land = MVFFFreeLand(mvff); found = (*findMethod)(rangeReturn, &oldRange, land, size, findDelete); @@ -288,20 +285,16 @@ static Res mvffFindFree(Range rangeReturn, MVFF mvff, Size size, /* We know that the found range must intersect the newly added * range. But it doesn't necessarily lie entirely within it. */ - AVER(found); - AVER(RangesOverlap(rangeReturn, &newRange)); + AVER_CRITICAL(found); + AVER_CRITICAL(RangesOverlap(rangeReturn, &newRange)); } - AVER(found); + AVER_CRITICAL(found); return ResOK; } -/* MVFFAlloc -- Allocate a block - * - * .alloc.critical: In manual-allocation-bound programs this is on the - * critical path. - */ +/* MVFFAlloc -- Allocate a block */ static Res MVFFAlloc(Addr *aReturn, Pool pool, Size size) { @@ -331,11 +324,7 @@ static Res MVFFAlloc(Addr *aReturn, Pool pool, Size size) } -/* MVFFFree -- free the given block - * - * .free.critical: In manual-allocation-bound programs this is on the - * critical path. - */ +/* MVFFFree -- free the given block */ static void MVFFFree(Pool pool, Addr old, Size size) { diff --git a/mps/code/splay.c b/mps/code/splay.c index 4017daf1414..d0fd563eccd 100644 --- a/mps/code/splay.c +++ b/mps/code/splay.c @@ -12,6 +12,12 @@ * .note.stack: It's important that the MPS have a bounded stack size, * and this is a problem for tree algorithms. Basically, we have to * avoid recursion. See design.mps.sp.sol.depth.no-recursion. + * + * .critical: In manual-allocation-bound programs using MVFF, many of + * these functions are on the critical paths via mps_alloc (and then + * PoolAlloc, MVFFAlloc, failoverFind*, cbsFind*, SplayTreeFind*) and + * mps_free (and then MVFFFree, failoverInsert, cbsInsert, + * SplayTreeInsert). */ @@ -506,6 +512,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn, SplayTree splay, TreeKey key, TreeCompareFunction compare) { + SplayUpdateNodeFunction updateNode; Tree middle, leftLast, rightFirst; Compare cmp; @@ -513,6 +520,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn, AVER_CRITICAL(FUNCHECK(compare)); AVER_CRITICAL(!SplayTreeIsEmpty(splay)); + updateNode = splay->updateNode; leftLast = TreeEMPTY; rightFirst = TreeEMPTY; middle = SplayTreeRoot(splay); @@ -540,7 +548,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn, if (!TreeHasLeft(middle)) goto stop; middle = SplayZigZigRev(middle, &rightFirst); - splay->updateNode(splay, TreeRight(rightFirst)); + updateNode(splay, TreeRight(rightFirst)); break; case CompareGREATER: if (!TreeHasRight(middle)) @@ -565,7 +573,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn, if (!TreeHasRight(middle)) goto stop; middle = SplayZagZagRev(middle, &leftLast); - splay->updateNode(splay, TreeLeft(leftLast)); + updateNode(splay, TreeLeft(leftLast)); break; case CompareLESS: if (!TreeHasLeft(middle)) @@ -589,13 +597,17 @@ stop: static Tree SplayUpdateLeftSpine(SplayTree splay, Tree node, Tree child) { + SplayUpdateNodeFunction updateNode; + AVERT_CRITICAL(SplayTree, splay); AVERT_CRITICAL(Tree, node); AVERT_CRITICAL(Tree, child); + + updateNode = splay->updateNode; while(node != TreeEMPTY) { Tree parent = TreeLeft(node); TreeSetLeft(node, child); /* un-reverse pointer */ - splay->updateNode(splay, node); + updateNode(splay, node); child = node; node = parent; } @@ -606,13 +618,17 @@ static Tree SplayUpdateLeftSpine(SplayTree splay, Tree node, Tree child) static Tree SplayUpdateRightSpine(SplayTree splay, Tree node, Tree child) { + SplayUpdateNodeFunction updateNode; + AVERT_CRITICAL(SplayTree, splay); AVERT_CRITICAL(Tree, node); AVERT_CRITICAL(Tree, child); + + updateNode = splay->updateNode; while (node != TreeEMPTY) { Tree parent = TreeRight(node); TreeSetRight(node, child); /* un-reverse pointer */ - splay->updateNode(splay, node); + updateNode(splay, node); child = node; node = parent; } @@ -725,7 +741,6 @@ static Compare SplaySplay(SplayTree splay, TreeKey key, /* SplayTreeInsert -- insert a node into a splay tree - * * * This function is used to insert a node into the tree. Splays the * tree at the node's key. If an attempt is made to insert a node that @@ -915,10 +930,9 @@ Bool SplayTreeNeighbours(Tree *leftReturn, Tree *rightReturn, Count count = SplayDebugCount(splay); #endif - - AVERT(SplayTree, splay); - AVER(leftReturn != NULL); - AVER(rightReturn != NULL); + AVERT_CRITICAL(SplayTree, splay); + AVER_CRITICAL(leftReturn != NULL); + AVER_CRITICAL(rightReturn != NULL); if (SplayTreeIsEmpty(splay)) { *leftReturn = *rightReturn = TreeEMPTY; @@ -936,14 +950,14 @@ Bool SplayTreeNeighbours(Tree *leftReturn, Tree *rightReturn, break; case CompareLESS: - AVER(!TreeHasLeft(stateStruct.middle)); + AVER_CRITICAL(!TreeHasLeft(stateStruct.middle)); *rightReturn = stateStruct.middle; *leftReturn = stateStruct.leftLast; found = TRUE; break; case CompareGREATER: - AVER(!TreeHasRight(stateStruct.middle)); + AVER_CRITICAL(!TreeHasRight(stateStruct.middle)); *leftReturn = stateStruct.middle; *rightReturn = stateStruct.rightFirst; found = TRUE; @@ -1101,8 +1115,8 @@ static Compare SplayFindFirstCompare(Tree node, TreeKey key) void *testClosure; SplayTree splay; - AVERT(Tree, node); - AVER(key != NULL); + AVERT_CRITICAL(Tree, node); + AVER_CRITICAL(key != NULL); /* Lift closure values into variables so that they aren't aliased by calls to the test functions. */ @@ -1140,8 +1154,8 @@ static Compare SplayFindLastCompare(Tree node, TreeKey key) void *testClosure; SplayTree splay; - AVERT(Tree, node); - AVER(key != NULL); + AVERT_CRITICAL(Tree, node); + AVER_CRITICAL(key != NULL); /* Lift closure values into variables so that they aren't aliased by calls to the test functions. */ @@ -1195,10 +1209,10 @@ Bool SplayFindFirst(Tree *nodeReturn, SplayTree splay, SplayFindClosureStruct closureStruct; Bool found; - AVER(nodeReturn != NULL); - AVERT(SplayTree, splay); - AVER(FUNCHECK(testNode)); - AVER(FUNCHECK(testTree)); + AVER_CRITICAL(nodeReturn != NULL); + AVERT_CRITICAL(SplayTree, splay); + AVER_CRITICAL(FUNCHECK(testNode)); + AVER_CRITICAL(FUNCHECK(testTree)); if (SplayTreeIsEmpty(splay) || !testTree(splay, SplayTreeRoot(splay), testClosure)) @@ -1258,10 +1272,10 @@ Bool SplayFindLast(Tree *nodeReturn, SplayTree splay, SplayFindClosureStruct closureStruct; Bool found; - AVER(nodeReturn != NULL); - AVERT(SplayTree, splay); - AVER(FUNCHECK(testNode)); - AVER(FUNCHECK(testTree)); + AVER_CRITICAL(nodeReturn != NULL); + AVERT_CRITICAL(SplayTree, splay); + AVER_CRITICAL(FUNCHECK(testNode)); + AVER_CRITICAL(FUNCHECK(testTree)); if (SplayTreeIsEmpty(splay) || !testTree(splay, SplayTreeRoot(splay), testClosure))