diff --git a/mps/code/cbs.c b/mps/code/cbs.c
index db4fa5efb46..a3c4f69e042 100644
--- a/mps/code/cbs.c
+++ b/mps/code/cbs.c
@@ -10,6 +10,11 @@
* collections of memory blocks.
*
* .sources: .
+ *
+ * .critical: In manual-allocation-bound programs using MVFF, many of
+ * these functions are on the critical paths via mps_alloc (and then
+ * PoolAlloc, MVFFAlloc, failoverFind*, cbsFind*) and mps_free (and
+ * then MVFFFree, failoverInsert, cbsInsert).
*/
#include "cbs.h"
@@ -121,11 +126,11 @@ static Bool cbsTestNode(SplayTree splay, Tree tree, void *closure)
CBSBlock block;
Size *sizeP = closure;
- AVERT(SplayTree, splay);
- AVERT(Tree, tree);
- AVER(sizeP != NULL);
- AVER(*sizeP > 0);
- AVER(IsA(CBSFast, cbsOfSplay(splay)));
+ AVERT_CRITICAL(SplayTree, splay);
+ AVERT_CRITICAL(Tree, tree);
+ AVER_CRITICAL(sizeP != NULL);
+ AVER_CRITICAL(*sizeP > 0);
+ AVER_CRITICAL(IsA(CBSFast, cbsOfSplay(splay)));
block = cbsBlockOfTree(tree);
@@ -138,11 +143,11 @@ static Bool cbsTestTree(SplayTree splay, Tree tree,
CBSFastBlock block;
Size *sizeP = closure;
- AVERT(SplayTree, splay);
- AVERT(Tree, tree);
- AVER(sizeP != NULL);
- AVER(*sizeP > 0);
- AVER(IsA(CBSFast, cbsOfSplay(splay)));
+ AVERT_CRITICAL(SplayTree, splay);
+ AVERT_CRITICAL(Tree, tree);
+ AVER_CRITICAL(sizeP != NULL);
+ AVER_CRITICAL(*sizeP > 0);
+ AVER_CRITICAL(IsA(CBSFast, cbsOfSplay(splay)));
block = cbsFastBlockOfTree(tree);
@@ -313,7 +318,7 @@ static void cbsFinish(Inst inst)
static Size cbsSize(Land land)
{
- CBS cbs = MustBeA(CBS, land);
+ CBS cbs = MustBeA_CRITICAL(CBS, land);
return cbs->size;
}
@@ -425,12 +430,12 @@ static void cbsBlockInsert(CBS cbs, CBSBlock block)
{
Bool b;
- AVERT(CBS, cbs);
- AVERT(CBSBlock, block);
+ AVERT_CRITICAL(CBS, cbs);
+ AVERT_CRITICAL(CBSBlock, block);
METER_ACC(cbs->treeSearch, cbs->treeSize);
b = SplayTreeInsert(cbsSplay(cbs), cbsBlockTree(block));
- AVER(b);
+ AVER_CRITICAL(b);
STATISTIC(++cbs->treeSize);
cbs->size += CBSBlockSize(block);
}
@@ -442,14 +447,11 @@ static void cbsBlockInsert(CBS cbs, CBSBlock block)
*
* .insert.alloc: Will only allocate a block if the range does not
* abut an existing range.
- *
- * .insert.critical: In manual-allocation-bound programs using MVFF
- * this is on the critical path.
*/
static Res cbsInsert(Range rangeReturn, Land land, Range range)
{
- CBS cbs = MustBeA(CBS, land);
+ CBS cbs = MustBeA_CRITICAL(CBS, land);
Bool b;
Res res;
Addr base, limit, newBase, newLimit;
@@ -486,7 +488,7 @@ static Res cbsInsert(Range rangeReturn, Land land, Range range)
leftMerge = FALSE;
} else {
leftCBS = cbsBlockOfTree(leftSplay);
- AVER(leftCBS->limit <= base);
+ AVER_CRITICAL(leftCBS->limit <= base);
leftMerge = leftCBS->limit == base;
}
@@ -876,15 +878,15 @@ static void cbsFindDeleteRange(Range rangeReturn, Range oldRangeReturn,
static Bool cbsFindFirst(Range rangeReturn, Range oldRangeReturn,
Land land, Size size, FindDelete findDelete)
{
- CBS cbs = MustBeA(CBS, land);
+ CBS cbs = MustBeA_CRITICAL(CBS, land);
Bool found;
Tree tree;
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVER(size > 0);
- AVER(SizeIsAligned(size, LandAlignment(land)));
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVER_CRITICAL(size > 0);
+ AVER_CRITICAL(SizeIsAligned(size, LandAlignment(land)));
+ AVERT_CRITICAL(FindDelete, findDelete);
METER_ACC(cbs->treeSearch, cbs->treeSize);
found = SplayFindFirst(&tree, cbsSplay(cbs), &cbsTestNode,
@@ -893,9 +895,9 @@ static Bool cbsFindFirst(Range rangeReturn, Range oldRangeReturn,
CBSBlock block;
RangeStruct range;
block = cbsBlockOfTree(tree);
- AVER(CBSBlockSize(block) >= size);
+ AVER_CRITICAL(CBSBlockSize(block) >= size);
RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block));
- AVER(RangeSize(&range) >= size);
+ AVER_CRITICAL(RangeSize(&range) >= size);
cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range,
size, findDelete);
}
@@ -952,15 +954,15 @@ static Bool cbsTestTreeInZones(SplayTree splay, Tree tree,
static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn,
Land land, Size size, FindDelete findDelete)
{
- CBS cbs = MustBeA(CBSFast, land);
+ CBS cbs = MustBeA_CRITICAL(CBSFast, land);
Bool found;
Tree tree;
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVER(size > 0);
- AVER(SizeIsAligned(size, LandAlignment(land)));
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVER_CRITICAL(size > 0);
+ AVER_CRITICAL(SizeIsAligned(size, LandAlignment(land)));
+ AVERT_CRITICAL(FindDelete, findDelete);
METER_ACC(cbs->treeSearch, cbs->treeSize);
found = SplayFindLast(&tree, cbsSplay(cbs), &cbsTestNode,
@@ -969,9 +971,9 @@ static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn,
CBSBlock block;
RangeStruct range;
block = cbsBlockOfTree(tree);
- AVER(CBSBlockSize(block) >= size);
+ AVER_CRITICAL(CBSBlockSize(block) >= size);
RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block));
- AVER(RangeSize(&range) >= size);
+ AVER_CRITICAL(RangeSize(&range) >= size);
cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range,
size, findDelete);
}
@@ -985,13 +987,13 @@ static Bool cbsFindLast(Range rangeReturn, Range oldRangeReturn,
static Bool cbsFindLargest(Range rangeReturn, Range oldRangeReturn,
Land land, Size size, FindDelete findDelete)
{
- CBS cbs = MustBeA(CBSFast, land);
+ CBS cbs = MustBeA_CRITICAL(CBSFast, land);
Bool found = FALSE;
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVER(size > 0);
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVER_CRITICAL(size > 0);
+ AVERT_CRITICAL(FindDelete, findDelete);
if (!SplayTreeIsEmpty(cbsSplay(cbs))) {
RangeStruct range;
@@ -1004,11 +1006,11 @@ static Bool cbsFindLargest(Range rangeReturn, Range oldRangeReturn,
METER_ACC(cbs->treeSearch, cbs->treeSize);
found = SplayFindFirst(&tree, cbsSplay(cbs), &cbsTestNode,
&cbsTestTree, &maxSize);
- AVER(found); /* maxSize is exact, so we will find it. */
+ AVER_CRITICAL(found); /* maxSize is exact, so we will find it. */
block = cbsBlockOfTree(tree);
- AVER(CBSBlockSize(block) >= maxSize);
+ AVER_CRITICAL(CBSBlockSize(block) >= maxSize);
RangeInit(&range, CBSBlockBase(block), CBSBlockLimit(block));
- AVER(RangeSize(&range) >= maxSize);
+ AVER_CRITICAL(RangeSize(&range) >= maxSize);
cbsFindDeleteRange(rangeReturn, oldRangeReturn, land, &range,
size, findDelete);
}
@@ -1022,7 +1024,7 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn,
Range oldRangeReturn, Land land, Size size,
ZoneSet zoneSet, Bool high)
{
- CBS cbs = MustBeA(CBSZoned, land);
+ CBS cbs = MustBeA_CRITICAL(CBSZoned, land);
CBSBlock block;
Tree tree;
cbsTestNodeInZonesClosureStruct closure;
@@ -1031,11 +1033,11 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn,
SplayFindFunction splayFind;
RangeStruct rangeStruct, oldRangeStruct;
- AVER(foundReturn != NULL);
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- /* AVERT(ZoneSet, zoneSet); */
- AVERT(Bool, high);
+ AVER_CRITICAL(foundReturn != NULL);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ /* AVERT_CRITICAL(ZoneSet, zoneSet); */
+ AVERT_CRITICAL(Bool, high);
landFind = high ? cbsFindLast : cbsFindFirst;
splayFind = high ? SplayFindLast : SplayFindFirst;
@@ -1064,10 +1066,10 @@ static Res cbsFindInZones(Bool *foundReturn, Range rangeReturn,
block = cbsBlockOfTree(tree);
- AVER(CBSBlockBase(block) <= closure.base);
- AVER(AddrOffset(closure.base, closure.limit) >= size);
- AVER(ZoneSetSub(ZoneSetOfRange(LandArena(land), closure.base, closure.limit), zoneSet));
- AVER(closure.limit <= CBSBlockLimit(block));
+ AVER_CRITICAL(CBSBlockBase(block) <= closure.base);
+ AVER_CRITICAL(AddrOffset(closure.base, closure.limit) >= size);
+ AVER_CRITICAL(ZoneSetSub(ZoneSetOfRange(LandArena(land), closure.base, closure.limit), zoneSet));
+ AVER_CRITICAL(closure.limit <= CBSBlockLimit(block));
if (!high)
RangeInit(&rangeStruct, closure.base, AddrAdd(closure.base, size));
diff --git a/mps/code/failover.c b/mps/code/failover.c
index d4139e2fe8f..86d1b193411 100644
--- a/mps/code/failover.c
+++ b/mps/code/failover.c
@@ -4,6 +4,11 @@
* Copyright (c) 2014 Ravenbrook Limited. See end of file for license.
*
* .design:
+ *
+ * .critical: In manual-allocation-bound programs using MVFF, many of
+ * these functions are on the critical paths via mps_alloc (and then
+ * PoolAlloc, MVFFAlloc, failoverFind*) and mps_free (and then
+ * MVFFFree, failoverInsert).
*/
#include "failover.h"
@@ -63,18 +68,18 @@ static void failoverFinish(Inst inst)
static Size failoverSize(Land land)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
return LandSize(fo->primary) + LandSize(fo->secondary);
}
static Res failoverInsert(Range rangeReturn, Land land, Range range)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
Res res;
- AVER(rangeReturn != NULL);
- AVERT(Range, range);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVERT_CRITICAL(Range, range);
/* Provide more opportunities for coalescence. See
* .
@@ -150,7 +155,7 @@ static Res failoverDelete(Range rangeReturn, Land land, Range range)
}
}
if (res == ResOK) {
- AVER(RangesNest(&oldRange, range));
+ AVER_CRITICAL(RangesNest(&oldRange, range));
RangeCopy(rangeReturn, &oldRange);
}
return res;
@@ -170,11 +175,11 @@ static Bool failoverIterate(Land land, LandVisitor visitor, void *closure)
static Bool failoverFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVERT_CRITICAL(FindDelete, findDelete);
/* See . */
(void)LandFlush(fo->primary, fo->secondary);
@@ -186,11 +191,11 @@ static Bool failoverFindFirst(Range rangeReturn, Range oldRangeReturn, Land land
static Bool failoverFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVERT_CRITICAL(FindDelete, findDelete);
/* See . */
(void)LandFlush(fo->primary, fo->secondary);
@@ -202,11 +207,11 @@ static Bool failoverFindLast(Range rangeReturn, Range oldRangeReturn, Land land,
static Bool failoverFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ AVERT_CRITICAL(FindDelete, findDelete);
/* See . */
(void)LandFlush(fo->primary, fo->secondary);
@@ -218,16 +223,16 @@ static Bool failoverFindLargest(Range rangeReturn, Range oldRangeReturn, Land la
static Bool failoverFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high)
{
- Failover fo = MustBeA(Failover, land);
+ Failover fo = MustBeA_CRITICAL(Failover, land);
Bool found = FALSE;
Res res;
- AVER(FALSE); /* TODO: this code is completely untested! */
- AVER(foundReturn != NULL);
- AVER(rangeReturn != NULL);
- AVER(oldRangeReturn != NULL);
- /* AVERT(ZoneSet, zoneSet); */
- AVERT(Bool, high);
+ AVER_CRITICAL(FALSE); /* TODO: this code is completely untested! */
+ AVER_CRITICAL(foundReturn != NULL);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVER_CRITICAL(oldRangeReturn != NULL);
+ /* AVERT_CRITICAL(ZoneSet, zoneSet); */
+ AVERT_CRITICAL(Bool, high);
/* See . */
(void)LandFlush(fo->primary, fo->secondary);
diff --git a/mps/code/land.c b/mps/code/land.c
index d5f7993a7f6..5e7e5452a56 100644
--- a/mps/code/land.c
+++ b/mps/code/land.c
@@ -148,10 +148,10 @@ void LandFinish(Land land)
* is on the critical path.
*/
-Size LandSize(Land land)
+Size (LandSize)(Land land)
{
/* .enter-leave.simple */
- AVERC_CRITICAL(Land, land);
+ AVERC(Land, land);
return Method(Land, land, sizeMethod)(land);
}
@@ -165,15 +165,15 @@ Size LandSize(Land land)
* this is on the critical path.
*/
-Res LandInsert(Range rangeReturn, Land land, Range range)
+Res (LandInsert)(Range rangeReturn, Land land, Range range)
{
Res res;
- AVER_CRITICAL(rangeReturn != NULL);
- AVERC_CRITICAL(Land, land);
- AVERT_CRITICAL(Range, range);
- AVER_CRITICAL(RangeIsAligned(range, land->alignment));
- AVER_CRITICAL(!RangeIsEmpty(range));
+ AVER(rangeReturn != NULL);
+ AVERC(Land, land);
+ AVERT(Range, range);
+ AVER(RangeIsAligned(range, land->alignment));
+ AVER(!RangeIsEmpty(range));
landEnter(land);
res = Method(Land, land, insert)(rangeReturn, land, range);
@@ -188,7 +188,7 @@ Res LandInsert(Range rangeReturn, Land land, Range range)
* See
*/
-Res LandDelete(Range rangeReturn, Land land, Range range)
+Res (LandDelete)(Range rangeReturn, Land land, Range range)
{
Res res;
@@ -213,11 +213,11 @@ Res LandDelete(Range rangeReturn, Land land, Range range)
* this is on the critical path.
*/
-Bool LandIterate(Land land, LandVisitor visitor, void *closure)
+Bool (LandIterate)(Land land, LandVisitor visitor, void *closure)
{
Bool b;
- AVERC_CRITICAL(Land, land);
- AVER_CRITICAL(FUNCHECK(visitor));
+ AVERC(Land, land);
+ AVER(FUNCHECK(visitor));
landEnter(land);
b = Method(Land, land, iterate)(land, visitor, closure);
@@ -233,11 +233,11 @@ Bool LandIterate(Land land, LandVisitor visitor, void *closure)
* See
*/
-Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure)
+Bool (LandIterateAndDelete)(Land land, LandDeleteVisitor visitor, void *closure)
{
Bool b;
- AVERC_CRITICAL(Land, land);
- AVER_CRITICAL(FUNCHECK(visitor));
+ AVERC(Land, land);
+ AVER(FUNCHECK(visitor));
landEnter(land);
b = Method(Land, land, iterateAndDelete)(land, visitor, closure);
@@ -252,7 +252,7 @@ Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure)
* See
*/
-Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
+Bool (LandFindFirst)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
Bool b;
@@ -276,7 +276,7 @@ Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size
* See
*/
-Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
+Bool (LandFindLast)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
Bool b;
@@ -300,7 +300,7 @@ Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size,
* See
*/
-Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
+Bool (LandFindLargest)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete)
{
Bool b;
@@ -324,7 +324,7 @@ Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size si
* See
*/
-Res LandFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high)
+Res (LandFindInZones)(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high)
{
Res res;
@@ -360,20 +360,25 @@ Res LandDescribe(Land land, mps_lib_FILE *stream, Count depth)
*
* closure argument is the destination Land. Attempt to insert the
* range into the destination.
+ *
+ * .flush.critical: In manual-allocation-bound programs using MVFF
+ * this is on the critical paths via mps_alloc (and then PoolAlloc,
+ * MVFFAlloc, failoverFind*, LandFlush) and mps_free (and then
+ * MVFFFree, failoverInsert, LandFlush).
*/
-static Bool landFlushVisitor(Bool *deleteReturn, Land land, Range range,
- void *closure)
+Bool LandFlushVisitor(Bool *deleteReturn, Land land, Range range,
+ void *closure)
{
Res res;
RangeStruct newRange;
Land dest;
- AVER(deleteReturn != NULL);
- AVERC(Land, land);
- AVERT(Range, range);
- AVER(closure != NULL);
+ AVER_CRITICAL(deleteReturn != NULL);
+ AVERC_CRITICAL(Land, land);
+ AVERT_CRITICAL(Range, range);
+ AVER_CRITICAL(closure != NULL);
- dest = closure;
+ dest = MustBeA_CRITICAL(Land, closure);
res = LandInsert(&newRange, dest, range);
if (res == ResOK) {
*deleteReturn = TRUE;
@@ -388,17 +393,14 @@ static Bool landFlushVisitor(Bool *deleteReturn, Land land, Range range,
/* LandFlush -- move ranges from src to dest
*
* See
- *
- * .flush.critical: In manual-allocation-bound programs using MVFF
- * this is on the critical path.
*/
-Bool LandFlush(Land dest, Land src)
+Bool (LandFlush)(Land dest, Land src)
{
- AVERC_CRITICAL(Land, dest);
- AVERC_CRITICAL(Land, src);
+ AVERC(Land, dest);
+ AVERC(Land, src);
- return LandIterateAndDelete(src, landFlushVisitor, dest);
+ return LandIterateAndDelete(src, LandFlushVisitor, dest);
}
diff --git a/mps/code/mpm.h b/mps/code/mpm.h
index 01f1785e402..c8139ba0e7b 100644
--- a/mps/code/mpm.h
+++ b/mps/code/mpm.h
@@ -224,7 +224,7 @@ extern Res PoolCreate(Pool *poolReturn, Arena arena, PoolClass klass,
extern void PoolDestroy(Pool pool);
extern BufferClass PoolDefaultBufferClass(Pool pool);
extern Res PoolAlloc(Addr *pReturn, Pool pool, Size size);
-extern void PoolFree(Pool pool, Addr old, Size size);
+extern void (PoolFree)(Pool pool, Addr old, Size size);
extern PoolGen PoolSegPoolGen(Pool pool, Seg seg);
extern Res PoolTraceBegin(Pool pool, Trace trace);
extern void PoolFreeWalk(Pool pool, FreeBlockVisitor f, void *p);
@@ -263,6 +263,9 @@ extern PoolDebugMixin PoolNoDebugMixin(Pool pool);
extern BufferClass PoolNoBufferClass(void);
extern Size PoolNoSize(Pool pool);
+#if !defined(AVER_AND_CHECK_ALL)
+#define PoolFree(pool, old, size) Method(Pool, pool, free)(pool, old, size)
+#endif /* !defined(AVER_AND_CHECK_ALL) */
/* Abstract Pool Classes Interface -- see */
extern void PoolClassMixInBuffer(PoolClass klass);
@@ -962,22 +965,36 @@ extern Res RootsIterate(Globals arena, RootIterateFn f, void *p);
extern Bool LandCheck(Land land);
#define LandArena(land) ((land)->arena)
#define LandAlignment(land) ((land)->alignment)
-extern Size LandSize(Land land);
+extern Size (LandSize)(Land land);
extern Res LandInit(Land land, LandClass klass, Arena arena, Align alignment, void *owner, ArgList args);
extern void LandFinish(Land land);
-extern Res LandInsert(Range rangeReturn, Land land, Range range);
-extern Res LandDelete(Range rangeReturn, Land land, Range range);
-extern Bool LandIterate(Land land, LandVisitor visitor, void *closure);
-extern Bool LandIterateAndDelete(Land land, LandDeleteVisitor visitor, void *closure);
-extern Bool LandFindFirst(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
-extern Bool LandFindLast(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
-extern Bool LandFindLargest(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
-extern Res LandFindInZones(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high);
+extern Res (LandInsert)(Range rangeReturn, Land land, Range range);
+extern Res (LandDelete)(Range rangeReturn, Land land, Range range);
+extern Bool (LandIterate)(Land land, LandVisitor visitor, void *closure);
+extern Bool (LandIterateAndDelete)(Land land, LandDeleteVisitor visitor, void *closure);
+extern Bool (LandFindFirst)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
+extern Bool (LandFindLast)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
+extern Bool (LandFindLargest)(Range rangeReturn, Range oldRangeReturn, Land land, Size size, FindDelete findDelete);
+extern Res (LandFindInZones)(Bool *foundReturn, Range rangeReturn, Range oldRangeReturn, Land land, Size size, ZoneSet zoneSet, Bool high);
extern Res LandDescribe(Land land, mps_lib_FILE *stream, Count depth);
-extern Bool LandFlush(Land dest, Land src);
-
+extern Bool LandFlushVisitor(Bool *deleteReturn, Land land, Range range, void *closure);
+extern Bool (LandFlush)(Land dest, Land src);
extern Size LandSlowSize(Land land);
extern Bool LandClassCheck(LandClass klass);
+
+#if !defined(AVER_AND_CHECK_ALL)
+#define LandSize(land) Method(Land, land, sizeMethod)(land)
+#define LandInsert(rangeReturn, land, range) Method(Land, land, insert)(rangeReturn, land, range)
+#define LandDelete(rangeReturn, land, range) Method(Land, land, delete)(rangeReturn, land, range)
+#define LandIterate(land, visitor, closure) Method(Land, land, iterate)(land, visitor, closure)
+#define LandIterateAndDelete(land, visitor, closure) Method(Land, land, iterateAndDelete)(land, visitor, closure)
+#define LandFindFirst(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findFirst)(rangeReturn, oldRangeReturn, land, size, findDelete)
+#define LandFindLast(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findLast)(rangeReturn, oldRangeReturn, land, size, findDelete)
+#define LandFindLargest(rangeReturn, oldRangeReturn, land, size, findDelete) Method(Land, land, findLargest)(rangeReturn, oldRangeReturn, land, size, findDelete)
+#define LandFindInZones(foundReturn, rangeReturn, oldRangeReturn, land, size, zoneSet, high) Method(Land, land, findInZones)(foundReturn, rangeReturn, oldRangeReturn, land, size, zoneSet, high)
+#define LandFlush(dest, src) LandIterateAndDelete(src, LandFlushVisitor, dest)
+#endif /* !defined(AVER_AND_CHECK_ALL) */
+
DECLARE_CLASS(Inst, LandClass, InstClass);
DECLARE_CLASS(Land, Land, Inst);
diff --git a/mps/code/pool.c b/mps/code/pool.c
index 8f46528030b..ad2f0338ab0 100644
--- a/mps/code/pool.c
+++ b/mps/code/pool.c
@@ -210,7 +210,7 @@ BufferClass PoolDefaultBufferClass(Pool pool)
/* PoolAlloc -- allocate a block of memory from a pool
*
* .alloc.critical: In manual-allocation-bound programs this is on the
- * critical path.
+ * critical path via mps_alloc.
*/
Res PoolAlloc(Addr *pReturn, Pool pool, Size size)
@@ -239,20 +239,16 @@ Res PoolAlloc(Addr *pReturn, Pool pool, Size size)
}
-/* PoolFree -- deallocate a block of memory allocated from the pool
- *
- * .free.critical: In manual-allocation-bound programs this is on the
- * critical path.
- */
+/* PoolFree -- deallocate a block of memory allocated from the pool */
-void PoolFree(Pool pool, Addr old, Size size)
+void (PoolFree)(Pool pool, Addr old, Size size)
{
- AVERT_CRITICAL(Pool, pool);
- AVER_CRITICAL(old != NULL);
+ AVERT(Pool, pool);
+ AVER(old != NULL);
/* The pool methods should check that old is in pool. */
- AVER_CRITICAL(size > 0);
- AVER_CRITICAL(AddrIsAligned(old, pool->alignment));
- AVER_CRITICAL(PoolHasRange(pool, old, AddrAdd(old, size)));
+ AVER(size > 0);
+ AVER(AddrIsAligned(old, pool->alignment));
+ AVER(PoolHasRange(pool, old, AddrAdd(old, size)));
Method(Pool, pool, free)(pool, old, size);
diff --git a/mps/code/poolmvff.c b/mps/code/poolmvff.c
index 6970e3e9d89..b45c0b1965e 100644
--- a/mps/code/poolmvff.c
+++ b/mps/code/poolmvff.c
@@ -10,12 +10,9 @@
*
* .design:
*
- * NOTE
- *
- * There's potential for up to 4% speed improvement by calling Land
- * methods statically instead of indirectly via the Land abstraction
- * (thus, cbsInsert instead of LandInsert, and so on). See
- *
+ * .critical: In manual-allocation-bound programs using MVFF, many of
+ * these functions are on the critical paths via mps_alloc (and then
+ * PoolAlloc, MVFFAlloc) and mps_free (and then PoolFree, MVFFFree).
*/
#include "cbs.h"
@@ -104,17 +101,9 @@ static void MVFFReduce(MVFF mvff)
RangeStruct freeRange, oldFreeRange;
Align grainSize;
- AVERT(MVFF, mvff);
+ AVERT_CRITICAL(MVFF, mvff);
arena = PoolArena(MVFFPool(mvff));
- /* NOTE: Memory is returned to the arena in the smallest units
- possible (arena grains). There's a possibility that this could
- lead to fragmentation in the arena (because allocation is in
- multiples of mvff->extendBy). If so, try setting grainSize =
- mvff->extendBy here. */
-
- grainSize = ArenaGrainSize(arena);
-
/* Try to return memory when the amount of free memory exceeds a
threshold fraction of the total memory. */
@@ -123,6 +112,14 @@ static void MVFFReduce(MVFF mvff)
if (freeSize < freeLimit)
return;
+ /* NOTE: Memory is returned to the arena in the smallest units
+ possible (arena grains). There's a possibility that this could
+ lead to fragmentation in the arena (because allocation is in
+ multiples of mvff->extendBy). If so, try setting grainSize =
+ mvff->extendBy here. */
+
+ grainSize = ArenaGrainSize(arena);
+
/* For hysteresis, return only a proportion of the free memory. */
targetFree = freeLimit / 2;
@@ -269,12 +266,12 @@ static Res mvffFindFree(Range rangeReturn, MVFF mvff, Size size,
RangeStruct oldRange;
Land land;
- AVER(rangeReturn != NULL);
- AVERT(MVFF, mvff);
- AVER(size > 0);
- AVER(SizeIsAligned(size, PoolAlignment(MVFFPool(mvff))));
- AVER(FUNCHECK(findMethod));
- AVERT(FindDelete, findDelete);
+ AVER_CRITICAL(rangeReturn != NULL);
+ AVERT_CRITICAL(MVFF, mvff);
+ AVER_CRITICAL(size > 0);
+ AVER_CRITICAL(SizeIsAligned(size, PoolAlignment(MVFFPool(mvff))));
+ AVER_CRITICAL(FUNCHECK(findMethod));
+ AVERT_CRITICAL(FindDelete, findDelete);
land = MVFFFreeLand(mvff);
found = (*findMethod)(rangeReturn, &oldRange, land, size, findDelete);
@@ -288,20 +285,16 @@ static Res mvffFindFree(Range rangeReturn, MVFF mvff, Size size,
/* We know that the found range must intersect the newly added
* range. But it doesn't necessarily lie entirely within it. */
- AVER(found);
- AVER(RangesOverlap(rangeReturn, &newRange));
+ AVER_CRITICAL(found);
+ AVER_CRITICAL(RangesOverlap(rangeReturn, &newRange));
}
- AVER(found);
+ AVER_CRITICAL(found);
return ResOK;
}
-/* MVFFAlloc -- Allocate a block
- *
- * .alloc.critical: In manual-allocation-bound programs this is on the
- * critical path.
- */
+/* MVFFAlloc -- Allocate a block */
static Res MVFFAlloc(Addr *aReturn, Pool pool, Size size)
{
@@ -331,11 +324,7 @@ static Res MVFFAlloc(Addr *aReturn, Pool pool, Size size)
}
-/* MVFFFree -- free the given block
- *
- * .free.critical: In manual-allocation-bound programs this is on the
- * critical path.
- */
+/* MVFFFree -- free the given block */
static void MVFFFree(Pool pool, Addr old, Size size)
{
diff --git a/mps/code/splay.c b/mps/code/splay.c
index 4017daf1414..d0fd563eccd 100644
--- a/mps/code/splay.c
+++ b/mps/code/splay.c
@@ -12,6 +12,12 @@
* .note.stack: It's important that the MPS have a bounded stack size,
* and this is a problem for tree algorithms. Basically, we have to
* avoid recursion. See design.mps.sp.sol.depth.no-recursion.
+ *
+ * .critical: In manual-allocation-bound programs using MVFF, many of
+ * these functions are on the critical paths via mps_alloc (and then
+ * PoolAlloc, MVFFAlloc, failoverFind*, cbsFind*, SplayTreeFind*) and
+ * mps_free (and then MVFFFree, failoverInsert, cbsInsert,
+ * SplayTreeInsert).
*/
@@ -506,6 +512,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn,
SplayTree splay, TreeKey key,
TreeCompareFunction compare)
{
+ SplayUpdateNodeFunction updateNode;
Tree middle, leftLast, rightFirst;
Compare cmp;
@@ -513,6 +520,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn,
AVER_CRITICAL(FUNCHECK(compare));
AVER_CRITICAL(!SplayTreeIsEmpty(splay));
+ updateNode = splay->updateNode;
leftLast = TreeEMPTY;
rightFirst = TreeEMPTY;
middle = SplayTreeRoot(splay);
@@ -540,7 +548,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn,
if (!TreeHasLeft(middle))
goto stop;
middle = SplayZigZigRev(middle, &rightFirst);
- splay->updateNode(splay, TreeRight(rightFirst));
+ updateNode(splay, TreeRight(rightFirst));
break;
case CompareGREATER:
if (!TreeHasRight(middle))
@@ -565,7 +573,7 @@ static Compare SplaySplitRev(SplayStateStruct *stateReturn,
if (!TreeHasRight(middle))
goto stop;
middle = SplayZagZagRev(middle, &leftLast);
- splay->updateNode(splay, TreeLeft(leftLast));
+ updateNode(splay, TreeLeft(leftLast));
break;
case CompareLESS:
if (!TreeHasLeft(middle))
@@ -589,13 +597,17 @@ stop:
static Tree SplayUpdateLeftSpine(SplayTree splay, Tree node, Tree child)
{
+ SplayUpdateNodeFunction updateNode;
+
AVERT_CRITICAL(SplayTree, splay);
AVERT_CRITICAL(Tree, node);
AVERT_CRITICAL(Tree, child);
+
+ updateNode = splay->updateNode;
while(node != TreeEMPTY) {
Tree parent = TreeLeft(node);
TreeSetLeft(node, child); /* un-reverse pointer */
- splay->updateNode(splay, node);
+ updateNode(splay, node);
child = node;
node = parent;
}
@@ -606,13 +618,17 @@ static Tree SplayUpdateLeftSpine(SplayTree splay, Tree node, Tree child)
static Tree SplayUpdateRightSpine(SplayTree splay, Tree node, Tree child)
{
+ SplayUpdateNodeFunction updateNode;
+
AVERT_CRITICAL(SplayTree, splay);
AVERT_CRITICAL(Tree, node);
AVERT_CRITICAL(Tree, child);
+
+ updateNode = splay->updateNode;
while (node != TreeEMPTY) {
Tree parent = TreeRight(node);
TreeSetRight(node, child); /* un-reverse pointer */
- splay->updateNode(splay, node);
+ updateNode(splay, node);
child = node;
node = parent;
}
@@ -725,7 +741,6 @@ static Compare SplaySplay(SplayTree splay, TreeKey key,
/* SplayTreeInsert -- insert a node into a splay tree
- *
*
* This function is used to insert a node into the tree. Splays the
* tree at the node's key. If an attempt is made to insert a node that
@@ -915,10 +930,9 @@ Bool SplayTreeNeighbours(Tree *leftReturn, Tree *rightReturn,
Count count = SplayDebugCount(splay);
#endif
-
- AVERT(SplayTree, splay);
- AVER(leftReturn != NULL);
- AVER(rightReturn != NULL);
+ AVERT_CRITICAL(SplayTree, splay);
+ AVER_CRITICAL(leftReturn != NULL);
+ AVER_CRITICAL(rightReturn != NULL);
if (SplayTreeIsEmpty(splay)) {
*leftReturn = *rightReturn = TreeEMPTY;
@@ -936,14 +950,14 @@ Bool SplayTreeNeighbours(Tree *leftReturn, Tree *rightReturn,
break;
case CompareLESS:
- AVER(!TreeHasLeft(stateStruct.middle));
+ AVER_CRITICAL(!TreeHasLeft(stateStruct.middle));
*rightReturn = stateStruct.middle;
*leftReturn = stateStruct.leftLast;
found = TRUE;
break;
case CompareGREATER:
- AVER(!TreeHasRight(stateStruct.middle));
+ AVER_CRITICAL(!TreeHasRight(stateStruct.middle));
*leftReturn = stateStruct.middle;
*rightReturn = stateStruct.rightFirst;
found = TRUE;
@@ -1101,8 +1115,8 @@ static Compare SplayFindFirstCompare(Tree node, TreeKey key)
void *testClosure;
SplayTree splay;
- AVERT(Tree, node);
- AVER(key != NULL);
+ AVERT_CRITICAL(Tree, node);
+ AVER_CRITICAL(key != NULL);
/* Lift closure values into variables so that they aren't aliased by
calls to the test functions. */
@@ -1140,8 +1154,8 @@ static Compare SplayFindLastCompare(Tree node, TreeKey key)
void *testClosure;
SplayTree splay;
- AVERT(Tree, node);
- AVER(key != NULL);
+ AVERT_CRITICAL(Tree, node);
+ AVER_CRITICAL(key != NULL);
/* Lift closure values into variables so that they aren't aliased by
calls to the test functions. */
@@ -1195,10 +1209,10 @@ Bool SplayFindFirst(Tree *nodeReturn, SplayTree splay,
SplayFindClosureStruct closureStruct;
Bool found;
- AVER(nodeReturn != NULL);
- AVERT(SplayTree, splay);
- AVER(FUNCHECK(testNode));
- AVER(FUNCHECK(testTree));
+ AVER_CRITICAL(nodeReturn != NULL);
+ AVERT_CRITICAL(SplayTree, splay);
+ AVER_CRITICAL(FUNCHECK(testNode));
+ AVER_CRITICAL(FUNCHECK(testTree));
if (SplayTreeIsEmpty(splay) ||
!testTree(splay, SplayTreeRoot(splay), testClosure))
@@ -1258,10 +1272,10 @@ Bool SplayFindLast(Tree *nodeReturn, SplayTree splay,
SplayFindClosureStruct closureStruct;
Bool found;
- AVER(nodeReturn != NULL);
- AVERT(SplayTree, splay);
- AVER(FUNCHECK(testNode));
- AVER(FUNCHECK(testTree));
+ AVER_CRITICAL(nodeReturn != NULL);
+ AVERT_CRITICAL(SplayTree, splay);
+ AVER_CRITICAL(FUNCHECK(testNode));
+ AVER_CRITICAL(FUNCHECK(testTree));
if (SplayTreeIsEmpty(splay) ||
!testTree(splay, SplayTreeRoot(splay), testClosure))