diff --git a/mps/code/arena.c b/mps/code/arena.c index 944b154d334..7733de760fb 100644 --- a/mps/code/arena.c +++ b/mps/code/arena.c @@ -8,7 +8,7 @@ #include "tract.h" #include "poolmv.h" #include "mpm.h" -#include "cbs.h" +#include "zonedcbs.h" #include "bt.h" @@ -16,17 +16,14 @@ SRCID(arena, "$Id$"); #define ArenaControlPool(arena) MV2Pool(&(arena)->controlPoolStruct) -#define ArenaCBSBlockPool(arena) (&(arena)->cbsBlockPoolStruct.poolStruct) -#define ArenaFreeCBS(arena) (&(arena)->freeCBS) -#define ArenaZoneCBS(arena, z) (&(arena)->zoneCBS[z]) +#define ArenaCBSBlockPool(arena) (&(arena)->zonedCBSBlockPoolStruct.poolStruct) +#define ArenaZonedCBS(arena) (&(arena)->zonedCBSStruct) /* Forward declarations */ static void ArenaTrivCompact(Arena arena, Trace trace); static void arenaFreePage(Arena arena, Addr base, Pool pool); -static Res arenaCBSInit(Arena arena); -static void arenaCBSFinish(Arena arena); /* ArenaTrivDescribe -- produce trivial description of an arena */ @@ -154,20 +151,9 @@ Bool ArenaCheck(Arena arena) CHECKL(LocusCheck(arena)); CHECKL(BoolCheck(arena->hasFreeCBS)); - if (arena->hasFreeCBS) { - Index i; - CHECKD(Pool, ArenaCBSBlockPool(arena)); - CHECKD(CBS, ArenaFreeCBS(arena)); - for (i = 0; i < NELEMS(arena->zoneCBS); ++i) - CHECKD(CBS, ArenaZoneCBS(arena, i)); - } + if (arena->hasFreeCBS) + ZonedCBSCheck(ArenaZonedCBS(arena)); - /* TODO: Thorough check summing CBSs against totals. The sum of the - sizes of the contents of the zone CBSs and the freeCBS should equal - the total allocatable free space in the chunks. */ - - AVER(NELEMS(arena->zoneCBS) == sizeof(ZoneSet) * CHAR_BIT); - return TRUE; } @@ -216,7 +202,24 @@ Res ArenaInit(Arena arena, ArenaClass class, Align alignment) arena->sig = ArenaSig; - res = arenaCBSInit(arena); + /* Initialise a pool to hold the arena's CBS blocks. This pool can't be + allowed to extend itself using ArenaAlloc because it is used during + ArenaAlloc, so MFSExtendSelf is set to FALSE. Failures to extend are + handled where the CBS is used. */ + + MPS_ARGS_BEGIN(piArgs) { + MPS_ARGS_ADD(piArgs, MPS_KEY_MFS_UNIT_SIZE, sizeof(CBSBlockStruct)); + MPS_ARGS_ADD(piArgs, MPS_KEY_EXTEND_BY, arena->alignment); + MPS_ARGS_ADD(piArgs, MFSExtendSelf, FALSE); + MPS_ARGS_DONE(piArgs); + res = PoolInit(ArenaCBSBlockPool(arena), arena, PoolClassMFS(), piArgs); + } MPS_ARGS_END(piArgs); + AVER(res == ResOK); /* no allocation, no failure expected */ + if (res != ResOK) + goto failMFSInit; + + res = ZonedCBSInit(ArenaZonedCBS(arena), arena, ArenaCBSBlockPool(arena), + ArenaAlign(arena)); if (res != ResOK) goto failCBSInit; @@ -229,8 +232,10 @@ Res ArenaInit(Arena arena, ArenaClass class, Align alignment) return ResOK; failReservoirInit: - arenaCBSFinish(arena); + ZonedCBSFinish(ArenaZonedCBS(arena)); failCBSInit: + PoolFinish(ArenaCBSBlockPool(arena)); +failMFSInit: GlobalsFinish(ArenaGlobals(arena)); failGlobalsInit: return res; @@ -328,6 +333,17 @@ void ArenaFinish(Arena arena) /* ArenaDestroy -- destroy the arena */ +static void arenaMFSPageFreeVisitor(Pool pool, Addr base, Size size, + void *closureP, Size closureS) +{ + AVERT(Pool, pool); + UNUSED(closureP); + UNUSED(closureS); + UNUSED(size); + AVER(size == ArenaAlign(PoolArena(pool))); + arenaFreePage(PoolArena(pool), base, pool); +} + void ArenaDestroy(Arena arena) { @@ -341,7 +357,17 @@ void ArenaDestroy(Arena arena) arena->poolReady = FALSE; ControlFinish(arena); - arenaCBSFinish(arena); + /* We must tear down the freeCBS before the chunks, because pages + containing CBS blocks might be allocated in those chunks. */ + AVER(arena->hasFreeCBS); + arena->hasFreeCBS = FALSE; + ZonedCBSFinish(ArenaZonedCBS(arena)); + + /* The CBS block pool can't free its own memory via ArenaFree because + that would use the ZonedCBS. */ + MFSFinishTracts(ArenaCBSBlockPool(arena), + arenaMFSPageFreeVisitor, NULL, 0); + PoolFinish(ArenaCBSBlockPool(arena)); /* Call class-specific finishing. This will call ArenaFinish. */ (*arena->class->finish)(arena); @@ -646,11 +672,7 @@ static void arenaExcludePage(Arena arena, Range pageRange) RangeStruct oldRange; Res res; - res = CBSDelete(&oldRange, - ArenaZoneCBS(arena, AddrZone(arena, RangeBase(pageRange))), - pageRange); - if (res == ResFAIL) /* block wasn't in zone CBS */ - res = CBSDelete(&oldRange, ArenaFreeCBS(arena), pageRange); + res = ZonedCBSDelete(&oldRange, ArenaZonedCBS(arena), pageRange); AVER(res == ResOK); /* we just gave memory to the CBSs */ } @@ -661,23 +683,22 @@ static void arenaExcludePage(Arena arena, Range pageRange) * in the basic allocator, so we allocate pages specially. */ -static Res arenaCBSInsert(Range rangeReturn, Arena arena, CBS cbs, Range range) +static Res arenaCBSInsert(Range rangeReturn, Arena arena, Range range) { Res res; AVER(rangeReturn != NULL); AVERT(Arena, arena); - AVERT(CBS, cbs); AVERT(Range, range); - res = CBSInsert(rangeReturn, cbs, range); + res = ZonedCBSInsert(rangeReturn, ArenaZonedCBS(arena), range); if (res == ResLIMIT) { /* freeCBS MFS pool ran out of blocks */ RangeStruct pageRange; res = arenaExtendCBSBlockPool(&pageRange, arena); if (res != ResOK) return res; - res = CBSInsert(rangeReturn, &arena->freeCBS, range); + res = ZonedCBSInsert(rangeReturn, ArenaZonedCBS(arena), range); AVER(res == ResOK); /* we just gave memory to the CBSs */ arenaExcludePage(arena, &pageRange); } @@ -695,17 +716,15 @@ static Res arenaCBSInsert(Range rangeReturn, Arena arena, CBS cbs, Range range) * IMPORTANT: May update rangeIO. */ -static void arenaCBSInsertSteal(Range rangeReturn, Arena arena, - CBS cbs, Range rangeIO) +static void arenaCBSInsertSteal(Range rangeReturn, Arena arena, Range rangeIO) { Res res; AVER(rangeReturn != NULL); AVERT(Arena, arena); - AVERT(CBS, cbs); AVERT(Range, rangeIO); - res = arenaCBSInsert(rangeReturn, arena, cbs, rangeIO); + res = arenaCBSInsert(rangeReturn, arena, rangeIO); if (res != ResOK) { Addr pageBase; @@ -726,7 +745,7 @@ static void arenaCBSInsertSteal(Range rangeReturn, Arena arena, MFSExtend(ArenaCBSBlockPool(arena), pageBase, ArenaAlign(arena)); /* Try again. */ - res = CBSInsert(rangeReturn, cbs, rangeIO); + res = ZonedCBSInsert(rangeReturn, ArenaZonedCBS(arena), rangeIO); AVER(res == ResOK); /* we just gave memory to the CBS */ } @@ -749,11 +768,11 @@ Res ArenaFreeCBSInsert(Arena arena, Addr base, Addr limit) AVERT(Arena, arena); RangeInit(&range, base, limit); - res = arenaCBSInsert(&oldRange, arena, ArenaFreeCBS(arena), &range); + res = arenaCBSInsert(&oldRange, arena, &range); if (res != ResOK) return res; - /* Make sure it didn't coalesce. */ + /* .chunk.no-coalesce: Make sure it didn't coalesce. */ AVER(RangesEqual(&oldRange, &range)); return ResOK; @@ -772,258 +791,15 @@ void ArenaFreeCBSDelete(Arena arena, Addr base, Addr limit) Res res; RangeInit(&range, base, limit); - res = CBSDelete(&oldRange, &arena->freeCBS, &range); + res = ZonedCBSDelete(&oldRange, ArenaZonedCBS(arena), &range); - if (res == ResFAIL) { - /* The address space must be divided between the freeCBS and zoneCBSs. */ - while (base < limit) { - Addr stripeLimit = AddrAlignUp(AddrAdd(base, 1), ArenaStripeSize(arena)); - CBS zoneCBS; - RangeStruct stripe; - if (stripeLimit > limit) - stripeLimit = limit; - zoneCBS = ArenaZoneCBS(arena, AddrZone(arena, base)); - RangeInit(&stripe, base, stripeLimit); - res = CBSDelete(&oldRange, ArenaFreeCBS(arena), &stripe); - - /* Delete and skip over the rest of the block we found in the - freeCBS, up to the next block that's in a zoneCBS (or the end). */ - if (res == ResOK && !RangesEqual(&oldRange, &stripe)) { - Addr skipLimit = RangeLimit(&oldRange); - if (skipLimit > limit) - skipLimit = limit; - if (stripeLimit < skipLimit) { - RangeStruct restOfBlock; - RangeInit(&restOfBlock, stripeLimit, skipLimit); - res = CBSDelete(&oldRange, ArenaFreeCBS(arena), &restOfBlock); - AVER(RangesEqual(&restOfBlock, &oldRange)); - base = skipLimit; - continue; - } - } - - if (res == ResFAIL) { - res = CBSDelete(&oldRange, zoneCBS, &stripe); - AVER(res != ResFAIL); /* must be in one of the CBSs */ - AVER(RangesEqual(&oldRange, &stripe)); - } - - AVER(res == ResOK); /* end of range, shouldn't fail */ - /* Could possibly delete up to the end of oldRange and save time. */ - base = stripeLimit; - } - } - - /* Shouldn't be any other kind of failure. */ + /* Shouldn't be any other kind of failure because we were only deleting + a non-coalesced block. See .chunk.no-coalesce. */ + /* FIXME: Document this rule about CBSs */ AVER(res == ResOK); } -/* arenaCBSInit -- initialise the arena's free CBSs */ - -static Res arenaCBSInit(Arena arena) -{ - Res res; - Index i; - - AVERT(Arena, arena); - AVER(!arena->hasFreeCBS); - - /* Initialise a pool to hold the arena's CBS blocks. This pool can't be - allowed to extend itself using ArenaAlloc because it is used during - ArenaAlloc, so MFSExtendSelf is set to FALSE. Failures to extend are - handled where the CBS is used. */ - - MPS_ARGS_BEGIN(piArgs) { - MPS_ARGS_ADD(piArgs, MPS_KEY_MFS_UNIT_SIZE, sizeof(CBSBlockStruct)); - MPS_ARGS_ADD(piArgs, MPS_KEY_EXTEND_BY, arena->alignment); - MPS_ARGS_ADD(piArgs, MFSExtendSelf, FALSE); - MPS_ARGS_DONE(piArgs); - res = PoolInit(ArenaCBSBlockPool(arena), arena, PoolClassMFS(), piArgs); - } MPS_ARGS_END(piArgs); - AVER(res == ResOK); /* no allocation, no failure expected */ - if (res != ResOK) - goto failMFSInit; - - /* Initialise the freeCBS. */ - MPS_ARGS_BEGIN(cbsiArgs) { - MPS_ARGS_ADD(cbsiArgs, CBSBlockPool, ArenaCBSBlockPool(arena)); - MPS_ARGS_DONE(cbsiArgs); - res = CBSInit(arena, ArenaFreeCBS(arena), arena, - arena->alignment, TRUE, cbsiArgs); - } MPS_ARGS_END(cbsiArgs); - AVER(res == ResOK); /* no allocation, no failure expected */ - if (res != ResOK) - goto failCBSInit; - /* Note that although freeCBS is initialised, it doesn't have any memory - for its blocks, so hasFreeCBS remains FALSE until later. */ - - /* Initialise the zoneCBSs. */ - for (i = 0; i < NELEMS(arena->zoneCBS); ++i) { - MPS_ARGS_BEGIN(cbsiArgs) { - MPS_ARGS_ADD(cbsiArgs, CBSBlockPool, ArenaCBSBlockPool(arena)); - MPS_ARGS_DONE(cbsiArgs); - res = CBSInit(arena, ArenaZoneCBS(arena, i), arena, - arena->alignment, TRUE, cbsiArgs); - } MPS_ARGS_END(cbsiArgs); - AVER(res == ResOK); /* no allocation, no failure expected */ - if (res != ResOK) - goto failZoneCBSInit; - } - - return ResOK; - -failZoneCBSInit: - while (i > 0) { - --i; - CBSFinish(&arena->zoneCBS[i]); - } - CBSFinish(&arena->freeCBS); -failCBSInit: - PoolFinish(ArenaCBSBlockPool(arena)); -failMFSInit: - return res; -} - - -/* arenaCBSFinish -- finish the arena's free CBSs */ - -static void arenaMFSPageFreeVisitor(Pool pool, Addr base, Size size, - void *closureP, Size closureS) -{ - AVERT(Pool, pool); - UNUSED(closureP); - UNUSED(closureS); - UNUSED(size); - AVER(size == ArenaAlign(PoolArena(pool))); - arenaFreePage(PoolArena(pool), base, pool); -} - -static void arenaCBSFinish(Arena arena) -{ - Index i; - - AVERT(Arena, arena); - - /* We must tear down the freeCBS before the chunks, because pages - containing CBS blocks might be allocated in those chunks. */ - AVER(arena->hasFreeCBS); - arena->hasFreeCBS = FALSE; - for (i = 0; i < NELEMS(arena->zoneCBS); ++i) - CBSFinish(&arena->zoneCBS[i]); - CBSFinish(&arena->freeCBS); - - /* The CBS block pool can't free its own memory via ArenaFree because - that would use the freeCBS. */ - MFSFinishTracts(ArenaCBSBlockPool(arena), - arenaMFSPageFreeVisitor, NULL, 0); - PoolFinish(ArenaCBSBlockPool(arena)); -} - - -/* arenaAllocFromCBS -- allocate memory using the free CBS - * - * The free CBS contains all the free address space we have in chunks, - * so this is the primary method of allocation. - * FIXME: Needs to take a "high" option to use CBSFindLastInZones. - */ - -static Bool arenaAllocFindInZoneCBS(Range rangeReturn, - Arena arena, ZoneSet zones, Bool high, - Size size) -{ - RangeStruct oldRange; - Index i; - CBSFindMethod find; - FindDelete fd; - - /* TODO: Use __builtin_ffsl or similar like this for 5% speed-up. - zones &= nonEmptyZoneCBS; - while (zones != ZoneSetEMPTY) { - int z = __builtin_ffsl((long)zones) - 1; - ... - zones &= ~((ZoneSet)1 << z); - } */ - - /* TODO: Does "high" really make sense for zone stripes? */ - /* TODO: How do we disable zones anyway? Just make zoneShift = WORD_WIDTH? - DL points out that if we had two zones, they'd both be blacklisted. */ - - /* Even though we have no guarantee that zone zero is at the bottom end - of anything, it makes sense to reverse the search when "high" is set, - because the point of it (presumably) is to separate memory usage into - two sets (high and low) that avoid interference. */ - - find = high ? CBSFindLast : CBSFindFirst; - fd = high ? FindDeleteHIGH : FindDeleteLOW; - - for (i = 0; i < NELEMS(arena->zoneCBS); ++i) { - Index zone = high ? NELEMS(arena->zoneCBS) - i - 1 : i; - if (ZoneSetIsMember(zones, zone) && - find(rangeReturn, &oldRange, ArenaZoneCBS(arena, zone), size, fd)) - return TRUE; - } - return FALSE; -} - -static Bool arenaAllocFindInFreeCBS(Range rangeReturn, - Arena arena, ZoneSet zones, Bool high, - Size size) -{ - Res res; - RangeStruct oldRange, restRange; - Addr allocLimit, stripeLimit, oldLimit, limit; - Index zone; - CBS zoneCBS; - - if (high) - res = CBSFindLastInZones(rangeReturn, &oldRange, &arena->freeCBS, size, - arena, zones); - else - res = CBSFindFirstInZones(rangeReturn, &oldRange, &arena->freeCBS, size, - arena, zones); - - if (res == ResLIMIT) { /* CBS block pool full */ - RangeStruct pageRange; - res = arenaExtendCBSBlockPool(&pageRange, arena); - if (res != ResOK) { /* couldn't get any memory for CBS nodes */ - /* Things are pretty dire at this point. We can't even get a bit - of memory to remember about allocating memory. So we'll pretend - that we couldn't find any. FIXME: Think about this. */ - return FALSE; - } - arenaExcludePage(arena, &pageRange); - res = CBSFindFirstInZones(rangeReturn, &oldRange, &arena->freeCBS, size, - arena, zones); - AVER(res == ResOK); - } - - if (res == ResFAIL) /* no suitable range, also defensive */ - return FALSE; - - /* Add the rest of the zone stripe to the zoneCBS so that subsequent - allocations in the zone are fast. */ - allocLimit = RangeLimit(rangeReturn); - stripeLimit = AddrAlignUp(allocLimit, (Size)1 << ArenaZoneShift(arena)); - oldLimit = RangeLimit(&oldRange); - limit = oldLimit < stripeLimit ? oldLimit : stripeLimit; - RangeInit(&restRange, allocLimit, limit); - AVER(RangesNest(&oldRange, &restRange)); - if (allocLimit < limit) { - res = CBSDelete(&oldRange, ArenaFreeCBS(arena), &restRange); - AVER(res == ResOK); /* we should just be bumping up a base */ - zone = AddrZone(arena, RangeBase(&restRange)); - zoneCBS = ArenaZoneCBS(arena, zone); - res = arenaCBSInsert(&oldRange, arena, zoneCBS, &restRange); - if (res != ResOK) { /* disasterously short on memory */ - /* Put it back. This should succeed. */ - res = CBSInsert(&oldRange, ArenaFreeCBS(arena), &restRange); - AVER(res == ResOK); - } - } - return TRUE; -} - static Res arenaAllocFromCBS(Tract *tractReturn, ZoneSet zones, Bool high, Size size, Pool pool) { @@ -1034,6 +810,7 @@ static Res arenaAllocFromCBS(Tract *tractReturn, ZoneSet zones, Bool high, Index baseIndex; Count pages; Res res; + FindDelete fd; AVER(tractReturn != NULL); /* ZoneSet is arbitrary */ @@ -1043,14 +820,37 @@ static Res arenaAllocFromCBS(Tract *tractReturn, ZoneSet zones, Bool high, AVER(SizeIsAligned(size, arena->alignment)); /* Step 1. Find a range of address space. */ + fd = high ? FindDeleteHIGH : FindDeleteLOW; + + if (high) + res = ZonedCBSFindLast(&range, &oldRange, ArenaZonedCBS(arena), + zones, size, fd); + else + res = ZonedCBSFindFirst(&range, &oldRange, ArenaZonedCBS(arena), + zones, size, fd); + + if (res == ResLIMIT) { + RangeStruct pageRange; + res = arenaExtendCBSBlockPool(&pageRange, arena); + if (res != ResOK) /* disasterously short on memory */ + return res; + arenaExcludePage(arena, &pageRange); + if (high) + res = ZonedCBSFindLast(&range, &oldRange, ArenaZonedCBS(arena), + zones, size, fd); + else + res = ZonedCBSFindFirst(&range, &oldRange, ArenaZonedCBS(arena), + zones, size, fd); + AVER(res != ResLIMIT); + } + + if (res == ResFAIL) /* out of address space */ + return ResRESOURCE; + + AVER(res == ResOK); /* unexpected error from ZoneCBS */ + if (res != ResOK) /* defensive return */ + return res; - /* We could check zoneCBS if size > stripeSize to avoid looking in the - zoneCBSs, but this probably isn't a win. */ - - if (!arenaAllocFindInZoneCBS(&range, arena, zones, high, size)) - if (!arenaAllocFindInFreeCBS(&range, arena, zones, high, size)) - return ResRESOURCE; - /* Step 2. Make memory available in the address space range. */ b = CHUNK_OF_ADDR(&chunk, arena, range.base); @@ -1072,8 +872,7 @@ static Res arenaAllocFromCBS(Tract *tractReturn, ZoneSet zones, Bool high, failMark: NOTREACHED; /* FIXME */ { - Res insertRes = arenaCBSInsert(&oldRange, arena, - ArenaFreeCBS(arena), &range); + Res insertRes = ZonedCBSInsert(&oldRange, ArenaZonedCBS(arena), &range); AVER(insertRes == ResOK); /* We only just deleted it. */ /* If the insert does fail, we lose some address space permanently. */ } @@ -1250,7 +1049,6 @@ void ArenaFree(Addr base, Size size, Pool pool) Addr wholeBase; Size wholeSize; RangeStruct range, oldRange; - CBS cbs; AVERT(Pool, pool); AVER(base != NULL); @@ -1286,22 +1084,7 @@ void ArenaFree(Addr base, Size size, Pool pool) RangeInit(&range, base, limit); - /* If the freed address space is entirely within one zone, add it to - the zone CBS so that it can be reallocated quickly. Otherwise add - it to the freeCBS, and it'll get chopped up later. */ - if (size <= ArenaStripeSize(arena) && - AddrZone(arena, base) == AddrZone(arena, AddrAdd(base, size - 1))) - cbs = ArenaZoneCBS(arena, AddrZone(arena, base)); - else - cbs = ArenaFreeCBS(arena); - arenaCBSInsertSteal(&oldRange, arena, cbs, &range); - - /* TODO: Consider moving empty zone stripes back to freeCBS. At the - moment we do not do this, partly for simplicity, and partly to - keep large objects apart from smaller ones, at the possible cost - of address space fragmentation. We probably do not want to do it - eagerly in any case, but lazily if we're unable to find address - space, even though that reduces first-fit. */ + arenaCBSInsertSteal(&oldRange, arena, &range); /* may update range */ (*arena->class->free)(RangeBase(&range), RangeSize(&range), pool); diff --git a/mps/code/mpmst.h b/mps/code/mpmst.h index ddffdf60570..4c0747e3b9c 100644 --- a/mps/code/mpmst.h +++ b/mps/code/mpmst.h @@ -625,6 +625,22 @@ typedef struct CBSStruct { } CBSStruct; +/* ZoneCBSStruct -- zoned coalescing block structure + * + * See . + */ + +#define ZonedCBSSig ((Sig)0x519209ED) /* SIGnature ZONED */ + +typedef struct ZonedCBSStruct { + Sig sig; + Arena arena; + Pool blockPool; /* shared pool holding CBS blocks */ + CBSStruct freeStruct; /* CBS of free address space not in zoneCBS */ + CBSStruct zoneStruct[MPS_WORD_WIDTH]; /* free address space per zone */ +} ZonedCBSStruct; + + /* ArenaStruct -- generic arena * * See . */ @@ -660,9 +676,8 @@ typedef struct mps_arena_s { ChunkCacheEntryStruct chunkCache; /* just one entry */ Bool hasFreeCBS; /* Is freeCBS available? */ - MFSStruct cbsBlockPoolStruct; /* Shared pool for CBS blocks */ - CBSStruct freeCBS; /* CBS of free address space not in zoneCBS */ - CBSStruct zoneCBS[MPS_WORD_WIDTH]; /* free address space per zone */ + MFSStruct zonedCBSBlockPoolStruct; + ZonedCBSStruct zonedCBSStruct; ZoneSet freeZones; /* zones not yet allocated */ /* locus fields () */ diff --git a/mps/code/mps.c b/mps/code/mps.c index 329c67de9a4..85c34f81359 100644 --- a/mps/code/mps.c +++ b/mps/code/mps.c @@ -74,6 +74,7 @@ #include "range.c" #include "freelist.c" #include "sa.c" +#include "zonedcbs.c" /* Additional pool classes */ diff --git a/mps/code/mps.xcodeproj/project.pbxproj b/mps/code/mps.xcodeproj/project.pbxproj index 9524465561a..69b47b767dc 100644 --- a/mps/code/mps.xcodeproj/project.pbxproj +++ b/mps/code/mps.xcodeproj/project.pbxproj @@ -1250,6 +1250,8 @@ 3114A6BA156E9768001E0AA3 /* walkt0.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = walkt0.c; sourceTree = ""; }; 3114A6C6156E9815001E0AA3 /* mpseventcnv */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = mpseventcnv; sourceTree = BUILT_PRODUCTS_DIR; }; 3114A6D0156E9829001E0AA3 /* eventcnv.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = eventcnv.c; sourceTree = ""; }; + 3115E70118BED7E000385449 /* zonedcbs.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = zonedcbs.h; sourceTree = ""; }; + 3115E70218BEDA1100385449 /* zonedcbs.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = zonedcbs.c; sourceTree = ""; }; 31160D921899540D0071EB17 /* abq.txt */ = {isa = PBXFileReference; lastKnownFileType = text; name = abq.txt; path = ../design/abq.txt; sourceTree = ""; }; 31160D931899540D0071EB17 /* alloc-frame.txt */ = {isa = PBXFileReference; lastKnownFileType = text; name = "alloc-frame.txt"; path = "../design/alloc-frame.txt"; sourceTree = ""; }; 31160D941899540D0071EB17 /* arena.txt */ = {isa = PBXFileReference; lastKnownFileType = text; name = arena.txt; path = ../design/arena.txt; sourceTree = ""; }; @@ -2135,6 +2137,8 @@ 31EEAC0E156AB27B00714D05 /* walk.c */, 3112ED3A18ABC57F00CC531A /* sa.h */, 3112ED3B18ABC75200CC531A /* sa.c */, + 3115E70118BED7E000385449 /* zonedcbs.h */, + 3115E70218BEDA1100385449 /* zonedcbs.c */, ); name = "MPM Core"; sourceTree = ""; diff --git a/mps/code/zonedcbs.c b/mps/code/zonedcbs.c new file mode 100644 index 00000000000..4a732f546d7 --- /dev/null +++ b/mps/code/zonedcbs.c @@ -0,0 +1,377 @@ +/* zonedcbs.c: ZONED COALESCING BLOCK STRUCTURE IMPLEMENTATION + * + * $Id$ + * Copyright (c) 2014 Ravenbrook Limited. See end of file for license. + * + * A Zone CBS is like a CBS but allows for efficient allocation in zones. + * Allocation in zones gives control over some parts of an object's address, + * so that we can later apply fast filters on the critical path. + * The Zone CBS is mainly used by the arena to allocate blocks to pools. + */ + +#include "zonedcbs.h" +#include "mpm.h" + +#define ZonedCBSBlockPool(zcbs) RVALUE((zcbs)->blockPool) +#define ZonedCBSFreeCBS(zcbs) (&(zcbs)->freeStruct) +#define ZonedCBSZoneCBS(zcbs, z) (&(zcbs)->zoneStruct[z]) +#define ZonedCBSNZones(zcbs) NELEMS((zcbs)->zoneStruct) + + +/* ZonedCBSCheck -- check consistency of zoned CBS structure */ + +Bool ZonedCBSCheck(ZonedCBS zcbs) +{ + Index i; + + CHECKS(ZonedCBS, zcbs); + CHECKU(Arena, zcbs->arena); + CHECKD(Pool, ZonedCBSBlockPool(zcbs)); + CHECKD(CBS, ZonedCBSFreeCBS(zcbs)); + CHECKL(ZonedCBSNZones(zcbs) == sizeof(ZoneSet) * CHAR_BIT); + CHECKL(ZonedCBSNZones(zcbs) == NELEMS(zcbs->zoneStruct)); + for (i = 0; i < ZonedCBSNZones(zcbs); ++i) + CHECKD(CBS, ZonedCBSZoneCBS(zcbs, i)); + + /* TODO: Thorough check summing CBSs against totals. The sum of the + sizes of the contents of the zone CBSs and the freeCBS should equal + the total allocatable free space in the chunks. The CBS ADT + probably ought to maintain totals like this too. */ + + return TRUE; +} + + +/* ZonedCBSInit -- initialise a Zoned CBS */ + +Res ZonedCBSInit(ZonedCBS zcbs, Arena arena, Pool blockPool, Align alignment) +{ + Index i; + Res res; + + AVER(zcbs != NULL); + AVERT(Pool, blockPool); + + zcbs->arena = arena; + zcbs->blockPool = blockPool; + + /* Initialise the freeCBS. */ + MPS_ARGS_BEGIN(cbsiArgs) { + MPS_ARGS_ADD(cbsiArgs, CBSBlockPool, blockPool); + MPS_ARGS_DONE(cbsiArgs); + res = CBSInit(arena, ZonedCBSFreeCBS(zcbs), zcbs, alignment, TRUE, cbsiArgs); + } MPS_ARGS_END(cbsiArgs); + AVER(res == ResOK); /* no allocation, no failure expected */ + if (res != ResOK) + goto failCBSInit; + /* Note that although freeCBS is initialised, it doesn't have any memory + for its blocks, so hasFreeCBS remains FALSE until later. */ + + /* Initialise the zoneCBSs. */ + for (i = 0; i < NELEMS(zcbs->zoneStruct); ++i) { + MPS_ARGS_BEGIN(cbsiArgs) { + MPS_ARGS_ADD(cbsiArgs, CBSBlockPool, blockPool); + MPS_ARGS_DONE(cbsiArgs); + res = CBSInit(arena, ZonedCBSZoneCBS(zcbs, i), arena, alignment, TRUE, cbsiArgs); + } MPS_ARGS_END(cbsiArgs); + AVER(res == ResOK); /* no allocation, no failure expected */ + if (res != ResOK) + goto failZoneCBSInit; + } + + zcbs->sig = ZonedCBSSig; + AVERT(ZonedCBS, zcbs); + + return ResOK; + +failZoneCBSInit: + while (i > 0) { + --i; + CBSFinish(ZonedCBSZoneCBS(zcbs, i)); + } + CBSFinish(ZonedCBSFreeCBS(zcbs)); +failCBSInit: + return res; +} + + +/* ZonedCBSFinish -- finish the zoned CBS */ + +void ZonedCBSFinish(ZonedCBS zcbs) +{ + Index i; + + AVERT(ZonedCBS, zcbs); + + zcbs->sig = SigInvalid; + + /* FIXME: Should be asserting that CBSs are empty? */ + + for (i = 0; i < ZonedCBSNZones(zcbs); ++i) + CBSFinish(ZonedCBSZoneCBS(zcbs, i)); + CBSFinish(ZonedCBSFreeCBS(zcbs)); +} + + +/* ZonedCBSInsert -- insert a range into the zoned CBS + * + * We just insert it into the free CBS. It will get split up and cached + * in the zoned CBSs by ZonedCBSFindFirst and ZonedCBSFindLast. + */ + +Res ZonedCBSInsert(Range rangeReturn, ZonedCBS zcbs, Range range) +{ + AVERT(ZonedCBS, zcbs); + + /* TODO: Consider moving empty zone stripes back to freeCBS. At the + moment we do not do this, partly for simplicity, and partly to + keep large objects apart from smaller ones, at the possible cost + of address space fragmentation. We probably do not want to do it + eagerly in any case, but lazily if we're unable to find address + space, even though that reduces first-fit. */ + + return CBSInsert(rangeReturn, ZonedCBSFreeCBS(zcbs), range); +} + + +/* ZonedCBSDelete -- delete a range from the zoned CBS + * + * The range may be split between the zone CBSs and the free CBS on zone + * stripe boundaries, in which case we have to iterate. + * + * FIXME: Document guarantees about res. + */ + +Res ZonedCBSDelete(Range oldRange, ZonedCBS zcbs, Range range) +{ + Res res; + ZoneSet zs; + Addr base, limit; + + AVER(oldRange != NULL); + AVERT(ZonedCBS, zcbs); + AVERT(Range, range); + + zs = ZoneSetOfRange(zcbs->arena, RangeBase(range), RangeLimit(range)); + if (ZoneSetIsSingle(zs)) { + Index zone = AddrZone(zcbs->arena, RangeBase(range)); + CBS zoneCBS = ZonedCBSZoneCBS(zcbs, zone); + res = CBSDelete(oldRange, zoneCBS, range); + if (res != ResFAIL) /* block was in zone CBS */ + return res; + } + + res = CBSDelete(oldRange, ZonedCBSFreeCBS(zcbs), range); + + if (res != ResFAIL) /* block was in free CBS */ + return res; + + /* The range may be divided between the free CBS and zone CBSs. */ + base = RangeBase(range); + limit = RangeLimit(range); + while (base < limit) { + Addr stripeLimit; + CBS zoneCBS; + RangeStruct stripe, oldStripe; + Index zone; + + stripeLimit = AddrAlignUp(AddrAdd(base, 1), ArenaStripeSize(zcbs->arena)); + if (stripeLimit > limit) + stripeLimit = limit; + + zone = AddrZone(zcbs->arena, base); + AVER(AddrZone(zcbs->arena, AddrSub(stripeLimit, 1)) == zone); + zoneCBS = ZonedCBSZoneCBS(zcbs, zone); + + RangeInit(&stripe, base, stripeLimit); + res = CBSDelete(&oldStripe, ZonedCBSFreeCBS(zcbs), &stripe); + + /* Optimisation: delete and skip over the rest of the block we + found in the free CBS, up to the next block that's in a zone CBS + (or the end). */ + if (res == ResOK && !RangesEqual(&oldStripe, &stripe)) { + Addr skipLimit = RangeLimit(&oldStripe); + if (skipLimit > limit) + skipLimit = limit; + if (stripeLimit < skipLimit) { + RangeStruct restOfBlock; + RangeInit(&restOfBlock, stripeLimit, skipLimit); + res = CBSDelete(&oldStripe, ZonedCBSFreeCBS(zcbs), &restOfBlock); + AVER(res == ResOK); /* FIXME: is this right? */ + AVER(RangesEqual(&oldStripe, &restOfBlock)); + base = skipLimit; + continue; + } + } + + if (res == ResFAIL) { + res = CBSDelete(&oldStripe, zoneCBS, &stripe); + AVER(res != ResFAIL); /* FIXME: not in any of the CBSs, return res */ + AVER(RangesEqual(&oldStripe, &stripe)); + } + + AVER(res == ResOK); /* FIXME: end of range, shouldn't fail? */ + + base = stripeLimit; + } + + /* Shouldn't be any other kind of failure. */ + AVER(res == ResOK); + return ResOK; +} + + +/* ZonedCBSFindFirst, ZonedCBSFindLast -- find a range in the zoned CBS */ + +static Res ZonedCBSFind(Range rangeReturn, + Range oldRangeReturn, + ZonedCBS zcbs, + ZoneSet zones, + Size size, + FindDelete findDelete, + Bool high) +{ + CBSFindMethod find; + Index i; + RangeStruct restRange; + Addr allocLimit, stripeLimit, oldLimit, limit; + Res res; + + AVER(rangeReturn != NULL); + AVER(oldRangeReturn != NULL); + AVERT(ZonedCBS, zcbs); + AVER(size > 0); + /* FIXME: findDelete? */ + /* FIXME: ZoneSet? */ + AVER(BoolCheck(high)); + + find = high ? CBSFindLast : CBSFindFirst; + + /* Try the zone CBSs first. */ + + /* We could check zoneCBS if size > stripeSize to avoid looking in the + zoneCBSs, but this probably isn't a win for arena allocation. */ + + /* TODO: Does "high" really make sense for zone stripes? */ + /* TODO: How do we disable zones anyway? Just make zoneShift = WORD_WIDTH? + DL points out that if we had two zones, they'd both be blacklisted. */ + + /* Even though we have no guarantee that zone zero is at the bottom end + of anything, it makes sense to reverse the search when "high" is set, + because the point of it (presumably) is to separate memory usage into + two sets (high and low) that avoid interference. */ + + /* TODO: Consider masking zones against a ZoneSet of non-empty zone CBSs */ + + for (i = 0; i < ZonedCBSNZones(zcbs); ++i) { + Index zone = high ? ZonedCBSNZones(zcbs) - i - 1 : i; + if (ZoneSetIsMember(zones, zone) && + find(rangeReturn, oldRangeReturn, ZonedCBSZoneCBS(zcbs, zone), + size, findDelete)) + return ResOK; + } + + /* Try the free CBS. */ + + if (high) + res = CBSFindLastInZones(rangeReturn, oldRangeReturn, + ZonedCBSFreeCBS(zcbs), size, zcbs->arena, zones); + else + res = CBSFindFirstInZones(rangeReturn, oldRangeReturn, + ZonedCBSFreeCBS(zcbs), size, zcbs->arena, zones); + if (res != ResOK) + return res; + + /* TODO: We may have failed to find because the address space is + fragmented between the zone CBSs and the free CBS. This isn't + a very important case for the arena, but it does make the Zone CBS + technically incorrect, and we should fix it. Flush the zone CBSs + back to the free CBS? */ + + /* Add the rest of the zone stripe to the zoneCBS so that subsequent + allocations in the zone are fast. This is what the ZonedCBS is + all about! */ + allocLimit = RangeLimit(rangeReturn); + stripeLimit = AddrAlignUp(allocLimit, ArenaStripeSize(zcbs->arena)); + oldLimit = RangeLimit(oldRangeReturn); + limit = oldLimit < stripeLimit ? oldLimit : stripeLimit; + RangeInit(&restRange, allocLimit, limit); + AVER(RangesNest(oldRangeReturn, &restRange)); + if (allocLimit < limit) { + Index zone; + CBS zoneCBS; + RangeStruct oldRange; + res = CBSDelete(&oldRange, ZonedCBSFreeCBS(zcbs), &restRange); + AVER(res == ResOK); /* we should just be bumping up a base */ + zone = AddrZone(zcbs->arena, RangeBase(&restRange)); + zoneCBS = ZonedCBSZoneCBS(zcbs, zone); + res = CBSInsert(&oldRange, zoneCBS, &restRange); + AVER(res != ResOK || RangesEqual(&oldRange, &restRange)); /* shouldn't coalesce */ + if (res != ResOK) { /* disasterously short on memory */ + /* Put it back. This should succeed. */ + res = CBSInsert(&oldRange, ZonedCBSFreeCBS(zcbs), &restRange); + AVER(res == ResOK); + } + } + + return ResOK; +} + + +Res ZonedCBSFindFirst(Range rangeReturn, Range oldRangeReturn, + ZonedCBS zcbs, ZoneSet zones, + Size size, FindDelete findDelete) +{ + return ZonedCBSFind(rangeReturn, oldRangeReturn, zcbs, zones, size, + findDelete, FALSE); +} + +Bool ZonedCBSFindLast(Range rangeReturn, Range oldRangeReturn, + ZonedCBS zcbs, ZoneSet zones, + Size size, FindDelete findDelete) +{ + return ZonedCBSFind(rangeReturn, oldRangeReturn, zcbs, zones, size, + findDelete, TRUE); +} + + +/* C. COPYRIGHT AND LICENSE + * + * Copyright (C) 2014 Ravenbrook Limited . + * All rights reserved. This is an open source license. Contact + * Ravenbrook for commercial licensing options. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Redistributions in any form must be accompanied by information on how + * to obtain complete source code for this software and any accompanying + * software that uses this software. The source code must either be + * included in the distribution or be available for no more than the cost + * of distribution plus a nominal fee, and must be freely redistributable + * under reasonable conditions. For an executable file, complete source + * code means the source code for all modules it contains. It does not + * include source code for modules or files that typically accompany the + * major components of the operating system on which the executable file + * runs. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/mps/code/zonedcbs.h b/mps/code/zonedcbs.h new file mode 100644 index 00000000000..6b30e062301 --- /dev/null +++ b/mps/code/zonedcbs.h @@ -0,0 +1,79 @@ +/* zonedcbs.h: ZONE COALESCING BLOCK STRUCTURE INTERFACE + * + * $Id$ + * Copyright (c) 2014 Ravenbrook Limited. See end of file for license. + * + * A Zone CBS is like a CBS but allows for efficient allocation in zones. + * Allocation in zones gives control over some parts of an object's address, + * so that we can later apply fast filters on the critical path. + * The Zone CBS is mainly used by the arena to allocate blocks to pools. + */ + +#ifndef zonedcbs_h +#define zonedcbs_h + + +typedef struct ZonedCBSStruct *ZonedCBS; + +/* ZoneCBSStruct is in mpmst.h because it is inlined in the ArenaStruct. */ + + +/* Basically the same interface as for CBS. See . */ + +extern Res ZonedCBSInit(ZonedCBS zcbs, Arena arena, Pool blockPool, + Align alignment); +extern void ZonedCBSFinish(ZonedCBS zcbs); +extern Bool ZonedCBSCheck(ZonedCBS zcbs); + +extern Res ZonedCBSInsert(Range rangeReturn, ZonedCBS zcbs, Range range); +extern Res ZonedCBSDelete(Range oldRange, ZonedCBS zcbs, Range range); +extern Bool ZonedCBSFindFirst(Range rangeReturn, Range oldRangeReturn, + ZonedCBS zcbs, ZoneSet zones, + Size size, FindDelete findDelete); +extern Bool ZonedCBSFindLast(Range rangeReturn, Range oldRangeReturn, + ZonedCBS zcbs, ZoneSet zones, + Size size, FindDelete findDelete); + + +#endif /* zonedcbs_h */ + +/* C. COPYRIGHT AND LICENSE + * + * Copyright (C) 2014 Ravenbrook Limited . + * All rights reserved. This is an open source license. Contact + * Ravenbrook for commercial licensing options. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Redistributions in any form must be accompanied by information on how + * to obtain complete source code for this software and any accompanying + * software that uses this software. The source code must either be + * included in the distribution or be available for no more than the cost + * of distribution plus a nominal fee, and must be freely redistributable + * under reasonable conditions. For an executable file, complete source + * code means the source code for all modules it contains. It does not + * include source code for modules or files that typically accompany the + * major components of the operating system on which the executable file + * runs. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */