diff --git a/mps/code/arenavm.c b/mps/code/arenavm.c index 7eba0f036f0..46f7fe1157c 100644 --- a/mps/code/arenavm.c +++ b/mps/code/arenavm.c @@ -47,7 +47,6 @@ typedef struct VMChunkStruct { VM vm; /* virtual memory handle */ Addr overheadMappedLimit; /* limit of pages mapped for overhead */ BT pageTableMapped; /* indicates mapped state of page table */ - RingStruct spareRing; Sig sig; /* */ } VMChunkStruct; @@ -82,6 +81,7 @@ typedef struct VMArenaStruct { /* VM arena structure */ Size extendMin; /* minimum arena increment */ ArenaVMExtendedCallback extended; ArenaVMContractedCallback contracted; + RingStruct spareRing; /* spare (free but mapped) tracts */ Sig sig; /* */ } VMArenaStruct; @@ -91,8 +91,8 @@ typedef struct VMArenaStruct { /* VM arena structure */ /* Forward declarations */ -static Size arenaUnmapSpare(Arena arena, Size size); -static Size chunkUnmapSpare(Chunk chunk, Size size); +static Size arenaUnmapSpare(Arena arena, Size size, Chunk filterChunk); +static void chunkUnmapSpare(Chunk chunk); extern ArenaClass VMArenaClassGet(void); extern ArenaClass VMNZArenaClassGet(void); static void VMCompact(Arena arena, Trace trace); @@ -116,7 +116,6 @@ static Bool VMChunkCheck(VMChunk vmchunk) CHECKL(AddrAdd((Addr)vmchunk->pageTableMapped, BTSize(chunk->pageTablePages)) <= vmchunk->overheadMappedLimit); /* .improve.check-table: Could check the consistency of the tables. */ - CHECKL(RingCheck(&vmchunk->spareRing)); return TRUE; } @@ -186,6 +185,8 @@ static Bool VMArenaCheck(VMArena vmArena) CHECKL(VMMapped(primary->vm) <= arena->committed); } + CHECKL(RingCheck(&vmArena->spareRing)); + /* FIXME: Can't check VMParams */ return TRUE; @@ -391,7 +392,6 @@ static Res VMChunkInit(Chunk chunk, BootBlock boot) } BTResRange(vmChunk->pageTableMapped, 0, chunk->pageTablePages); - RingInit(&vmChunk->spareRing); return ResOK; @@ -413,10 +413,9 @@ static void vmChunkDestroy(Chunk chunk) vmChunk = Chunk2VMChunk(chunk); AVERT(VMChunk, vmChunk); - chunkUnmapSpare(chunk, AddrOffset(chunk->base, chunk->limit)); + chunkUnmapSpare(chunk); AVER(BTIsResRange(vmChunk->pageTableMapped, 0, chunk->pageTablePages)); - AVER(RingIsSingle(&vmChunk->spareRing)); vmChunk->sig = SigInvalid; vm = vmChunk->vm; @@ -533,6 +532,7 @@ static Res VMArenaInit(Arena *arenaReturn, ArenaClass class, ArgList args) vmArena->vm = arenaVM; vmArena->spareSize = 0; + RingInit(&vmArena->spareRing); /* Copy the stack-allocated VM parameters into their home in the VMArena. */ AVER(sizeof(vmArena->vmParams) == sizeof(vmParams)); @@ -633,6 +633,11 @@ static void VMArenaFinish(Arena arena) Chunk chunk = RING_ELT(Chunk, chunkRing, node); vmChunkDestroy(chunk); } + + /* Destroying the chunks should have purged and removed all spare pages. */ + AVER(RingIsSingle(&vmArena->spareRing)); + + /* Destroying the chunks should leave only the arena's own VM. */ AVER(arena->committed == VMMapped(arenaVM)); vmArena->sig = SigInvalid; @@ -678,7 +683,8 @@ static void VMArenaSpareCommitExceeded(Arena arena) if (arena->spareCommitted > arena->spareCommitLimit) (void)arenaUnmapSpare(arena, - arena->spareCommitted - arena->spareCommitLimit); + arena->spareCommitted - arena->spareCommitLimit, + NULL); } @@ -1386,7 +1392,7 @@ static Res vmAllocComm(Addr *baseReturn, Tract *baseTractReturn, at the new address. */ /* TODO: Investigate implementing VMRemap so that we can guarantee success if we have enough spare pages. */ - if (arenaUnmapSpare(arena, size) == 0) + if (arenaUnmapSpare(arena, size, NULL) == 0) goto failPagesMap; res = pagesMarkAllocated(vmArena, vmChunk, baseIndex, pages, pool); } @@ -1439,95 +1445,93 @@ static Res VMNZAlloc(Addr *baseReturn, Tract *baseTractReturn, } -/* chunkUnmapSpare -- return spare pages within a chunk to the OS +/* chunkUnmapAroundPage -- unmap spare pages in a chunk including this one * - * The size is the desired amount to purge, and the amount that was purged is - * returned. + * Unmap the spare page passed, and possibly other pages in the chunk, so + * that the total size unmapped doesn't exceed the size passed. */ -static Size chunkUnmapSpare(Chunk chunk, Size size) +static Size chunkUnmapAroundPage(Chunk chunk, Size size, Page page) { VMChunk vmChunk; Size purged = 0; Size pageSize; + Index basePage, limitPage; + Index basePTI, limitPTI; + Addr base, limit; AVERT(Chunk, chunk); vmChunk = Chunk2VMChunk(chunk); AVERT(VMChunk, vmChunk); - /* max is arbitrary */ + AVER(PageType(page) == PageTypeSpare); + /* size is arbitrary */ pageSize = ChunkPageSize(chunk); /* Not required by this code, but expected. */ AVER(SizeIsAligned(size, pageSize)); + - /* Start by looking at the oldest page on the spare ring, to try to - get some LRU behaviour from the spare pages cache. */ - while (purged < size && !RingIsSingle(&vmChunk->spareRing)) { - Page page = PageOfSpareRing(RingNext(&vmChunk->spareRing)); - Index basePage = (Index)(page - chunk->pageTable); - Index limitPage = basePage; - Index basePTI, limitPTI; - Addr base, limit; + basePage = (Index)(page - chunk->pageTable); + limitPage = basePage; - /* To avoid excessive calls to the OS, coalesce with spare pages above - and below, even though these may be recent. */ - do { - sparePageRelease(vmChunk, limitPage); - PageInit(chunk, limitPage); - ++limitPage; - purged += pageSize; - } while (purged < size && - limitPage < chunk->pages && - pageType(vmChunk, limitPage) == PageTypeSpare); - while (purged < size && - basePage > 0 && - pageType(vmChunk, basePage - 1) == PageTypeSpare) { - --basePage; - sparePageRelease(vmChunk, basePage); - PageInit(chunk, basePage); - purged += pageSize; - } + /* To avoid excessive calls to the OS, coalesce with spare pages above + and below, even though these may be recent. */ + do { + sparePageRelease(vmChunk, limitPage); + PageInit(chunk, limitPage); + ++limitPage; + purged += pageSize; + } while (purged < size && + limitPage < chunk->pages && + pageType(vmChunk, limitPage) == PageTypeSpare); + while (purged < size && + basePage > 0 && + pageType(vmChunk, basePage - 1) == PageTypeSpare) { + --basePage; + sparePageRelease(vmChunk, basePage); + PageInit(chunk, basePage); + purged += pageSize; + } - vmArenaUnmap(VMChunkVMArena(vmChunk), - vmChunk->vm, - PageIndexBase(chunk, basePage), - PageIndexBase(chunk, limitPage)); + vmArenaUnmap(VMChunkVMArena(vmChunk), + vmChunk->vm, + PageIndexBase(chunk, basePage), + PageIndexBase(chunk, limitPage)); - /* Now attempt to unmap the part of the page table that's no longer - in use because we've made a run of pages free. This scan will - also catch any adjacent unused pages, though they ought to have - been caught by previous scans. */ + /* Now attempt to unmap the part of the page table that's no longer + in use because we've made a run of pages free. This scan will + also catch any adjacent unused pages, though they ought to have + been caught by previous scans. */ - while (basePage > 0 && - pageDescIsMapped(vmChunk, basePage) && - pageType(vmChunk, basePage) == PageTypeFree) - --basePage; - /* basePage is zero or the lowest page whose descritor we can't unmap */ + while (basePage > 0 && + pageDescIsMapped(vmChunk, basePage) && + pageType(vmChunk, basePage) == PageTypeFree) + --basePage; + /* basePage is zero or the lowest page whose descritor we can't unmap */ - while (limitPage < chunk->pages && - pageDescIsMapped(vmChunk, limitPage) && - pageType(vmChunk, limitPage) == PageTypeFree) - ++limitPage; - /* limitPage is chunk->pages or the highest page whose descriptor we - can't unmap */ + while (limitPage < chunk->pages && + pageDescIsMapped(vmChunk, limitPage) && + pageType(vmChunk, limitPage) == PageTypeFree) + ++limitPage; + /* limitPage is chunk->pages or the highest page whose descriptor we + can't unmap */ - tablePagesUsed(&basePTI, &limitPTI, chunk, basePage, limitPage); - base = TablePageIndexBase(chunk, basePTI); - limit = TablePageIndexBase(chunk, limitPTI); - if (!pageDescIsMapped(vmChunk, basePage) || - pageType(vmChunk, basePage) != PageTypeFree) - base = AddrAdd(base, chunk->pageSize); - if (base < limit) { - if (limitPage < chunk->pages && - pageType(vmChunk, limitPage) != PageTypeFree) - limit = AddrSub(limit, chunk->pageSize); - if (base < limit) { /* might be nothing left to unmap */ - vmArenaUnmap(VMChunkVMArena(vmChunk), vmChunk->vm, base, limit); - BTResRange(vmChunk->pageTableMapped, - PageTablePageIndex(chunk, base), - PageTablePageIndex(chunk, limit)); - } + tablePagesUsed(&basePTI, &limitPTI, chunk, basePage, limitPage); + base = TablePageIndexBase(chunk, basePTI); + limit = TablePageIndexBase(chunk, limitPTI); + if (!pageDescIsMapped(vmChunk, basePage) || + pageType(vmChunk, basePage) != PageTypeFree) + base = AddrAdd(base, chunk->pageSize); + if (base < limit) { + if (limitPage < chunk->pages && + pageType(vmChunk, limitPage) != PageTypeFree) + limit = AddrSub(limit, chunk->pageSize); + if (base < limit) { /* might be nothing left to unmap */ + vmArenaUnmap(VMChunkVMArena(vmChunk), vmChunk->vm, base, limit); + BTResRange(vmChunk->pageTableMapped, + PageTablePageIndex(chunk, base), + PageTablePageIndex(chunk, limit)); } } @@ -1538,34 +1542,64 @@ static Size chunkUnmapSpare(Chunk chunk, Size size) /* arenaUnmapSpare -- return spare pages to the OS * * The size is the desired amount to purge, and the amount that was purged is - * returned. - * - * TODO: This is only an LRU algorithm within each chunk, and will tend to - * purge more pages from the first chunk in the ring. A single ring for - * the whole arena should work, since purging before destroying a chunk - * should remove any entries in that chunk. + * returned. If filter is not NULL, then only pages within that chunk are + * unmapped. */ #define ArenaChunkRing(arena) (&(arena)->chunkRing) -static Size arenaUnmapSpare(Arena arena, Size size) +static Size arenaUnmapSpare(Arena arena, Size size, Chunk filter) { - Ring node, next; + Ring node; Size purged = 0; + VMArena vmArena; AVERT(Arena, arena); + vmArena = Arena2VMArena(arena); + AVERT(VMArena, vmArena); + if (filter != NULL) + AVERT(Chunk, filter); - RING_FOR(node, ArenaChunkRing(arena), next) { - Chunk chunk = RING_ELT(Chunk, chunkRing, node); - if (purged >= size) - break; - purged += chunkUnmapSpare(chunk, size); + /* Start by looking at the oldest page on the spare ring, to try to + get some LRU behaviour from the spare pages cache. */ + /* RING_FOR won't work here, because chunkUnmapAroundPage deletes many + entries from the spareRing, often including the "next" entry. However, + it doesn't delete entries from other chunks, so we can use them to step + around the ring. */ + node = &vmArena->spareRing; + while (RingNext(node) != &vmArena->spareRing && purged < size) { + Ring next = RingNext(node); + Page page = PageOfSpareRing(next); + Chunk chunk; + Bool b; + /* Use the fact that the page table resides in the chunk to find the + chunk that owns the page. */ + b = ChunkOfAddr(&chunk, arena, (Addr)page); + AVER(b); + if (filter == NULL || chunk == filter) { + purged += chunkUnmapAroundPage(chunk, size - purged, page); + /* chunkUnmapAroundPage must delete the page it's passed from the ring, + or we can't make progress and there will be an infinite loop */ + AVER(RingNext(node) != next); + } else + node = next; } return purged; } +/* chunkUnmapSpare -- unmap all spare pages in a chunk */ + +static void chunkUnmapSpare(Chunk chunk) +{ + AVERT(Chunk, chunk); + (void)arenaUnmapSpare(ChunkArena(chunk), + AddrOffset(chunk->base, chunk->limit), + chunk); +} + + /* VMFree -- free a region in the arena */ static void VMFree(Addr base, Size size, Pool pool) @@ -1614,7 +1648,7 @@ static void VMFree(Addr base, Size size, Pool pool) /* We must init the page's spare ring because it is a union with the tract and will contain junk. */ RingInit(PageSpareRing(page)); - RingAppend(&vmChunk->spareRing, PageSpareRing(page)); + RingAppend(&vmArena->spareRing, PageSpareRing(page)); } arena->spareCommitted += ChunkPagesToSize(chunk, piLimit - piBase); BTResRange(chunk->allocTable, piBase, piLimit); @@ -1650,8 +1684,7 @@ static void VMCompact(Arena arena, Trace trace) /* Ensure there are no spare (mapped) pages left in the chunk. This could be short-cut if we're about to destroy the chunk, provided we can do the correct accounting in the arena. */ - chunkUnmapSpare(chunk, size); - AVER(RingIsSingle(&Chunk2VMChunk(chunk)->spareRing)); + chunkUnmapSpare(chunk); vmChunkDestroy(chunk);