From dc7050326aae0f6e14b527fd69ed77332098ccd5 Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Mon, 20 Aug 2007 18:02:18 +0100 Subject: [PATCH 1/6] Mps br/diagtag: my first awldescribe(), using statistic_write. Copied from Perforce Change: 163157 ServerID: perforce.ravenbrook.com --- mps/code/poolawl.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/mps/code/poolawl.c b/mps/code/poolawl.c index d52e157ce24..73730fad891 100644 --- a/mps/code/poolawl.c +++ b/mps/code/poolawl.c @@ -1203,6 +1203,53 @@ static void AWLWalk(Pool pool, Seg seg, FormattedObjectsStepMethod f, } +/* AWLDescribe -- describe the contents of the AWL pool + * + */ +static Res AWLDescribe(Pool pool, mps_lib_FILE *stream) +{ + AWL awl; + Res res; + + if(!CHECKT(Pool, pool)) + return ResFAIL; + awl = Pool2AWL(pool); + if(!CHECKT(AWL, awl)) + return ResFAIL; + if(stream == NULL) + return ResFAIL; + + res = WriteF(stream, + "AWL $P {\n", (WriteFP)awl, + " pool $P ($U)\n", + (WriteFP)pool, (WriteFU)pool->serial, + NULL); + if(res != ResOK) + return res; + + res = WriteF(stream, " @@@@ AWLDescribe not complete yet.\n", + NULL); + if(res != ResOK) + return res; + + res = WriteF(stream, + STATISTIC_WRITE(" goodScans $U\n", (WriteFU)awl->stats.goodScans) + STATISTIC_WRITE(" badScans $U\n", (WriteFU)awl->stats.badScans) + STATISTIC_WRITE(" savedScans $U\n", (WriteFU)awl->stats.savedScans) + STATISTIC_WRITE(" savedAccesses $U\n", (WriteFU)awl->stats.savedAccesses) + STATISTIC_WRITE(" declined $U\n", (WriteFU)awl->stats.declined) + NULL); + if(res != ResOK) + return res; + + res = WriteF(stream, "} AWL $P\n", (WriteFP)awl, NULL); + if(res != ResOK) + return res; + + return ResOK; +} + + /* AWLPoolClass -- the class definition */ DEFINE_POOL_CLASS(AWLPoolClass, this) @@ -1226,6 +1273,7 @@ DEFINE_POOL_CLASS(AWLPoolClass, this) this->fixEmergency = AWLFix; this->reclaim = AWLReclaim; this->walk = AWLWalk; + this->describe = AWLDescribe; } From a534cb9949656230da51ea59048e4734910ed51d Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Fri, 14 Sep 2007 18:38:09 +0100 Subject: [PATCH 2/6] Mps br/diagtag: investigate awl behaviour: trace.c: TraceStart: for all pool->class->name == "AWL", PoolDescribe. config.h: bigger DIAG_BUFFER_SIZE diag.c: just the bits of AWL PoolDescribe that I want. Copied from Perforce Change: 163242 ServerID: perforce.ravenbrook.com --- mps/code/config.h | 2 +- mps/code/diag.c | 13 +++++++++++++ mps/code/trace.c | 10 ++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mps/code/config.h b/mps/code/config.h index a93f7ee60b8..7e01f3a0e0d 100644 --- a/mps/code/config.h +++ b/mps/code/config.h @@ -308,7 +308,7 @@ #ifdef DIAG_WITH_STREAM_AND_WRITEF /* DIAG_BUFFER_SIZE: 10 screenfuls: 10x80x25 = 20000 */ -#define DIAG_BUFFER_SIZE ((Size)20000) +#define DIAG_BUFFER_SIZE ((Size)880000) #else #define DIAG_BUFFER_SIZE ((Size)1) #endif diff --git a/mps/code/diag.c b/mps/code/diag.c index 46fb1c4342a..02e26be8245 100644 --- a/mps/code/diag.c +++ b/mps/code/diag.c @@ -27,9 +27,22 @@ typedef struct RuleStruct { /* RulesGlobal -- throw away some diags (see INSTRUCTIONS below) */ +struct RuleStruct RulesGlobalX[] = { + { "+", "*", "*", "*" }, + { "-", "DIAGTEST_", "*", "*" }, + { NULL, "", "", "" } +}; + struct RuleStruct RulesGlobal[] = { { "+", "*", "*", "*" }, { "-", "DIAGTEST_", "*", "*" }, + { "-", "TraceStart", "*", "*" }, + { "+", "TraceStart", "*", "because code" }, + { "+", "TraceStart", "*", "goodScans " }, + { "+", "TraceStart", "*", "badScans " }, + { "+", "TraceStart", "*", "savedScans " }, + { "+", "TraceStart", "*", "savedAccesses " }, + { "+", "TraceStart", "*", "declined " }, { NULL, "", "", "" } }; diff --git a/mps/code/trace.c b/mps/code/trace.c index 7c3c6fe97e1..a98a2c9b3cf 100644 --- a/mps/code/trace.c +++ b/mps/code/trace.c @@ -1942,6 +1942,16 @@ void TraceStart(Trace trace, double mortality, double finishingTime) TraceStartGenDesc_diag(&arena->topGen, -1); } + { + Ring node, nextNode; + RING_FOR(node, &(ArenaGlobals(arena)->poolRing), nextNode) { + Pool pool = RING_ELT(Pool, arenaRing, node); + if (StringEqual(pool->class->name, "AWL")) { + DIAG( (void) PoolDescribe(pool, DIAG_STREAM); ); + } + } + } + DIAG_END( "TraceStart" ); res = RootsIterate(ArenaGlobals(arena), rootGrey, (void *)trace); From 9a143fdd0b2d6c2df7cbad8ae349b0d873da2d83 Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Thu, 27 Sep 2007 17:53:16 +0100 Subject: [PATCH 3/6] Mps br/awldiag: branch from master Investigating poolawl.c, improving diagnostic output. Copied from Perforce Change: 163294 ServerID: perforce.ravenbrook.com From df4387b93b11e8945196afac8aa746d49c19078f Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Wed, 3 Oct 2007 18:30:17 +0100 Subject: [PATCH 4/6] Mps awldiag: (comments only) clarify arenaaccess, accessset, and awlsegalloc Copied from Perforce Change: 163342 ServerID: perforce.ravenbrook.com --- mps/code/global.c | 14 +++++++++++--- mps/code/mpmtypes.h | 10 ++++++++++ mps/code/poolawl.c | 5 ++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/mps/code/global.c b/mps/code/global.c index b86f9baaca9..71247fc3f56 100644 --- a/mps/code/global.c +++ b/mps/code/global.c @@ -531,9 +531,17 @@ MutatorFaultContext mps_exception_info = NULL; /* ArenaAccess -- deal with an access fault * - * This is called when a protected address is accessed. The mode - * corresponds to which mode flags need to be cleared in order for the - * access to continue. */ + * Our signal handler calls this when a protected address is accessed. + * + * "mode" states which attempted operations (AccessREAD, WRITE) were + * caught by the signal handler. Note: this may be a conservative + * exaggeration, because the handler cannot always precisely determine + * what operation was being attempted (see for example + * protsgix.c#1:.sigh.mode). + * + * MPS must lift prohibition of the "mode" operations, in order for the + * access to continue. + */ Bool ArenaAccess(Addr addr, AccessSet mode, MutatorFaultContext context) { diff --git a/mps/code/mpmtypes.h b/mps/code/mpmtypes.h index c92662d2c94..c8a9c353751 100644 --- a/mps/code/mpmtypes.h +++ b/mps/code/mpmtypes.h @@ -270,10 +270,20 @@ typedef Res (*RootScanRegMethod)(ScanState ss, Thread thread, void *p, size_t s) #define SigInvalid ((Sig)0x51915BAD) #define SizeMAX ((Size)-1) + +/* AccessSet -- operations that access memory + * + * Note: in most MPS source code, these operation codes are used in the + * non-permissive sense, ie: "intercept/trap/prohibit this operation". + * For example, when a segment is added to the greylist, mutator read + * access is _prohibited_, by calling: + * ShieldRaise(arena, seg, AccessREAD); + */ #define AccessSetEMPTY ((AccessSet)0) /* */ #define AccessREAD ((AccessSet)(1<<0)) #define AccessWRITE ((AccessSet)(1<<1)) #define AccessSetWIDTH (2) + #define RefSetEMPTY BS_EMPTY(RefSet) #define RefSetUNIV BS_UNIV(RefSet) #define ZoneSetEMPTY BS_EMPTY(ZoneSet) diff --git a/mps/code/poolawl.c b/mps/code/poolawl.c index 73730fad891..9d3ea5e5bff 100644 --- a/mps/code/poolawl.c +++ b/mps/code/poolawl.c @@ -451,7 +451,10 @@ static Res AWLSegCreate(AWLSeg *awlsegReturn, } -/* AWLSegAlloc -- allocate an object in a given segment */ +/* AWLSegAlloc -- find a space >= size in the segment (or fail) + * + * Note: despite the name, this function does not actually 'allocate'. + */ static Bool AWLSegAlloc(Addr *baseReturn, Addr *limitReturn, AWLSeg awlseg, AWL awl, Size size) From 95b2ce185801e19bdffbfd51e045e2c3d24e57f5 Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Wed, 24 Oct 2007 16:44:19 +0100 Subject: [PATCH 5/6] Mps br/awldiag: des/poolawl: add guide; preserve initial design. Copied from Perforce Change: 163424 ServerID: perforce.ravenbrook.com --- mps/design/poolawl/index.html | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/mps/design/poolawl/index.html b/mps/design/poolawl/index.html index 78078d0ec0a..85ba91b6d34 100644 --- a/mps/design/poolawl/index.html +++ b/mps/design/poolawl/index.html @@ -6,7 +6,7 @@ -Automatic weak linked +Automatic Weak Linked @@ -26,8 +26,28 @@
+

Automatic Weak Linked

+ +

This document contains a guide to the Automatic Weak Linked poolclass, followed by the historical initial design. References, History, Copyright and License are at the end.

+ +
+ +

Guide

+ +

Readership: any MPS developer. Not confidential.

+ +

Introduction

+

+ [not yet written] +

+ + +
+ +

Initial Design

+ 
                          AUTOMATIC WEAK LINKED
                            design.mps.poolawl
@@ -495,6 +515,8 @@ buffers and pools and exit
+
+

A. References

@@ -525,13 +547,15 @@ buffers and pools and exit - - - + + + + +
2002-06-07RBConverted from MMInfo database design document.
2007-10-24RHSKAdd guide; preserve initial design.
From 91648d6e010f2fd4084b3299e790d72fe140ebd8 Mon Sep 17 00:00:00 2001 From: Richard Kistruck Date: Tue, 4 Dec 2007 18:50:11 +0000 Subject: [PATCH 6/6] Mps br/awldiag: tool/mpsclasses.py: show the mps class hierarchy A simple parser that finds DEFINE_CLASS (et al) macros, builds the MPS class hierarchy, and prints the tree to stdout. Copied from Perforce Change: 163588 ServerID: perforce.ravenbrook.com --- mps/tool/mpsclasses.py | 297 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100755 mps/tool/mpsclasses.py diff --git a/mps/tool/mpsclasses.py b/mps/tool/mpsclasses.py new file mode 100755 index 00000000000..5710e4e4bf8 --- /dev/null +++ b/mps/tool/mpsclasses.py @@ -0,0 +1,297 @@ +#!/usr/bin/python + +# mpsclasses.py: show the MPS class hierarchy +# $Id$ +# Copyright (c) 2007 Ravenbrook Limited. See end of file for license. + +# This file parses MPS C source for MPS class definitions +# (DEFINE_CLASS() et al), and prints out the class hierarchy. +# +# USAGE: +# +# Invoke it with the list of MPS source files, like this: +# cd code +# ../tool/mpsclasses.py *.c +# +# Example output line (to stdout): +# : : : : : AWLPoolClass (POOL) <<< AbstractCollectPoolClass +++ Pool:Format +# +# This means: +# the MPS class "AWLPoolClass" is defined +# using the DEFINE_POOL_CLASS macro (not DEFINE_CLASS) +# its parent is AbstractCollectPoolClass +# it uses the PoolClassMixInFormat mixin. + + +# VERSIONS +# -- first checkin -- +# Version 06, Version 07 +# tidied and commented +# +# Version 05 +# sorted: shallowest first, then alphabetically +# +# Version 04 +# shows forest of classes, one class per line + +import re +import fileinput + +# examples: +""" + DEFINE_CLASS(RankBufClass, class) + { + INHERIT_CLASS(class, SegBufClass); + ... + } + + DEFINE_POOL_CLASS(AWLPoolClass, this) + { + INHERIT_CLASS(this, AbstractCollectPoolClass); + PoolClassMixInFormat(this); + ... + } +""" + +# Grammar: +# C -> D I? M? E +# + + +class GramTerm(object): + """a GramTerm holds information about a terminal in the grammar""" + def __init__(self, name, patt, show): + super(GramTerm, self).__init__() + self.name = name + self.patt = patt + self.show = show + + +# ___D___ -- DEFINE_(_)CLASS ( name +# +patt_D = re.compile( r""" + DEFINE_ + (?P [A-Z]*) [_]? # family + CLASS + \s* [(] + \s* (?P [A-Za-z_][A-Za-z0-9_]* ) # name +""", re.VERBOSE) + +def show_D(match): + print ("%s (%s)" % (match.group("name", "family"))) + +term_D = GramTerm("D", patt_D, show_D) + + +# ___I___ -- INHERIT_CLASS ( this , parentname +# +patt_I = re.compile( r""" + INHERIT_CLASS + \s* [(] + \s* ( [A-Za-z_][A-Za-z0-9_]* ) # this + \s* [,] + \s* (?P [A-Za-z_][A-Za-z0-9_]* ) # parentname +""", re.VERBOSE) + +def show_I(match): + print " <<< %s" % (match.group("parentname")) + +term_I = GramTerm("I", patt_I, show_I) + + +# ___M___ -- ClassMixIn ( this ) ; +# +patt_M = re.compile( r""" + (?P [A-Za-z_][A-Za-z0-9_]*) # family + ClassMixIn + (?P [A-Za-z0-9_]*) # mixin + \s* [(] + \s* ( [A-Za-z_][A-Za-z0-9_]* ) # this + \s* [)] + \s* [;] +""", re.VERBOSE) + +def show_M(match): + print " +++ %s [%s]" % (match.group("mixin", "family")) + +term_M = GramTerm("M", patt_M, show_M) + + +# ___E___ -- } +# +patt_E = re.compile( r""" + [}] +""", re.VERBOSE) + +def show_E(match): + print "..." + +term_E = GramTerm("E", patt_E, show_E) + + +class NoMoreInput(Exception): + """NoMoreInput""" + +class MPSClass(object): + """represents an MPS Class, as created by DEFINE_CLASS() et al""" + def __init__(self, match_D): + """init from match_D""" + super(MPSClass, self).__init__() + self.name, self.family = match_D.group("name", "family") + self.parentname = None + self.mixinnames = [] + self.descendants = 0 + + def add_I(self, match_I): + """add_I: parentname to inherit from""" + assert(self.parentname == None) + self.parentname = match_I.group("parentname") + + def add_M(self, match_M): + """add_M: mixinname""" + self.mixinnames.append("%s:%s" % match_M.group("family", "mixin")) + + def show(self, prefix): + mix = "" + sep = " +++ " + for m in self.mixinnames: + mix = mix + sep + m + sep = ", " + print ("%s%s (%s) <<< %s%s" + % (prefix, self.name, self.family, self.parentname, mix)) + +def main(): + lines = fileinput.input() + tops = [] + CChildListfromName = {} + # (a cmp() function works even with very old Pythons) + ChildrenSort = lambda C1, C2: cmp( + [C1.descendants, C1.name.lower()], + [C2.descendants, C2.name.lower()]) + + def calc_descendants_tree(CT): + for child in CChildListfromName[CT.name]: + CT.descendants += calc_descendants_tree(child) + return CT.descendants + 1 + + def show_tree(CT, prefix): + CT.show(prefix) + CChildListfromName[CT.name].sort(ChildrenSort) + for child in CChildListfromName[CT.name]: + show_tree(child, prefix + ": ") + + try: + while True: + C = next_C(lines) + + # with C + #C.show() + + if C.name not in CChildListfromName: + CChildListfromName[C.name] = [] + if C.parentname == None: + tops.append(C) + else: + if C.parentname not in CChildListfromName: + CChildListfromName[C.parentname] = [] + CChildListfromName[C.parentname].append(C) + + C = None + + except NoMoreInput: + assert(C == None) + + tot = 0 + for CT in tops: + tot += calc_descendants_tree(CT) + + tops.sort(ChildrenSort) + for CT in tops: + show_tree(CT, "") + print + print "%d classes in total" % tot + +def next_C(lines): + """find the next class definition C""" + + # .DIME -- find next DEFINE_CLASS + for l in lines: + match_D = term_D.patt.search(l) + if match_D != None: + break + for t in (term_I, term_M): + match = t.patt.search(l) + if match != None: + print "ERROR parsing .DIME: got I or M: ", l + raise + else: + raise NoMoreInput() + # with match_D + #term_D.show(match_D) + C = MPSClass(match_D) + + # D.IME -- find next InheritFrom, Mixin, or End + for l in lines: + for t in (term_D, term_I, term_M, term_E): + match = t.patt.search(l) + if match != None: + break + else: + continue + # with t + #t.show(match) + if t == term_D: + print "ERROR parsing D.IME: got (another) D; expected I, M, or E" + raise + elif t == term_I: + C.add_I(match) + elif t == term_M: + C.add_M(match) + elif t == term_E: + return C + else: + raise NoMoreInput() + +if __name__ == "__main__": + main() + + +# C. COPYRIGHT AND LICENSE +# +# Copyright (C) 2007 Ravenbrook Limited . +# All rights reserved. This is an open source license. Contact +# Ravenbrook for commercial licensing options. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Redistributions in any form must be accompanied by information on how +# to obtain complete source code for this software and any accompanying +# software that uses this software. The source code must either be +# included in the distribution or be available for no more than the cost +# of distribution plus a nominal fee, and must be freely redistributable +# under reasonable conditions. For an executable file, complete source +# code means the source code for all modules it contains. It does not +# include source code for modules or files that typically accompany the +# major components of the operating system on which the executable file +# runs. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.