From b123e07cb1009b60c03d3ef75ec850d13034d9a0 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Fri, 12 Oct 2012 14:46:05 +0100 Subject: [PATCH 01/14] Branch for measurement work. Copied from Perforce Change: 179849 ServerID: perforce.ravenbrook.com From d26c0518f1f812290bb0188d6014a962fc048ca6 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Fri, 12 Oct 2012 15:09:20 +0100 Subject: [PATCH 02/14] Very first draft of sqlite code; this barely compiles and is really just a proof-of-concept that i can write code to create, read, and write sqlite databases. Copied from Perforce Change: 179853 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 199 ++++++++++++++++++++++++++++++++++++++++++++ mps/code/eventsql.h | 51 ++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 mps/code/eventsql.c create mode 100644 mps/code/eventsql.h diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c new file mode 100644 index 00000000000..753a8aafc08 --- /dev/null +++ b/mps/code/eventsql.c @@ -0,0 +1,199 @@ +#include "config.h" + +#include "eventdef.h" +#include "eventpro.h" + +#include +#include +#include + +unsigned int verbosity = 4; + +#define LOG_ALWAYS 0 +#define LOG_OFTEN 1 +#define LOG_SOMETIMES 2 +#define LOG_SELDOM 3 +#define LOG_RARELY 4 + +static void vlog(unsigned int level, const char *format, va_list args) +{ + if (level <= verbosity) { + fflush(stderr); /* sync */ + fprintf(stderr, "log %d: ", level); + vfprintf(stderr, format, args); + fprintf(stderr, "\n"); + } +} + +static void log(unsigned int level, const char *format, ...) +{ + va_list args; + va_start(args, format); + vlog(level, format, args); + va_end(args); +} + +#if 0 /* UNUSED */ + +static void error(const char *format, ...) +{ + va_list args; + fprintf(stderr, "Fatal error: "); + va_start(args, format); + vlog(LOG_ALWAYS, format, args); + va_end(args); + exit(1); +} +#endif /* UNUSED */ + +static void sqlite_error(int res, sqlite3 *db, const char *format, ...) +{ + log(LOG_ALWAYS, "Fatal SQL error %d", res); + va_list args; + va_start(args, format); + vlog(LOG_ALWAYS, format, args); + va_end(args); + log(LOG_ALWAYS, "SQLite message: %s\n", sqlite3_errmsg(db)); + exit(1); +} + +static void openDatabase(sqlite3 **dbReturn) +{ + sqlite3 *db; + int res; + + const char *filename = getenv("MPS_EVENT_DATABASE"); + if(filename == NULL) + filename = "mpsevent.db"; + log(LOG_OFTEN, "Opening %s.", filename); + + res = sqlite3_open_v2(filename, + &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + NULL); /* use default sqlite_vfs object */ + + if (res != SQLITE_OK) + sqlite_error(res, db, "Opening %s failed", filename); + *dbReturn = db; + return; +} + +const char *createStatements[] = { + "CREATE TABLE IF NOT EXISTS event_kind (name TEXT," + " description TEXT," + " enum INTEGER PRIMARY KEY)", + "CREATE TABLE IF NOT EXISTS event_type (name TEXT," + " code INTEGER PRIMARY KEY," + " always INTEGER," + " kind INTEGER," + " FOREIGN KEY (kind) REFERENCES event_kind(enum));", +}; + +const char *populateStatements[] = { +}; + +#define EVENT_KIND_DO_INSERT(X, name, description) \ + res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of name \"" #name "\" failed."); \ + res = sqlite3_bind_text(statement, 2, description, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of description \"" description "\" failed."); \ + res = sqlite3_bind_int(statement, 3, i); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of enum %d failed.", i); \ + ++i; \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "event_kind insert of name \"" #name "\" failed."); \ + if (sqlite3_changes(db) != 0)\ + log(LOG_SOMETIMES, "Insert of event_kind row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset event_kind insert statement."); + +#define EVENT_TYPE_DO_INSERT(X, name, code, always, kind) \ + res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of name \"" #name "\" failed."); \ + res = sqlite3_bind_int(statement, 2, code); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of code %d failed.", code); \ + res = sqlite3_bind_int(statement, 3, always); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of always for name \"" #name "\" failed."); \ + res = sqlite3_bind_int(statement, 4, EventKind##kind); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of kind for name \"" #name "\" failed."); \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "event_type insert of name \"" #name "\" failed."); \ + if (sqlite3_changes(db) != 0) \ + log(LOG_SOMETIMES, "Insert of event_type row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset event_type insert statement."); + +static void fillTables(sqlite3 *db) +{ + int i; + sqlite3_stmt *statement; + + int res = sqlite3_prepare_v2(db, + "INSERT OR IGNORE INTO event_kind (name, description, enum)" + "VALUES (?, ?, ?)", + -1, /* prepare whole string as statement */ + &statement, + NULL); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_kind preparation failed"); + + i = 0; + EventKindENUM(EVENT_KIND_DO_INSERT, X); + + res = sqlite3_finalize(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_kind finalize failed"); + + res = sqlite3_prepare_v2(db, + "INSERT OR IGNORE INTO event_type (name, code, always, kind)" + "VALUES (?, ?, ?, ?)", + -1, /* prepare whole string as statement */ + &statement, + NULL); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_type preparation failed"); + + EVENT_LIST(EVENT_TYPE_DO_INSERT, X); + + res = sqlite3_finalize(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_type finalize failed"); +} + +static void makeTables(sqlite3 *db) +{ + int i; + + for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { + log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); + int res = sqlite3_exec(db, + createStatements[i], + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); + } +} + + +int main(void) +{ + sqlite3 *db; + + openDatabase(&db); + makeTables(db); + fillTables(db); + return 0; +} diff --git a/mps/code/eventsql.h b/mps/code/eventsql.h new file mode 100644 index 00000000000..a5d718fbffc --- /dev/null +++ b/mps/code/eventsql.h @@ -0,0 +1,51 @@ +/* -- Event Database Definitions + * + * $Id$ + * Copyright (C) 2012 Ravenbrook Limited. See end of file for license. + */ + +#ifndef eventsql_h +#define eventsql_h + +#endif /* eventsql_h */ + +/* C. COPYRIGHT AND LICENSE + * + * Copyright (C) 2001-2002 Ravenbrook Limited . + * All rights reserved. This is an open source license. Contact + * Ravenbrook for commercial licensing options. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Redistributions in any form must be accompanied by information on how + * to obtain complete source code for this software and any accompanying + * software that uses this software. The source code must either be + * included in the distribution or be available for no more than the cost + * of distribution plus a nominal fee, and must be freely redistributable + * under reasonable conditions. For an executable file, complete source + * code means the source code for all modules it contains. It does not + * include source code for modules or files that typically accompany the + * major components of the operating system on which the executable file + * runs. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ From 54a4a5568f783e01cafabe1aa110697aaf9b5c9a Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Sat, 13 Oct 2012 12:16:24 +0100 Subject: [PATCH 03/14] Event sql interface now creates a table for each type of event. Copied from Perforce Change: 179868 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 308 ++++++++++++++++++++++++-------------------- 1 file changed, 170 insertions(+), 138 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 753a8aafc08..23a50f8063f 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -17,183 +17,215 @@ unsigned int verbosity = 4; static void vlog(unsigned int level, const char *format, va_list args) { - if (level <= verbosity) { - fflush(stderr); /* sync */ - fprintf(stderr, "log %d: ", level); - vfprintf(stderr, format, args); - fprintf(stderr, "\n"); - } + if (level <= verbosity) { + fflush(stderr); /* sync */ + fprintf(stderr, "log %d: ", level); + vfprintf(stderr, format, args); + fprintf(stderr, "\n"); + } } static void log(unsigned int level, const char *format, ...) { - va_list args; - va_start(args, format); - vlog(level, format, args); - va_end(args); + va_list args; + va_start(args, format); + vlog(level, format, args); + va_end(args); } #if 0 /* UNUSED */ static void error(const char *format, ...) { - va_list args; - fprintf(stderr, "Fatal error: "); - va_start(args, format); - vlog(LOG_ALWAYS, format, args); - va_end(args); - exit(1); + va_list args; + fprintf(stderr, "Fatal error: "); + va_start(args, format); + vlog(LOG_ALWAYS, format, args); + va_end(args); + exit(1); } #endif /* UNUSED */ - + static void sqlite_error(int res, sqlite3 *db, const char *format, ...) { - log(LOG_ALWAYS, "Fatal SQL error %d", res); - va_list args; - va_start(args, format); - vlog(LOG_ALWAYS, format, args); - va_end(args); - log(LOG_ALWAYS, "SQLite message: %s\n", sqlite3_errmsg(db)); - exit(1); + log(LOG_ALWAYS, "Fatal SQL error %d", res); + va_list args; + va_start(args, format); + vlog(LOG_ALWAYS, format, args); + va_end(args); + log(LOG_ALWAYS, "SQLite message: %s\n", sqlite3_errmsg(db)); + exit(1); } static void openDatabase(sqlite3 **dbReturn) { - sqlite3 *db; - int res; - - const char *filename = getenv("MPS_EVENT_DATABASE"); - if(filename == NULL) - filename = "mpsevent.db"; - log(LOG_OFTEN, "Opening %s.", filename); - - res = sqlite3_open_v2(filename, - &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - NULL); /* use default sqlite_vfs object */ - - if (res != SQLITE_OK) - sqlite_error(res, db, "Opening %s failed", filename); - *dbReturn = db; - return; + sqlite3 *db; + int res; + + const char *filename = getenv("MPS_EVENT_DATABASE"); + if(filename == NULL) + filename = "mpsevent.db"; + log(LOG_OFTEN, "Opening %s.", filename); + + res = sqlite3_open_v2(filename, + &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + NULL); /* use default sqlite_vfs object */ + + if (res != SQLITE_OK) + sqlite_error(res, db, "Opening %s failed", filename); + *dbReturn = db; + return; } +/* Macro magic to make a CREATE TABLE statement for each event type. */ + +#define EVENT_PARAM_SQL_TYPE_A "INTEGER" +#define EVENT_PARAM_SQL_TYPE_P "INTEGER" +#define EVENT_PARAM_SQL_TYPE_U "INTEGER" +#define EVENT_PARAM_SQL_TYPE_W "INTEGER" +#define EVENT_PARAM_SQL_TYPE_D "REAL " +#define EVENT_PARAM_SQL_TYPE_S "TEXT " +#define EVENT_PARAM_SQL_TYPE_B "INTEGER" + +#define EVENT_PARAM_SQL_COLUMN(X, index, sort, ident) \ + "\"" #ident "\" " EVENT_PARAM_SQL_TYPE_##sort ", " + +#define EVENT_TABLE_CREATE(X, name, code, always, kind) \ + "CREATE TABLE IF NOT EXISTS EVENT_" #name " ( " \ + EVENT_##name##_PARAMS(EVENT_PARAM_SQL_COLUMN, X) \ + "time INTEGER ) ", + const char *createStatements[] = { - "CREATE TABLE IF NOT EXISTS event_kind (name TEXT," - " description TEXT," - " enum INTEGER PRIMARY KEY)", - "CREATE TABLE IF NOT EXISTS event_type (name TEXT," - " code INTEGER PRIMARY KEY," - " always INTEGER," - " kind INTEGER," - " FOREIGN KEY (kind) REFERENCES event_kind(enum));", + "CREATE TABLE IF NOT EXISTS event_kind (name TEXT," + " description TEXT," + " enum INTEGER PRIMARY KEY)", + + "CREATE TABLE IF NOT EXISTS event_type (name TEXT," + " code INTEGER PRIMARY KEY," + " always INTEGER," + " kind INTEGER," + " FOREIGN KEY (kind) REFERENCES event_kind(enum));", + +EVENT_LIST(EVENT_TABLE_CREATE, X) }; const char *populateStatements[] = { }; #define EVENT_KIND_DO_INSERT(X, name, description) \ - res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_kind bind of name \"" #name "\" failed."); \ - res = sqlite3_bind_text(statement, 2, description, -1, SQLITE_STATIC); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_kind bind of description \"" description "\" failed."); \ - res = sqlite3_bind_int(statement, 3, i); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_kind bind of enum %d failed.", i); \ - ++i; \ - res = sqlite3_step(statement); \ - if (res != SQLITE_DONE) \ - sqlite_error(res, db, "event_kind insert of name \"" #name "\" failed."); \ - if (sqlite3_changes(db) != 0)\ - log(LOG_SOMETIMES, "Insert of event_kind row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ - res = sqlite3_reset(statement); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Couldn't reset event_kind insert statement."); + res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of name \"" #name "\" failed."); \ + res = sqlite3_bind_text(statement, 2, description, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of description \"" description "\" failed."); \ + res = sqlite3_bind_int(statement, 3, i); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_kind bind of enum %d failed.", i); \ + ++i; \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "event_kind insert of name \"" #name "\" failed."); \ + if (sqlite3_changes(db) != 0) \ + log(LOG_SOMETIMES, "Insert of event_kind row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset event_kind insert statement."); -#define EVENT_TYPE_DO_INSERT(X, name, code, always, kind) \ - res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_type bind of name \"" #name "\" failed."); \ - res = sqlite3_bind_int(statement, 2, code); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_type bind of code %d failed.", code); \ - res = sqlite3_bind_int(statement, 3, always); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_type bind of always for name \"" #name "\" failed."); \ - res = sqlite3_bind_int(statement, 4, EventKind##kind); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "event_type bind of kind for name \"" #name "\" failed."); \ - res = sqlite3_step(statement); \ - if (res != SQLITE_DONE) \ - sqlite_error(res, db, "event_type insert of name \"" #name "\" failed."); \ - if (sqlite3_changes(db) != 0) \ - log(LOG_SOMETIMES, "Insert of event_type row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ - res = sqlite3_reset(statement); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Couldn't reset event_type insert statement."); +#define EVENT_TYPE_DO_INSERT(X, name, code, always, kind) \ + res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of name \"" #name "\" failed."); \ + res = sqlite3_bind_int(statement, 2, code); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of code %d failed.", code); \ + res = sqlite3_bind_int(statement, 3, always); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of always for name \"" #name "\" failed."); \ + res = sqlite3_bind_int(statement, 4, EventKind##kind); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of kind for name \"" #name "\" failed."); \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "event_type insert of name \"" #name "\" failed."); \ + if (sqlite3_changes(db) != 0) \ + log(LOG_SOMETIMES, "Insert of event_type row for \"" #name "\" affected %d rows.", sqlite3_changes(db)); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset event_type insert statement."); static void fillTables(sqlite3 *db) { - int i; - sqlite3_stmt *statement; - - int res = sqlite3_prepare_v2(db, - "INSERT OR IGNORE INTO event_kind (name, description, enum)" - "VALUES (?, ?, ?)", - -1, /* prepare whole string as statement */ - &statement, - NULL); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_kind preparation failed"); - - i = 0; - EventKindENUM(EVENT_KIND_DO_INSERT, X); - - res = sqlite3_finalize(statement); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_kind finalize failed"); - - res = sqlite3_prepare_v2(db, - "INSERT OR IGNORE INTO event_type (name, code, always, kind)" - "VALUES (?, ?, ?, ?)", - -1, /* prepare whole string as statement */ - &statement, - NULL); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_type preparation failed"); - - EVENT_LIST(EVENT_TYPE_DO_INSERT, X); - - res = sqlite3_finalize(statement); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_type finalize failed"); + int i; + sqlite3_stmt *statement; + int res; + + res = sqlite3_prepare_v2(db, + "INSERT OR IGNORE INTO event_kind (name, description, enum)" + "VALUES (?, ?, ?)", + -1, /* prepare whole string as statement */ + &statement, + NULL); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_kind preparation failed"); + + i = 0; + EventKindENUM(EVENT_KIND_DO_INSERT, X); + + res = sqlite3_finalize(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_kind finalize failed"); + + res = sqlite3_prepare_v2(db, + "INSERT OR IGNORE INTO event_type (name, code, always, kind)" + "VALUES (?, ?, ?, ?)", + -1, /* prepare whole string as statement */ + &statement, + NULL); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_type preparation failed"); + + EVENT_LIST(EVENT_TYPE_DO_INSERT, X); + + res = sqlite3_finalize(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_type finalize failed"); } static void makeTables(sqlite3 *db) { - int i; + int i; + int res; + + for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { + log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); + res = sqlite3_exec(db, + createStatements[i], + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); + } +} - for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { - log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); - int res = sqlite3_exec(db, - createStatements[i], - NULL, /* No callback */ - NULL, /* No callback closure */ - NULL); /* error messages handled by sqlite_error */ - if (res != SQLITE_OK) - sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); - } +void closeDatabase(sqlite3 *db) +{ + int res = sqlite3_close(db); + if (res != SQLITE_OK) + sqlite_error(res, db, "Closing database failed"); + log(LOG_ALWAYS, "Closed database."); } int main(void) { - sqlite3 *db; - - openDatabase(&db); - makeTables(db); - fillTables(db); - return 0; + sqlite3 *db; + + openDatabase(&db); + makeTables(db); + fillTables(db); + closeDatabase(db); + return 0; } From fc5acb36a7ea08dd12a173094d1679e5335a5cec Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Mon, 15 Oct 2012 00:32:37 +0100 Subject: [PATCH 04/14] Functioning event/sql interface. Copied from Perforce Change: 179880 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 405 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 346 insertions(+), 59 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 23a50f8063f..f1905231e3a 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -5,9 +5,26 @@ #include #include +#include #include -unsigned int verbosity = 4; +#define DATABASE_NAME_ENVAR "MPS_EVENT_DATABASE" +#define DEFAULT_DATABASE_NAME "mpsevent.db" + +#define TELEMETRY_FILENAME_ENVAR "MPS_TELEMETRY_FILENAME" +#define DEFAULT_TELEMETRY_FILENAME "mpsio.log" + +/* we output rows of dots. One dot per SMALL_TICK events, + * BIG_TICK dots per row. */ + +#define SMALL_TICK 1000 +#define BIG_TICK 50 + +/* Utility code for logging to stderr with multiple log levels, + * and for reporting errors. + */ + +unsigned int verbosity = 4; /* TODO command-line -v switches */ #define LOG_ALWAYS 0 #define LOG_OFTEN 1 @@ -33,8 +50,6 @@ static void log(unsigned int level, const char *format, ...) va_end(args); } -#if 0 /* UNUSED */ - static void error(const char *format, ...) { va_list args; @@ -44,7 +59,6 @@ static void error(const char *format, ...) va_end(args); exit(1); } -#endif /* UNUSED */ static void sqlite_error(int res, sqlite3 *db, const char *format, ...) { @@ -57,15 +71,18 @@ static void sqlite_error(int res, sqlite3 *db, const char *format, ...) exit(1); } -static void openDatabase(sqlite3 **dbReturn) +/* openDatabase(p) opens the database file and returns a SQLite 3 + * database connection object. */ + +static sqlite3 *openDatabase(void) { sqlite3 *db; int res; - const char *filename = getenv("MPS_EVENT_DATABASE"); + const char *filename = getenv(DATABASE_NAME_ENVAR); if(filename == NULL) - filename = "mpsevent.db"; - log(LOG_OFTEN, "Opening %s.", filename); + filename = DEFAULT_DATABASE_NAME; + log(LOG_ALWAYS, "Opening %s.", filename); res = sqlite3_open_v2(filename, &db, @@ -74,8 +91,99 @@ static void openDatabase(sqlite3 **dbReturn) if (res != SQLITE_OK) sqlite_error(res, db, "Opening %s failed", filename); - *dbReturn = db; - return; + return db; +} + +/* closeDatabase(db) closes the database opened by openDatabase(). */ + +static void closeDatabase(sqlite3 *db) +{ + int res = sqlite3_close(db); + if (res != SQLITE_OK) + sqlite_error(res, db, "Closing database failed"); + log(LOG_ALWAYS, "Closed database."); +} + +/* We need to be able to test for the existence of a table. The + * SQLite3 API seems to have no way to explore metadata like this, + * unless it is compiled in a particular way (in which case the + * function sqlite3_table_column_metadata could be used). Without + * that assistance, we can use a simple SQL trick (which could also + * tell us the number of rows in the table if we cared): */ + +static int tableExists(sqlite3* db, const char *tableName) +{ + const char *format = "SELECT SUM(1) FROM %s"; + char *sql; + int res; + + sql = malloc(strlen(format) + strlen(tableName)); + if (!sql) + error("Out of memory."); + sprintf(sql, format, tableName); + res = sqlite3_exec(db, + sql, + NULL, /* put in a callback here if we really want to know the number of rows */ + NULL, /* callback closure */ + NULL); /* error messages handled by sqlite_error */ + switch(res) { + case SQLITE_OK: + return 1; /* table exists */ + break; + case SQLITE_ERROR: + return 0; /* table does not exist; we can + probably do a better test for this case. */ + break; + default: + sqlite_error(res, db, "Table test failed: %s", tableName); + } + /* UNREACHED */ + return 0; +} + +/* Unit test for tableExists() */ + +static const char *tableTests[] = { + "event_kind", + "spong", + "EVENT_SegSplit", +}; + +static void testTableExists(sqlite3 *db) +{ + int i; + for (i=0; i < (sizeof(tableTests)/sizeof(tableTests[0])); ++i) { + if (tableExists(db, tableTests[i])) + printf("Table exists: %s\n", tableTests[i]); + else + printf("Table does not exist: %s\n", tableTests[i]); + } +} + +/* Utility functions for SQLite statements. */ + +static sqlite3_stmt *prepareStatement(sqlite3 *db, + const char *sql) +{ + int res; + sqlite3_stmt *statement; + log(LOG_SELDOM, "Preparing statement %s", sql); + res = sqlite3_prepare_v2(db, sql, + -1, /* prepare whole string as statement */ + &statement, + NULL); + if (res != SQLITE_OK) + sqlite_error(res, db, "statementpreparation failed: %s", sql); + return statement; +} + +static void finalizeStatement(sqlite3 *db, + sqlite3_stmt *statement) +{ + int res; + res = sqlite3_finalize(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_type finalize failed"); } /* Macro magic to make a CREATE TABLE statement for each event type. */ @@ -94,7 +202,10 @@ static void openDatabase(sqlite3 **dbReturn) #define EVENT_TABLE_CREATE(X, name, code, always, kind) \ "CREATE TABLE IF NOT EXISTS EVENT_" #name " ( " \ EVENT_##name##_PARAMS(EVENT_PARAM_SQL_COLUMN, X) \ - "time INTEGER ) ", + "time INTEGER, " \ + "log_serial INTEGER)", + +/* An array of table-creation statement strings. */ const char *createStatements[] = { "CREATE TABLE IF NOT EXISTS event_kind (name TEXT," @@ -107,11 +218,37 @@ const char *createStatements[] = { " kind INTEGER," " FOREIGN KEY (kind) REFERENCES event_kind(enum));", + "CREATE TABLE IF NOT EXISTS event_log (name TEXT," + " file_id INTEGER," + " size INTEGER," + " time_sec INTEGER," + " time_nsec INTEGER," + " completed INTEGER," + " serial INTEGER PRIMARY KEY AUTOINCREMENT)", + EVENT_LIST(EVENT_TABLE_CREATE, X) }; -const char *populateStatements[] = { -}; +/* makeTables makes all the tables. */ + +static void makeTables(sqlite3 *db) +{ + int i; + int res; + + for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { + log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); + res = sqlite3_exec(db, + createStatements[i], + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); + } +} + +/* Populate the metadata "glue" tables event_kind and event_type. */ #define EVENT_KIND_DO_INSERT(X, name, description) \ res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ @@ -155,77 +292,227 @@ const char *populateStatements[] = { if (res != SQLITE_OK) \ sqlite_error(res, db, "Couldn't reset event_type insert statement."); -static void fillTables(sqlite3 *db) +static void fillGlueTables(sqlite3 *db) { int i; + Res res; sqlite3_stmt *statement; - int res; - - res = sqlite3_prepare_v2(db, - "INSERT OR IGNORE INTO event_kind (name, description, enum)" - "VALUES (?, ?, ?)", - -1, /* prepare whole string as statement */ - &statement, - NULL); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_kind preparation failed"); + + statement = prepareStatement(db, + "INSERT OR IGNORE INTO event_kind (name, description, enum)" + "VALUES (?, ?, ?)"); i = 0; EventKindENUM(EVENT_KIND_DO_INSERT, X); - res = sqlite3_finalize(statement); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_kind finalize failed"); - - res = sqlite3_prepare_v2(db, - "INSERT OR IGNORE INTO event_type (name, code, always, kind)" - "VALUES (?, ?, ?, ?)", - -1, /* prepare whole string as statement */ - &statement, - NULL); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_type preparation failed"); + finalizeStatement(db, statement); + statement = prepareStatement(db, + "INSERT OR IGNORE INTO event_type (name, code, always, kind)" + "VALUES (?, ?, ?, ?)"); EVENT_LIST(EVENT_TYPE_DO_INSERT, X); - res = sqlite3_finalize(statement); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_type finalize failed"); + finalizeStatement(db, statement); } -static void makeTables(sqlite3 *db) +/* Populate the actual event tables. */ + +#define EVENT_TYPE_DECLARE_STATEMENT(X, name, code, always, kind) \ + sqlite3_stmt *stmt_##name; + +#define EVENT_PARAM_PREPARE_IDENT(X, index, sort, ident) "\"" #ident "\", " + +#define EVENT_PARAM_PREPARE_PLACE(X, index, sort, ident) "?, " + +#define EVENT_TYPE_PREPARE_STATEMENT(X, name, code, always, kind) \ + stmt_##name = \ + prepareStatement(db, \ + "INSERT INTO EVENT_" #name " (" \ + EVENT_##name##_PARAMS(EVENT_PARAM_PREPARE_IDENT, X) \ + "log_serial, time) VALUES (" \ + EVENT_##name##_PARAMS(EVENT_PARAM_PREPARE_PLACE,X) \ + "?, ?)"); + +#define EVENT_TYPE_FINALIZE_STATEMENT(X, name, code, always, kind) \ + finalizeStatement(db, stmt_##name); + +#define EVENT_PARAM_BIND_INTEGER(name, index, sort, ident) \ + res = sqlite3_bind_int64(statement, index+1, (unsigned long) event->name.f##index); + +#define EVENT_PARAM_BIND_REAL(name, index, sort, ident) \ + res = sqlite3_bind_double(statement, index+1, event->name.f##index); + +#define EVENT_PARAM_BIND_TEXT(name, index, sort, ident) \ + res = sqlite3_bind_text(statement, index+1, event->name.f##index, -1, SQLITE_STATIC); + +#define EVENT_PARAM_BIND_A EVENT_PARAM_BIND_INTEGER +#define EVENT_PARAM_BIND_P EVENT_PARAM_BIND_INTEGER +#define EVENT_PARAM_BIND_U EVENT_PARAM_BIND_INTEGER +#define EVENT_PARAM_BIND_W EVENT_PARAM_BIND_INTEGER +#define EVENT_PARAM_BIND_D EVENT_PARAM_BIND_REAL +#define EVENT_PARAM_BIND_S EVENT_PARAM_BIND_TEXT +#define EVENT_PARAM_BIND_B EVENT_PARAM_BIND_INTEGER + +#define EVENT_PARAM_BIND(name, index, sort, ident) \ + EVENT_PARAM_BIND_##sort (name, index, sort, ident) \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Event " #name " bind of ident " #ident "failed."); \ + last_index = index+1; + + +#define EVENT_TYPE_WRITE_SQL(X, name, code, always, kind) \ + case code: \ + { \ + sqlite3_stmt *statement = stmt_##name; \ + int last_index = 0; \ + int res; \ + /* bind all the parameters of this particular event with macro magic. */ \ + EVENT_##name##_PARAMS(EVENT_PARAM_BIND, name) \ + /* bind the fields we store for every event */ \ + res = sqlite3_bind_int64(statement, last_index+1, log_serial); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Event " #name " bind of log_serial failed."); \ + res = sqlite3_bind_int64(statement, last_index+2, clock); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Event " #name " bind of clock failed."); \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "insert of event \"" #name "\" failed."); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset insert statement for \"" #name "\"."); \ + } \ + break; + +/* readLog -- read and parse log + */ + +static void readLog(EventProc proc, + sqlite3 *db) { - int i; - int res; - - for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { - log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); - res = sqlite3_exec(db, - createStatements[i], - NULL, /* No callback */ - NULL, /* No callback closure */ - NULL); /* error messages handled by sqlite_error */ - if (res != SQLITE_OK) - sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); + int log_serial = 0; /* TODO get this from event_log table */ + size_t eventCount = 0; + /* declare statements for every event type */ + EVENT_LIST(EVENT_TYPE_DECLARE_STATEMENT, X); + + /* prepare statements for every event type */ + EVENT_LIST(EVENT_TYPE_PREPARE_STATEMENT, X); + + while (TRUE) { /* loop for each event */ + Event event; + EventCode code; + EventClock clock; + Res res; + + /* Read and parse event. */ + res = EventRead(&event, proc); + if (res == ResFAIL) break; /* eof */ + if (res != ResOK) error("Truncated log"); + clock = event->any.clock; + code = event->any.code; + + /* Write event to SQLite. */ + switch (code) { + EVENT_LIST(EVENT_TYPE_WRITE_SQL, X); + } + EventDestroy(proc, event); + eventCount++; + if ((eventCount % SMALL_TICK) == 0) { + fprintf(stdout, "."); + fflush(stdout); + if (((eventCount / SMALL_TICK) % BIG_TICK) == 0) { + log(LOG_OFTEN, "%lu events.", (unsigned long)eventCount); + } + } } + log(LOG_OFTEN, "Wrote %lu events to SQL.", (unsigned long)eventCount); + /* finalize all the statements */ + EVENT_LIST(EVENT_TYPE_FINALIZE_STATEMENT, X); } -void closeDatabase(sqlite3 *db) + + +static Res logReader(void *file, void *p, size_t len) { - int res = sqlite3_close(db); - if (res != SQLITE_OK) - sqlite_error(res, db, "Closing database failed"); - log(LOG_ALWAYS, "Closed database."); + size_t n; + + n = fread(p, 1, len, (FILE *)file); + return (n < len) ? (feof((FILE *)file) ? ResFAIL : ResIO) : ResOK; +} + +#if 0 +/* TODO. Find out whether the database already contains this log file. + * Should probably also fopen it, and use fileno() and fstat() */ + +static int logFileSerial(sqlite3 *db, const char *filename) +{ + struct stat st; + sqlite3_stmt *statement; + + stat(filename, &st); + statement = prepareStatement(db, + "SELECT serial, completed FROM event_log WHERE" + "file_id = ? AND size = ? AND time_sec = ? AND time_nsec = ?"); + /* TODO: Get stat results, look up in event_log, return accordingly */ +} +#endif + +static FILE *openLog(void) +{ + const char *filename; + FILE *input; + + filename = getenv(TELEMETRY_FILENAME_ENVAR); + if(filename == NULL) + filename = DEFAULT_TELEMETRY_FILENAME; + input = fopen(filename, "rb"); + + if (input == NULL) + error("unable to open %s", filename); + + return input; +} + +static void writeEventsToSQL(sqlite3 *db) +{ + Res res; + EventProc proc; + FILE *input; + + input = openLog(); + res = EventProcCreate(&proc, logReader, (void *)input); + if (res != ResOK) + error("Can't init EventProc module: error %d.", res); + + readLog(proc, db); + + EventProcDestroy(proc); + (void)fclose(input); } int main(void) { sqlite3 *db; + + /* TODO: command line args + * -v verbosity; + * -f log filename; + * -d database filename; + * -r rebuild glue tables (by dropping them); + * -t unit tests? + */ - openDatabase(&db); + db = openDatabase(); makeTables(db); - fillTables(db); + fillGlueTables(db); + + writeEventsToSQL(db); + + if (0) { + testTableExists(db); + } + closeDatabase(db); return 0; } From 82a32fa22a447f6e019ecfa5fb2274094a7e4275 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Mon, 15 Oct 2012 00:35:40 +0100 Subject: [PATCH 05/14] Eventcnv was printing event clock values incorrectly: the upper 32 bits and then all the bits. have to mask the top bits when printing the bottom ones. Copied from Perforce Change: 179881 ServerID: perforce.ravenbrook.com --- mps/code/clock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mps/code/clock.h b/mps/code/clock.h index 881d74e2715..920a69c9bee 100644 --- a/mps/code/clock.h +++ b/mps/code/clock.h @@ -120,7 +120,7 @@ __extension__ typedef unsigned long long EventClock; #define EVENT_CLOCK_PRINT(stream, clock) \ fprintf(stream, "%08lX%08lX", \ (unsigned long)((clock) >> 32), \ - (unsigned long)(clock)) + (unsigned long)((clock) & 0xffffffff)) #define EVENT_CLOCK_WRITE(stream, clock) \ WriteF(stream, "$W$W", (WriteFW)((clock) >> 32), (WriteFW)clock, NULL) From a47d640fe86ad3974f8406a5b4388266e890f2c1 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Tue, 16 Oct 2012 22:19:54 +0100 Subject: [PATCH 06/14] Eventsql goes hundreds of times faster (because all event inserts are now in a single transaction). Copied from Perforce Change: 179904 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 173 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 143 insertions(+), 30 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index f1905231e3a..446627ed3b5 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -7,6 +7,7 @@ #include #include #include +#include #define DATABASE_NAME_ENVAR "MPS_EVENT_DATABASE" #define DEFAULT_DATABASE_NAME "mpsevent.db" @@ -109,7 +110,12 @@ static void closeDatabase(sqlite3 *db) * unless it is compiled in a particular way (in which case the * function sqlite3_table_column_metadata could be used). Without * that assistance, we can use a simple SQL trick (which could also - * tell us the number of rows in the table if we cared): */ + * tell us the number of rows in the table if we cared). + * + * TODO: Update this to use the sqlite_master table: + * SELECT FROM sqlite_master WHERE type='table' AND name=? + * and fix the above comment. + */ static int tableExists(sqlite3* db, const char *tableName) { @@ -121,16 +127,22 @@ static int tableExists(sqlite3* db, const char *tableName) if (!sql) error("Out of memory."); sprintf(sql, format, tableName); + log(LOG_RARELY, "Testing for existence of table '%s' with SQL: %s", tableName, sql); res = sqlite3_exec(db, sql, NULL, /* put in a callback here if we really want to know the number of rows */ NULL, /* callback closure */ NULL); /* error messages handled by sqlite_error */ + free(sql); + switch(res) { case SQLITE_OK: + log(LOG_RARELY, "Table '%s' exists.", tableName); + return 1; /* table exists */ break; case SQLITE_ERROR: + log(LOG_RARELY, "Table '%s' does not exist.", tableName); return 0; /* table does not exist; we can probably do a better test for this case. */ break; @@ -186,6 +198,108 @@ static void finalizeStatement(sqlite3 *db, sqlite_error(res, db, "event_type finalize failed"); } +/* every time we put events from a log file into a database file, we + * add the log file to the event_log table, and get a serial number + * from SQL which is then attached to all event rows from that log. + * We use this to record overall SQL activity, to deter mistaken + * attempts to add the same log file twice, and to allow events from + * several different log files to share the same SQL file. */ + +static unsigned long logSerial = 0; + +static void registerLogFile(sqlite3 *db, + const char *filename, + int force) +{ + struct stat st; + sqlite3_stmt *statement; + int res; + const unsigned char *name; + unsigned long completed; + + res = stat(filename, &st); + if (res != 0) + error("Couldn't stat() %s", filename); + + statement = prepareStatement(db, + "SELECT name, serial, completed FROM event_log" + " WHERE file_id = ? AND size = ? AND modtime = ?"); + res = sqlite3_bind_int64(statement, 1, st.st_ino); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log bind of file_id failed."); + res = sqlite3_bind_int64(statement, 2, st.st_size); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log bind of size failed."); + res = sqlite3_bind_int64(statement, 3, st.st_mtime); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log bind of modtime failed."); + + res = sqlite3_step(statement); + switch(res) { + case SQLITE_DONE: + log(LOG_SOMETIMES, "No log file matching '%s' found in database.", filename); + break; + case SQLITE_ROW: + name = sqlite3_column_text(statement, 1); + logSerial = sqlite3_column_int(statement, 2); + completed = sqlite3_column_int(statement, 3); + log(LOG_ALWAYS, "Log file matching '%s' already in event_log, named \"%s\" (serial %lu, completed %lu).", + filename, name, logSerial, completed); + if (!force) { + log(LOG_ALWAYS, "Exiting. Specify -f to force events into SQL anyway."); + exit(0); + } + log(LOG_ALWAYS, "Continuing anyway because -f specified."); + break; + default: + sqlite_error(res, db, "select from event_log failed."); + } + finalizeStatement(db, statement); + statement = prepareStatement(db, + "INSERT into event_log (name, file_id, size, modtime, completed)" + " VALUES (?, ?, ?, ?, 0)"); + res = sqlite3_bind_text(statement, 1, filename, -1, SQLITE_STATIC); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log insert bind of name failed."); + res = sqlite3_bind_int64(statement, 2, st.st_ino); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log insert bind of file_id failed."); + res = sqlite3_bind_int64(statement, 3, st.st_size); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log insert bind of size failed."); + res = sqlite3_bind_int64(statement, 4, st.st_mtime); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log insert bind of modtime failed."); + res = sqlite3_step(statement); + if (res != SQLITE_DONE) + sqlite_error(res, db, "insert into event_log failed."); + logSerial = sqlite3_last_insert_rowid(db); + log(LOG_SOMETIMES, "Log file added to event_log with serial %lu", + filename, logSerial); + finalizeStatement(db, statement); +} + +static void logFileCompleted(sqlite3 *db, + unsigned long completed) +{ + sqlite3_stmt *statement; + int res; + + statement = prepareStatement(db, + "UPDATE event_log SET completed=? WHERE serial=?"); + res = sqlite3_bind_int64(statement, 2, logSerial); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log update bind of serial failed."); + res = sqlite3_bind_int64(statement, 1, completed); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log update bind of completed failed."); + res = sqlite3_step(statement); + if (res != SQLITE_DONE) + sqlite_error(res, db, "insert into event_log failed."); + log(LOG_OFTEN, "Marked in event_log: %lu events", completed); + finalizeStatement(db, statement); +} + /* Macro magic to make a CREATE TABLE statement for each event type. */ #define EVENT_PARAM_SQL_TYPE_A "INTEGER" @@ -221,8 +335,7 @@ const char *createStatements[] = { "CREATE TABLE IF NOT EXISTS event_log (name TEXT," " file_id INTEGER," " size INTEGER," - " time_sec INTEGER," - " time_nsec INTEGER," + " modtime INTEGER," " completed INTEGER," " serial INTEGER PRIMARY KEY AUTOINCREMENT)", @@ -369,10 +482,10 @@ static void fillGlueTables(sqlite3 *db) /* bind all the parameters of this particular event with macro magic. */ \ EVENT_##name##_PARAMS(EVENT_PARAM_BIND, name) \ /* bind the fields we store for every event */ \ - res = sqlite3_bind_int64(statement, last_index+1, log_serial); \ + res = sqlite3_bind_int64(statement, last_index+1, logSerial); \ if (res != SQLITE_OK) \ sqlite_error(res, db, "Event " #name " bind of log_serial failed."); \ - res = sqlite3_bind_int64(statement, last_index+2, clock); \ + res = sqlite3_bind_int64(statement, last_index+2, event->any.clock); \ if (res != SQLITE_OK) \ sqlite_error(res, db, "Event " #name " bind of clock failed."); \ res = sqlite3_step(statement); \ @@ -390,25 +503,31 @@ static void fillGlueTables(sqlite3 *db) static void readLog(EventProc proc, sqlite3 *db) { - int log_serial = 0; /* TODO get this from event_log table */ size_t eventCount = 0; + int res; /* declare statements for every event type */ EVENT_LIST(EVENT_TYPE_DECLARE_STATEMENT, X); /* prepare statements for every event type */ EVENT_LIST(EVENT_TYPE_PREPARE_STATEMENT, X); + res = sqlite3_exec(db, + "BEGIN", + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "BEGIN failed"); + while (TRUE) { /* loop for each event */ Event event; EventCode code; - EventClock clock; Res res; /* Read and parse event. */ res = EventRead(&event, proc); if (res == ResFAIL) break; /* eof */ if (res != ResOK) error("Truncated log"); - clock = event->any.clock; code = event->any.code; /* Write event to SQLite. */ @@ -425,7 +544,16 @@ static void readLog(EventProc proc, } } } + res = sqlite3_exec(db, + "COMMIT", + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "COMMIT failed"); + log(LOG_OFTEN, "Wrote %lu events to SQL.", (unsigned long)eventCount); + logFileCompleted(db, eventCount); /* finalize all the statements */ EVENT_LIST(EVENT_TYPE_FINALIZE_STATEMENT, X); } @@ -440,24 +568,7 @@ static Res logReader(void *file, void *p, size_t len) return (n < len) ? (feof((FILE *)file) ? ResFAIL : ResIO) : ResOK; } -#if 0 -/* TODO. Find out whether the database already contains this log file. - * Should probably also fopen it, and use fileno() and fstat() */ - -static int logFileSerial(sqlite3 *db, const char *filename) -{ - struct stat st; - sqlite3_stmt *statement; - - stat(filename, &st); - statement = prepareStatement(db, - "SELECT serial, completed FROM event_log WHERE" - "file_id = ? AND size = ? AND time_sec = ? AND time_nsec = ?"); - /* TODO: Get stat results, look up in event_log, return accordingly */ -} -#endif - -static FILE *openLog(void) +static FILE *openLog(sqlite3 *db) { const char *filename; FILE *input; @@ -465,6 +576,8 @@ static FILE *openLog(void) filename = getenv(TELEMETRY_FILENAME_ENVAR); if(filename == NULL) filename = DEFAULT_TELEMETRY_FILENAME; + + registerLogFile(db, filename, 0); input = fopen(filename, "rb"); if (input == NULL) @@ -479,7 +592,7 @@ static void writeEventsToSQL(sqlite3 *db) EventProc proc; FILE *input; - input = openLog(); + input = openLog(db); res = EventProcCreate(&proc, logReader, (void *)input); if (res != ResOK) error("Can't init EventProc module: error %d.", res); @@ -506,11 +619,11 @@ int main(void) db = openDatabase(); makeTables(db); fillGlueTables(db); - writeEventsToSQL(db); - if (0) { - testTableExists(db); + if (0) { /* avoid unused-function warnings. + Put test code or calls to experimental functions here. */ + testTableExists(db); } closeDatabase(db); From fa8cda3e9cb9243c6a3f6eebdaa2ba42827f6ebd Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Wed, 17 Oct 2012 03:03:54 +0100 Subject: [PATCH 07/14] Add command-line controls to eventsql. Copied from Perforce Change: 179908 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 219 ++++++++++++++++++++++++++++++++------------ 1 file changed, 159 insertions(+), 60 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 446627ed3b5..8e0b0761998 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -25,7 +25,7 @@ * and for reporting errors. */ -unsigned int verbosity = 4; /* TODO command-line -v switches */ +unsigned int verbosity = 0; #define LOG_ALWAYS 0 #define LOG_OFTEN 1 @@ -72,6 +72,77 @@ static void sqlite_error(int res, sqlite3 *db, const char *format, ...) exit(1); } +static char *prog; /* program name */ +static int rebuild = FALSE; +static int runTests = FALSE; +static int force = FALSE; +static char *databaseName = NULL; +static char *logFileName = NULL; + +static void usage(void) +{ + fprintf(stderr, + "Usage: %s [-rtf] [-l ] [-d ] [-v -v -v ...]\n", + prog); +} + +static void usageError(void) +{ + usage(); + error("Bad usage"); +} + +/* parseArgs -- parse command line arguments */ + +static void parseArgs(int argc, char *argv[]) +{ + int i = 1; + + if (argc >= 1) + prog = argv[0]; + else + prog = "unknown"; + + while (i < argc) { /* consider argument i */ + if (argv[i][0] == '-') { /* it's an option argument */ + switch (argv[i][1]) { + case 'v': /* verbosity */ + ++ verbosity; + break; + case 'r': /* rebuild */ + rebuild = TRUE; + break; + case 'f': /* force */ + force = TRUE; + break; + case 't': /* run tests */ + runTests = TRUE; + break; + case 'l': /* log file name */ + ++ i; + if (i == argc) + usageError(); + else + logFileName = argv[i]; + break; + case 'd': /* database file name */ + ++ i; + if (i == argc) + usageError(); + else + databaseName = argv[i]; + break; + case '?': case 'h': /* help */ + usage(); + break; + default: + usageError(); + } + } /* if option */ + ++ i; + } +} + /* openDatabase(p) opens the database file and returns a SQLite 3 * database connection object. */ @@ -79,19 +150,22 @@ static sqlite3 *openDatabase(void) { sqlite3 *db; int res; - - const char *filename = getenv(DATABASE_NAME_ENVAR); - if(filename == NULL) - filename = DEFAULT_DATABASE_NAME; - log(LOG_ALWAYS, "Opening %s.", filename); - - res = sqlite3_open_v2(filename, + + if (!databaseName) { + databaseName = getenv(DATABASE_NAME_ENVAR); + if(!databaseName) + databaseName = DEFAULT_DATABASE_NAME; + } + res = sqlite3_open_v2(databaseName, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL); /* use default sqlite_vfs object */ if (res != SQLITE_OK) - sqlite_error(res, db, "Opening %s failed", filename); + sqlite_error(res, db, "Opening %s failed", databaseName); + + log(LOG_ALWAYS, "Writing to %s.",databaseName); + return db; } @@ -102,7 +176,7 @@ static void closeDatabase(sqlite3 *db) int res = sqlite3_close(db); if (res != SQLITE_OK) sqlite_error(res, db, "Closing database failed"); - log(LOG_ALWAYS, "Closed database."); + log(LOG_ALWAYS, "Closed %s.", databaseName); } /* We need to be able to test for the existence of a table. The @@ -198,7 +272,22 @@ static void finalizeStatement(sqlite3 *db, sqlite_error(res, db, "event_type finalize failed"); } -/* every time we put events from a log file into a database file, we +static void runStatement(sqlite3 *db, + const char *sql, + const char *description) +{ + int res; + res = sqlite3_exec(db, + sql, + NULL, /* No callback */ + NULL, /* No callback closure */ + NULL); /* error messages handled by sqlite_error */ + if (res != SQLITE_OK) + sqlite_error(res, db, "%s failed - statement %s", description, sql); +} + + +/* Every time we put events from a log file into a database file, we * add the log file to the event_log table, and get a serial number * from SQL which is then attached to all event rows from that log. * We use this to record overall SQL activity, to deter mistaken @@ -208,8 +297,7 @@ static void finalizeStatement(sqlite3 *db, static unsigned long logSerial = 0; static void registerLogFile(sqlite3 *db, - const char *filename, - int force) + const char *filename) { struct stat st; sqlite3_stmt *statement; @@ -240,9 +328,9 @@ static void registerLogFile(sqlite3 *db, log(LOG_SOMETIMES, "No log file matching '%s' found in database.", filename); break; case SQLITE_ROW: - name = sqlite3_column_text(statement, 1); - logSerial = sqlite3_column_int(statement, 2); - completed = sqlite3_column_int(statement, 3); + name = sqlite3_column_text(statement, 0); + logSerial = sqlite3_column_int(statement, 1); + completed = sqlite3_column_int(statement, 2); log(LOG_ALWAYS, "Log file matching '%s' already in event_log, named \"%s\" (serial %lu, completed %lu).", filename, name, logSerial, completed); if (!force) { @@ -347,17 +435,35 @@ EVENT_LIST(EVENT_TABLE_CREATE, X) static void makeTables(sqlite3 *db) { int i; - int res; for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); + runStatement(db, createStatements[i], "Table creation"); + } +} + +const char *glueTables[] = { + "event_kind", + "event_type", +}; + +static void dropGlueTables(sqlite3 *db) +{ + int i; + int res; + char sql[1024]; + + log(LOG_ALWAYS, "Dropping glue tables so they are rebuilt."); + + for (i=0; i < (sizeof(glueTables)/sizeof(glueTables[0])); ++i) { + log(LOG_SOMETIMES, "Dropping table %s", glueTables[i]); + sprintf(sql, "DROP TABLE %s", glueTables[i]); res = sqlite3_exec(db, - createStatements[i], + sql, NULL, /* No callback */ NULL, /* No callback closure */ NULL); /* error messages handled by sqlite_error */ - if (res != SQLITE_OK) - sqlite_error(res, db, "Table creation failed: %s", createStatements[i]); + /* Don't check for errors. */ } } @@ -504,20 +610,14 @@ static void readLog(EventProc proc, sqlite3 *db) { size_t eventCount = 0; - int res; + /* declare statements for every event type */ EVENT_LIST(EVENT_TYPE_DECLARE_STATEMENT, X); /* prepare statements for every event type */ EVENT_LIST(EVENT_TYPE_PREPARE_STATEMENT, X); - res = sqlite3_exec(db, - "BEGIN", - NULL, /* No callback */ - NULL, /* No callback closure */ - NULL); /* error messages handled by sqlite_error */ - if (res != SQLITE_OK) - sqlite_error(res, db, "BEGIN failed"); + runStatement(db, "BEGIN", "Transaction start"); while (TRUE) { /* loop for each event */ Event event; @@ -536,30 +636,29 @@ static void readLog(EventProc proc, } EventDestroy(proc, event); eventCount++; - if ((eventCount % SMALL_TICK) == 0) { - fprintf(stdout, "."); - fflush(stdout); - if (((eventCount / SMALL_TICK) % BIG_TICK) == 0) { - log(LOG_OFTEN, "%lu events.", (unsigned long)eventCount); + if (verbosity > LOG_ALWAYS) { + if ((eventCount % SMALL_TICK) == 0) { + printf("."); + fflush(stdout); + if (((eventCount / SMALL_TICK) % BIG_TICK) == 0) { + log(LOG_OFTEN, "%lu events.", (unsigned long)eventCount); + } } } } - res = sqlite3_exec(db, - "COMMIT", - NULL, /* No callback */ - NULL, /* No callback closure */ - NULL); /* error messages handled by sqlite_error */ - if (res != SQLITE_OK) - sqlite_error(res, db, "COMMIT failed"); + if (verbosity > LOG_ALWAYS) { + printf("\n"); + fflush(stdout); + } + runStatement(db, "COMMIT", "Transaction finish"); - log(LOG_OFTEN, "Wrote %lu events to SQL.", (unsigned long)eventCount); + log(LOG_ALWAYS, "Wrote %lu events to SQL.", (unsigned long)eventCount); logFileCompleted(db, eventCount); + /* finalize all the statements */ EVENT_LIST(EVENT_TYPE_FINALIZE_STATEMENT, X); } - - static Res logReader(void *file, void *p, size_t len) { size_t n; @@ -570,18 +669,21 @@ static Res logReader(void *file, void *p, size_t len) static FILE *openLog(sqlite3 *db) { - const char *filename; FILE *input; - filename = getenv(TELEMETRY_FILENAME_ENVAR); - if(filename == NULL) - filename = DEFAULT_TELEMETRY_FILENAME; + if (!logFileName) { + logFileName = getenv(TELEMETRY_FILENAME_ENVAR); + if(logFileName == NULL) + logFileName = DEFAULT_TELEMETRY_FILENAME; + } - registerLogFile(db, filename, 0); - input = fopen(filename, "rb"); + registerLogFile(db, logFileName); + input = fopen(logFileName, "rb"); if (input == NULL) - error("unable to open %s", filename); + error("unable to open %s", logFileName); + + log(LOG_ALWAYS, "Reading %s.", logFileName); return input; } @@ -604,25 +706,22 @@ static void writeEventsToSQL(sqlite3 *db) } -int main(void) +int main(int argc, char *argv[]) { sqlite3 *db; - /* TODO: command line args - * -v verbosity; - * -f log filename; - * -d database filename; - * -r rebuild glue tables (by dropping them); - * -t unit tests? - */ + parseArgs(argc, argv); db = openDatabase(); + if (rebuild) { + dropGlueTables(db); + } makeTables(db); fillGlueTables(db); writeEventsToSQL(db); - if (0) { /* avoid unused-function warnings. - Put test code or calls to experimental functions here. */ + if (runTests) { + /* TODO: more unit tests in here */ testTableExists(db); } From 2dd20913e4d9bfc948282c11cfa988907ad9a85e Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Wed, 17 Oct 2012 14:49:00 +0100 Subject: [PATCH 08/14] Added long explanatory comment, and license, to eventsql.c. removed eventsql.h (what was i thinking?). Copied from Perforce Change: 179915 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 112 +++++++++++++++++++++++++++++++++++++++++++- mps/code/eventsql.h | 51 -------------------- 2 files changed, 110 insertions(+), 53 deletions(-) delete mode 100644 mps/code/eventsql.h diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 8e0b0761998..d4401192942 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -1,5 +1,72 @@ -#include "config.h" +/* eventsql.c: event log to SQLite importer + * Copyright (c) 2012 Ravenbrook Limited. See end of file for license. + * + * This is a command-line tool that imports events from a binary + * format telemetry output file from the MPS into a SQLite database + * file. + * + * The default MPS library will write a telemetry stream to a file called + * "mpsio.log" when the environment variable MPS_TELEMETRY_CONTROL is set + * to an integer whose bits select event kinds. For example: + * + * MPS_TELEMETRY_CONTROL=7 amcss + * + * will run the amcss test program and emit a file with event kinds 0, 1, 2. + * The file can then be imported into a SQLite database with this command: + * + * eventsql + * + * Note that the eventsql program can only read streams that come from an + * MPS compiled on the same platform. + * + * Each event type gets its own table in the database. These tables + * are created from the definitions in eventdef.h if they don't + * already exist. Each event becomes a single row in the appropriate + * table, which has a column for each event parameter, a time column + * for the event time field, and a log_serial column to identify the + * log file. + * + * The program also creates three other tables: two 'glue' tables + * containing event metadata - event_kind (one row per kind) and + * event_type (one row per type), again derived from eventdef.h - and + * the event_log table which has one row per log file imported (the + * log_serial column in the event tables is a primary key to this + * event_log table). + * + * No tables are created if they already exist, unless the -r + * (rebuild) switch is given. + * + * Options: + * + * -v (verbose): Increase verbosity. eventsql logs to stderr. By + * default, it doesn't log much; it can be made more and more + * loquacious by adding more -v switches. Given one or more -v + * switches, it also writes 'ticks' (periods) to stdout, to show + * progress (one dot is 100k events). + * + * -t (test): Run unit tests on parts of eventsql. There aren't many + * of these. + * + * -f (force): Import the events to SQL even if the SQL database + * already includes a record of importing this event log file (matched by + * size, modtime, and filesystem ID. + * + * -r (rebuild): Drop the glue tables from SQL, which will force them + * to be recreated. Important if you change event types or kinds in + * eventdef.h. + * + * -l : Import events from the named logfile. If not + * specified, eventsql will use the MPS_TELEMETRY_FILENAME environment + * variable, and default to mpsio.log. + * + * -d : Import events to the named database file. If not + * specified, eventsql will use the MPS_EVENT_DATABASE environment + * variable, and default to mpsevent.db. + * + * $Id$ + */ +#include "config.h" #include "eventdef.h" #include "eventpro.h" @@ -18,7 +85,7 @@ /* we output rows of dots. One dot per SMALL_TICK events, * BIG_TICK dots per row. */ -#define SMALL_TICK 1000 +#define SMALL_TICK 100000 #define BIG_TICK 50 /* Utility code for logging to stderr with multiple log levels, @@ -728,3 +795,44 @@ int main(int argc, char *argv[]) closeDatabase(db); return 0; } + +/* COPYRIGHT AND LICENSE + * + * Copyright (C) 2012 Ravenbrook Limited . + * All rights reserved. This is an open source license. Contact + * Ravenbrook for commercial licensing options. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Redistributions in any form must be accompanied by information on how + * to obtain complete source code for this software and any accompanying + * software that uses this software. The source code must either be + * included in the distribution or be available for no more than the cost + * of distribution plus a nominal fee, and must be freely redistributable + * under reasonable conditions. For an executable file, complete source + * code means the source code for all modules it contains. It does not + * include source code for modules or files that typically accompany the + * major components of the operating system on which the executable file + * runs. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/mps/code/eventsql.h b/mps/code/eventsql.h deleted file mode 100644 index a5d718fbffc..00000000000 --- a/mps/code/eventsql.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -- Event Database Definitions - * - * $Id$ - * Copyright (C) 2012 Ravenbrook Limited. See end of file for license. - */ - -#ifndef eventsql_h -#define eventsql_h - -#endif /* eventsql_h */ - -/* C. COPYRIGHT AND LICENSE - * - * Copyright (C) 2001-2002 Ravenbrook Limited . - * All rights reserved. This is an open source license. Contact - * Ravenbrook for commercial licensing options. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. Redistributions in any form must be accompanied by information on how - * to obtain complete source code for this software and any accompanying - * software that uses this software. The source code must either be - * included in the distribution or be available for no more than the cost - * of distribution plus a nominal fee, and must be freely redistributable - * under reasonable conditions. For an executable file, complete source - * code means the source code for all modules it contains. It does not - * include source code for modules or files that typically accompany the - * major components of the operating system on which the executable file - * runs. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - * PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ From efdc6b4018a2889d23e47679a2d696aa4fc4c76d Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Thu, 18 Oct 2012 17:00:31 +0100 Subject: [PATCH 09/14] Re-jigged eventsql so it uses plain-text intermediate files and handles stdin ok. Copied from Perforce Change: 179947 ServerID: perforce.ravenbrook.com --- mps/code/eventcnv.c | 15 +- mps/code/eventsql.c | 420 ++++++++++++++++++++++++++------------------ 2 files changed, 259 insertions(+), 176 deletions(-) diff --git a/mps/code/eventcnv.c b/mps/code/eventcnv.c index fcd569add74..42c84c376df 100644 --- a/mps/code/eventcnv.c +++ b/mps/code/eventcnv.c @@ -499,7 +499,7 @@ static void readLog(EventProc proc) case EventLabelCode: switch (style) { - case '\0': case 'C': + case '\0': { const char *sym = LabelText(proc, event->Label.f1); printf(style == '\0' ? @@ -507,12 +507,10 @@ static void readLog(EventProc proc) ", %"PRIuLONGEST", ", (ulongest_t)event->Label.f0); if (sym != NULL) { - printStr(sym, (style == 'C')); + printStr(sym, 0); } else { - printf(style == '\0' ? - "sym %05"PRIXLONGEST : - "sym %"PRIXLONGEST"\"", - (ulongest_t)event->Label.f1); + printf("sym %05"PRIXLONGEST , + (ulongest_t)event->Label.f1); } } break; @@ -521,6 +519,11 @@ static void readLog(EventProc proc) (ulongest_t)event->Label.f0, (ulongest_t)event->Label.f1); break; + case 'C': + printf(", %"PRIuLONGEST", %"PRIuLONGEST, + (ulongest_t)event->Label.f0, + (ulongest_t)event->Label.f1); + break; } break; diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index d4401192942..9b069ac65e6 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -1,30 +1,38 @@ -/* eventsql.c: event log to SQLite importer +/* eventsql.c: event log to SQLite importer. + * + * $Id* + * * Copyright (c) 2012 Ravenbrook Limited. See end of file for license. * - * This is a command-line tool that imports events from a binary - * format telemetry output file from the MPS into a SQLite database - * file. + * This is a command-line tool that imports events from a text-format + * MPS telemetry file into a SQLite database file. * - * The default MPS library will write a telemetry stream to a file called - * "mpsio.log" when the environment variable MPS_TELEMETRY_CONTROL is set - * to an integer whose bits select event kinds. For example: + * The default MPS library will write a binary-format telemetry file. + * The binary-format file can be converted into a text-format file + * using the eventcnv program with the -SC -v options. For + * binary-format compatibility, eventcnv has to be run on the same + * platform as the MPS. * - * MPS_TELEMETRY_CONTROL=7 amcss + * These ASCII text-format files have one line per event, and can be + * manipulated by various splendid systems in the usual Unix way. This + * eventsql program is one such. * - * will run the amcss test program and emit a file with event kinds 0, 1, 2. - * The file can then be imported into a SQLite database with this command: - * - * eventsql - * - * Note that the eventsql program can only read streams that come from an - * MPS compiled on the same platform. + * Note that eventsql can read streams that come from an MPS running + * on another platform. This is another reason why we use the + * intermediate text-format file, rather than reading the + * binary-format file directly: you can run the MPS and then eventcnv + * on the target platform, then optionally analyse the resulting text + * file on some other machine. * * Each event type gets its own table in the database. These tables * are created from the definitions in eventdef.h if they don't * already exist. Each event becomes a single row in the appropriate * table, which has a column for each event parameter, a time column * for the event time field, and a log_serial column to identify the - * log file. + * log file. Because the database schema depends on the event + * definitions in eventdef.h, eventsql has to be compiled using the + * same event header files as those used to compile the MPS and + * eventcnv which generated and processed the telemetry output. * * The program also creates three other tables: two 'glue' tables * containing event metadata - event_kind (one row per kind) and @@ -45,19 +53,19 @@ * progress (one dot is 100k events). * * -t (test): Run unit tests on parts of eventsql. There aren't many - * of these. + * of these. TODO: write more unit tests. * - * -f (force): Import the events to SQL even if the SQL database - * already includes a record of importing this event log file (matched by - * size, modtime, and filesystem ID. + * -f (force): Import the events to SQL even if the SQL database + * already includes a record of importing a matching log file. * * -r (rebuild): Drop the glue tables from SQL, which will force them * to be recreated. Important if you change event types or kinds in * eventdef.h. * - * -l : Import events from the named logfile. If not - * specified, eventsql will use the MPS_TELEMETRY_FILENAME environment - * variable, and default to mpsio.log. + * -l : Import events from the named logfile. Defaults to + * stdin. If the specified file (matched by size and modtime) has + * previously been imported to the same database, it will not be + * imported again unless -f is specified. * * -d : Import events to the named database file. If not * specified, eventsql will use the MPS_EVENT_DATABASE environment @@ -66,9 +74,10 @@ * $Id$ */ +#include "misc.h" #include "config.h" #include "eventdef.h" -#include "eventpro.h" +#include "eventcom.h" #include #include @@ -79,9 +88,6 @@ #define DATABASE_NAME_ENVAR "MPS_EVENT_DATABASE" #define DEFAULT_DATABASE_NAME "mpsevent.db" -#define TELEMETRY_FILENAME_ENVAR "MPS_TELEMETRY_FILENAME" -#define DEFAULT_TELEMETRY_FILENAME "mpsio.log" - /* we output rows of dots. One dot per SMALL_TICK events, * BIG_TICK dots per row. */ @@ -231,7 +237,7 @@ static sqlite3 *openDatabase(void) if (res != SQLITE_OK) sqlite_error(res, db, "Opening %s failed", databaseName); - log(LOG_ALWAYS, "Writing to %s.",databaseName); + log(LOG_OFTEN, "Writing to %s.",databaseName); return db; } @@ -243,7 +249,7 @@ static void closeDatabase(sqlite3 *db) int res = sqlite3_close(db); if (res != SQLITE_OK) sqlite_error(res, db, "Closing database failed"); - log(LOG_ALWAYS, "Closed %s.", databaseName); + log(LOG_SOMETIMES, "Closed %s.", databaseName); } /* We need to be able to test for the existence of a table. The @@ -268,7 +274,7 @@ static int tableExists(sqlite3* db, const char *tableName) if (!sql) error("Out of memory."); sprintf(sql, format, tableName); - log(LOG_RARELY, "Testing for existence of table '%s' with SQL: %s", tableName, sql); + log(LOG_SELDOM, "Testing for existence of table '%s' with SQL: %s", tableName, sql); res = sqlite3_exec(db, sql, NULL, /* put in a callback here if we really want to know the number of rows */ @@ -278,12 +284,12 @@ static int tableExists(sqlite3* db, const char *tableName) switch(res) { case SQLITE_OK: - log(LOG_RARELY, "Table '%s' exists.", tableName); + log(LOG_SELDOM, "Table '%s' exists.", tableName); return 1; /* table exists */ break; case SQLITE_ERROR: - log(LOG_RARELY, "Table '%s' does not exist.", tableName); + log(LOG_SELDOM, "Table '%s' does not exist.", tableName); return 0; /* table does not exist; we can probably do a better test for this case. */ break; @@ -344,6 +350,7 @@ static void runStatement(sqlite3 *db, const char *description) { int res; + log(LOG_SELDOM, "%s: %s", description, sql); res = sqlite3_exec(db, sql, NULL, /* No callback */ @@ -366,70 +373,74 @@ static unsigned long logSerial = 0; static void registerLogFile(sqlite3 *db, const char *filename) { - struct stat st; sqlite3_stmt *statement; int res; const unsigned char *name; unsigned long completed; + unsigned long long file_size; + unsigned long long file_modtime; - res = stat(filename, &st); - if (res != 0) - error("Couldn't stat() %s", filename); - - statement = prepareStatement(db, - "SELECT name, serial, completed FROM event_log" - " WHERE file_id = ? AND size = ? AND modtime = ?"); - res = sqlite3_bind_int64(statement, 1, st.st_ino); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_log bind of file_id failed."); - res = sqlite3_bind_int64(statement, 2, st.st_size); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_log bind of size failed."); - res = sqlite3_bind_int64(statement, 3, st.st_mtime); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_log bind of modtime failed."); - - res = sqlite3_step(statement); - switch(res) { - case SQLITE_DONE: - log(LOG_SOMETIMES, "No log file matching '%s' found in database.", filename); - break; - case SQLITE_ROW: - name = sqlite3_column_text(statement, 0); - logSerial = sqlite3_column_int(statement, 1); - completed = sqlite3_column_int(statement, 2); - log(LOG_ALWAYS, "Log file matching '%s' already in event_log, named \"%s\" (serial %lu, completed %lu).", - filename, name, logSerial, completed); - if (!force) { - log(LOG_ALWAYS, "Exiting. Specify -f to force events into SQL anyway."); - exit(0); + if (filename) { + struct stat st; + res = stat(filename, &st); + if (res != 0) + error("Couldn't stat() %s", filename); + file_size = st.st_size; + file_modtime = st.st_mtime; + + statement = prepareStatement(db, + "SELECT name, serial, completed FROM event_log" + " WHERE size = ? AND modtime = ?"); + res = sqlite3_bind_int64(statement, 1, file_size); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log bind of size failed."); + res = sqlite3_bind_int64(statement, 2, file_modtime); + if (res != SQLITE_OK) + sqlite_error(res, db, "event_log bind of modtime failed."); + res = sqlite3_step(statement); + switch(res) { + case SQLITE_DONE: + log(LOG_SOMETIMES, "No log file matching '%s' found in database.", filename); + break; + case SQLITE_ROW: + name = sqlite3_column_text(statement, 0); + logSerial = sqlite3_column_int(statement, 1); + completed = sqlite3_column_int(statement, 2); + log(force ? LOG_OFTEN : LOG_ALWAYS, "Log file matching '%s' already in event_log, named \"%s\" (serial %lu, completed %lu).", + filename, name, logSerial, completed); + if (force) { + log(LOG_OFTEN, "Continuing anyway because -f specified."); + } else { + log(LOG_ALWAYS, "Exiting. Specify -f to force events into SQL anyway."); + exit(0); + } + break; + default: + sqlite_error(res, db, "select from event_log failed."); } - log(LOG_ALWAYS, "Continuing anyway because -f specified."); - break; - default: - sqlite_error(res, db, "select from event_log failed."); + finalizeStatement(db, statement); + } else { /* stdin */ + filename = ""; + file_size = 0; + file_modtime = 0; } - finalizeStatement(db, statement); statement = prepareStatement(db, - "INSERT into event_log (name, file_id, size, modtime, completed)" - " VALUES (?, ?, ?, ?, 0)"); + "INSERT into event_log (name, size, modtime, completed)" + " VALUES (?, ?, ?, 0)"); res = sqlite3_bind_text(statement, 1, filename, -1, SQLITE_STATIC); if (res != SQLITE_OK) sqlite_error(res, db, "event_log insert bind of name failed."); - res = sqlite3_bind_int64(statement, 2, st.st_ino); - if (res != SQLITE_OK) - sqlite_error(res, db, "event_log insert bind of file_id failed."); - res = sqlite3_bind_int64(statement, 3, st.st_size); + res = sqlite3_bind_int64(statement, 2, file_size); if (res != SQLITE_OK) sqlite_error(res, db, "event_log insert bind of size failed."); - res = sqlite3_bind_int64(statement, 4, st.st_mtime); + res = sqlite3_bind_int64(statement, 3, file_modtime); if (res != SQLITE_OK) sqlite_error(res, db, "event_log insert bind of modtime failed."); res = sqlite3_step(statement); if (res != SQLITE_DONE) sqlite_error(res, db, "insert into event_log failed."); logSerial = sqlite3_last_insert_rowid(db); - log(LOG_SOMETIMES, "Log file added to event_log with serial %lu", + log(LOG_SOMETIMES, "Log file %s added to event_log with serial %lu", filename, logSerial); finalizeStatement(db, statement); } @@ -451,7 +462,7 @@ static void logFileCompleted(sqlite3 *db, res = sqlite3_step(statement); if (res != SQLITE_DONE) sqlite_error(res, db, "insert into event_log failed."); - log(LOG_OFTEN, "Marked in event_log: %lu events", completed); + log(LOG_SOMETIMES, "Marked in event_log: %lu events", completed); finalizeStatement(db, statement); } @@ -488,7 +499,6 @@ const char *createStatements[] = { " FOREIGN KEY (kind) REFERENCES event_kind(enum));", "CREATE TABLE IF NOT EXISTS event_log (name TEXT," - " file_id INTEGER," " size INTEGER," " modtime INTEGER," " completed INTEGER," @@ -502,9 +512,9 @@ EVENT_LIST(EVENT_TABLE_CREATE, X) static void makeTables(sqlite3 *db) { int i; + log(LOG_SOMETIMES, "Creating tables."); for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { - log(LOG_SOMETIMES, "Creating tables. SQL command: %s", createStatements[i]); runStatement(db, createStatements[i], "Table creation"); } } @@ -581,8 +591,8 @@ static void dropGlueTables(sqlite3 *db) static void fillGlueTables(sqlite3 *db) { int i; - Res res; sqlite3_stmt *statement; + int res; statement = prepareStatement(db, "INSERT OR IGNORE INTO event_kind (name, description, enum)" @@ -622,61 +632,100 @@ static void fillGlueTables(sqlite3 *db) #define EVENT_TYPE_FINALIZE_STATEMENT(X, name, code, always, kind) \ finalizeStatement(db, stmt_##name); -#define EVENT_PARAM_BIND_INTEGER(name, index, sort, ident) \ - res = sqlite3_bind_int64(statement, index+1, (unsigned long) event->name.f##index); +#define EVENT_PARAM_BIND_A bind_int +#define EVENT_PARAM_BIND_P bind_int +#define EVENT_PARAM_BIND_U bind_int +#define EVENT_PARAM_BIND_W bind_int +#define EVENT_PARAM_BIND_D bind_real +#define EVENT_PARAM_BIND_S bind_text +#define EVENT_PARAM_BIND_B bind_int -#define EVENT_PARAM_BIND_REAL(name, index, sort, ident) \ - res = sqlite3_bind_double(statement, index+1, event->name.f##index); - -#define EVENT_PARAM_BIND_TEXT(name, index, sort, ident) \ - res = sqlite3_bind_text(statement, index+1, event->name.f##index, -1, SQLITE_STATIC); - -#define EVENT_PARAM_BIND_A EVENT_PARAM_BIND_INTEGER -#define EVENT_PARAM_BIND_P EVENT_PARAM_BIND_INTEGER -#define EVENT_PARAM_BIND_U EVENT_PARAM_BIND_INTEGER -#define EVENT_PARAM_BIND_W EVENT_PARAM_BIND_INTEGER -#define EVENT_PARAM_BIND_D EVENT_PARAM_BIND_REAL -#define EVENT_PARAM_BIND_S EVENT_PARAM_BIND_TEXT -#define EVENT_PARAM_BIND_B EVENT_PARAM_BIND_INTEGER - -#define EVENT_PARAM_BIND(name, index, sort, ident) \ - EVENT_PARAM_BIND_##sort (name, index, sort, ident) \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Event " #name " bind of ident " #ident "failed."); \ +#define EVENT_PARAM_BIND(X, index, sort, ident) \ + p = EVENT_PARAM_BIND_##sort (db, statement, eventCount, index+1, p); \ last_index = index+1; - #define EVENT_TYPE_WRITE_SQL(X, name, code, always, kind) \ case code: \ - { \ - sqlite3_stmt *statement = stmt_##name; \ - int last_index = 0; \ - int res; \ + statement = stmt_##name; \ /* bind all the parameters of this particular event with macro magic. */ \ - EVENT_##name##_PARAMS(EVENT_PARAM_BIND, name) \ - /* bind the fields we store for every event */ \ - res = sqlite3_bind_int64(statement, last_index+1, logSerial); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Event " #name " bind of log_serial failed."); \ - res = sqlite3_bind_int64(statement, last_index+2, event->any.clock); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Event " #name " bind of clock failed."); \ - res = sqlite3_step(statement); \ - if (res != SQLITE_DONE) \ - sqlite_error(res, db, "insert of event \"" #name "\" failed."); \ - res = sqlite3_reset(statement); \ - if (res != SQLITE_OK) \ - sqlite_error(res, db, "Couldn't reset insert statement for \"" #name "\"."); \ - } \ - break; + EVENT_##name##_PARAMS(EVENT_PARAM_BIND, X) \ + break; -/* readLog -- read and parse log +static char *bind_int(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +{ + char *q; + long long val; + int res; + + if ((p[0] != ',') || (p[1] != ' ')) + error("event %llu field %d not preceded by \", \": %s", + count, index, p); + + p += 2; + val = strtoll(p, &q, 0); + if (q == p) + error("event %llu field %d not an integer: %s", + count, index, p); + + res = sqlite3_bind_int64(stmt, index, val); + if (res != SQLITE_OK) + sqlite_error(res, db, "event %llu field %d bind failed", count, index); + return q; +} + +static char *bind_real(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +{ + char *q; + double val; + int res; + + if ((p[0] != ',') || (p[1] != ' ')) + error("event %llu field %d not preceded by \", \": %s", + count, index, p); + + p += 2; + val = strtod(p, &q); + if (q == p) + error("event %llu field %d not a floating-point value: %s", + count, index, p); + + res = sqlite3_bind_double(stmt, index, val); + if (res != SQLITE_OK) + sqlite_error(res, db, "event %llu field %d bind failed", count, index); + return q; +} + +static char *bind_text(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +{ + char *q; + int res; + + if ((p[0] != ',') || (p[1] != ' ') || (p[2] != '"')) + error("event %llu string field %d not preceded by \", \\\"\": %s", + count, index, p); + + p += 3; + q = p; + while((*q != '\n') && (*q != '\0')) { + ++ q; + } + if ((q == p) || (q[-1] != '"')) + error("event %llu string field %d has no closing quote mark.", + count, index); + + res = sqlite3_bind_text(stmt, index, p, q-p-1, SQLITE_STATIC); + if (res != SQLITE_OK) + sqlite_error(res, db, "event %llu field %d bind failed", count, index); + return q; +} + +/* readLog -- read and parse log. Returns the number of events written. */ -static void readLog(EventProc proc, - sqlite3 *db) +static unsigned long long readLog(FILE *input, + sqlite3 *db) { - size_t eventCount = 0; + unsigned long long eventCount = 0; /* declare statements for every event type */ EVENT_LIST(EVENT_TYPE_DECLARE_STATEMENT, X); @@ -687,28 +736,70 @@ static void readLog(EventProc proc, runStatement(db, "BEGIN", "Transaction start"); while (TRUE) { /* loop for each event */ - Event event; - EventCode code; - Res res; - - /* Read and parse event. */ - res = EventRead(&event, proc); - if (res == ResFAIL) break; /* eof */ - if (res != ResOK) error("Truncated log"); - code = event->any.code; - + char line[1024]; + char *p; + char *q; + int last_index=0; + sqlite3_stmt *statement; + int res; + unsigned long long clock; + int code; + + p = fgets(line, 1024, input); + if (!p) { + if (feof(input)) + break; + else + error("Couldn't read line after event %llu", eventCount); + } + + eventCount++; + + clock = strtoll(p, &q, 16); + if (q == p) + error("event %llu clock field not a hex integer: %s", + eventCount, p); + + if ((q[0] != ',') || (q[1] != ' ')) + error("event %llu code field not preceded by \", \": %s", + eventCount, q); + + p = q + 2; + + code = strtol(p, &q, 0); + if (q == p) + error("event %llu code field %d not an integer: %s", + eventCount, index, p); + p = q; /* Write event to SQLite. */ switch (code) { + /* this macro sets statement and last_index */ EVENT_LIST(EVENT_TYPE_WRITE_SQL, X); + default: + error("Event %llu has Unknown event code %d", eventCount, code); } - EventDestroy(proc, event); - eventCount++; + /* bind the fields we store for every event */ \ + res = sqlite3_bind_int64(statement, last_index+1, logSerial); + if (res != SQLITE_OK) + sqlite_error(res, db, "Event %llu bind of log_serial failed.", eventCount); + res = sqlite3_bind_int64(statement, last_index+2, clock); + if (res != SQLITE_OK) + sqlite_error(res, db, "Event %llu bind of clock failed.", eventCount); + res = sqlite3_step(statement); + if (res != SQLITE_DONE) + sqlite_error(res, db, "insert of event %llu failed.", eventCount); + res = sqlite3_reset(statement); + if (res != SQLITE_OK) + sqlite_error(res, db, "Couldn't reset insert statement of event %llu", eventCount); + if (verbosity > LOG_ALWAYS) { if ((eventCount % SMALL_TICK) == 0) { printf("."); fflush(stdout); if (((eventCount / SMALL_TICK) % BIG_TICK) == 0) { - log(LOG_OFTEN, "%lu events.", (unsigned long)eventCount); + printf("\n"); + fflush(stdout); + log(LOG_SOMETIMES, "%lu events.", (unsigned long)eventCount); } } } @@ -718,64 +809,48 @@ static void readLog(EventProc proc, fflush(stdout); } runStatement(db, "COMMIT", "Transaction finish"); - - log(LOG_ALWAYS, "Wrote %lu events to SQL.", (unsigned long)eventCount); logFileCompleted(db, eventCount); /* finalize all the statements */ EVENT_LIST(EVENT_TYPE_FINALIZE_STATEMENT, X); -} -static Res logReader(void *file, void *p, size_t len) -{ - size_t n; - - n = fread(p, 1, len, (FILE *)file); - return (n < len) ? (feof((FILE *)file) ? ResFAIL : ResIO) : ResOK; + return eventCount; } static FILE *openLog(sqlite3 *db) { FILE *input; + registerLogFile(db, logFileName); if (!logFileName) { - logFileName = getenv(TELEMETRY_FILENAME_ENVAR); - if(logFileName == NULL) - logFileName = DEFAULT_TELEMETRY_FILENAME; + input = stdin; + logFileName = ""; + } else { + input = fopen(logFileName, "r"); + if (input == NULL) + error("unable to open %s", logFileName); } - registerLogFile(db, logFileName); - input = fopen(logFileName, "rb"); - - if (input == NULL) - error("unable to open %s", logFileName); - - log(LOG_ALWAYS, "Reading %s.", logFileName); + log(LOG_OFTEN, "Reading %s.", logFileName ? logFileName : "standard input"); return input; } -static void writeEventsToSQL(sqlite3 *db) +static unsigned long long writeEventsToSQL(sqlite3 *db) { - Res res; - EventProc proc; FILE *input; - + unsigned long long count; input = openLog(db); - res = EventProcCreate(&proc, logReader, (void *)input); - if (res != ResOK) - error("Can't init EventProc module: error %d.", res); - - readLog(proc, db); - - EventProcDestroy(proc); + count = readLog(input, db); (void)fclose(input); + return count; } int main(int argc, char *argv[]) { sqlite3 *db; + unsigned long long count; parseArgs(argc, argv); @@ -785,7 +860,12 @@ int main(int argc, char *argv[]) } makeTables(db); fillGlueTables(db); - writeEventsToSQL(db); + count = writeEventsToSQL(db); + log(LOG_ALWAYS, "Imported %llu events from %s to %s, serial %lu.", + (unsigned long)count, + logFileName, + databaseName, + logSerial); if (runTests) { /* TODO: more unit tests in here */ From f0757a16404b0d4e90facee261b78101568a66cd Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Fri, 19 Oct 2012 07:32:40 +0100 Subject: [PATCH 10/14] Add event_param glue table. Copied from Perforce Change: 179953 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 72 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 9b069ac65e6..1e995bb0f67 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -1,6 +1,6 @@ /* eventsql.c: event log to SQLite importer. * - * $Id* + * $Id$ * * Copyright (c) 2012 Ravenbrook Limited. See end of file for license. * @@ -34,12 +34,12 @@ * same event header files as those used to compile the MPS and * eventcnv which generated and processed the telemetry output. * - * The program also creates three other tables: two 'glue' tables - * containing event metadata - event_kind (one row per kind) and - * event_type (one row per type), again derived from eventdef.h - and - * the event_log table which has one row per log file imported (the - * log_serial column in the event tables is a primary key to this - * event_log table). + * The program also creates several other tables: three 'glue' tables + * containing event metadata - event_kind (one row per kind), + * event_type (one row per type), and event_param (one row per + * parameter), all derived from eventdef.h - and the event_log table + * which has one row per log file imported (the log_serial column in + * the event tables is a primary key to this event_log table). * * No tables are created if they already exist, unless the -r * (rebuild) switch is given. @@ -332,7 +332,7 @@ static sqlite3_stmt *prepareStatement(sqlite3 *db, &statement, NULL); if (res != SQLITE_OK) - sqlite_error(res, db, "statementpreparation failed: %s", sql); + sqlite_error(res, db, "statement preparation failed: %s", sql); return statement; } @@ -342,7 +342,7 @@ static void finalizeStatement(sqlite3 *db, int res; res = sqlite3_finalize(statement); if (res != SQLITE_OK) - sqlite_error(res, db, "event_type finalize failed"); + sqlite_error(res, db, "statement finalize failed"); } static void runStatement(sqlite3 *db, @@ -366,7 +366,12 @@ static void runStatement(sqlite3 *db, * from SQL which is then attached to all event rows from that log. * We use this to record overall SQL activity, to deter mistaken * attempts to add the same log file twice, and to allow events from - * several different log files to share the same SQL file. */ + * several different log files to share the same SQL file. + * + * When reading events from stdin, we can't so easily avoid the + * duplication (unless we, e.g., take a hash of the event set); we + * assume that the user is smart enough not to do that. + */ static unsigned long logSerial = 0; @@ -498,6 +503,12 @@ const char *createStatements[] = { " kind INTEGER," " FOREIGN KEY (kind) REFERENCES event_kind(enum));", + "CREATE TABLE IF NOT EXISTS event_param (type INTEGER," + " param_index INTEGER," + " sort TEXT," + " ident TEXT," + " FOREIGN KEY (type) REFERENCES event_type(code));", + "CREATE TABLE IF NOT EXISTS event_log (name TEXT," " size INTEGER," " modtime INTEGER," @@ -522,6 +533,7 @@ static void makeTables(sqlite3 *db) const char *glueTables[] = { "event_kind", "event_type", + "event_param", }; static void dropGlueTables(sqlite3 *db) @@ -544,7 +556,8 @@ static void dropGlueTables(sqlite3 *db) } } -/* Populate the metadata "glue" tables event_kind and event_type. */ +/* Populate the metadata "glue" tables event_kind, event_type, and + * event_param. */ #define EVENT_KIND_DO_INSERT(X, name, description) \ res = sqlite3_bind_text(statement, 1, #name, -1, SQLITE_STATIC); \ @@ -588,6 +601,31 @@ static void dropGlueTables(sqlite3 *db) if (res != SQLITE_OK) \ sqlite_error(res, db, "Couldn't reset event_type insert statement."); +#define EVENT_PARAM_DO_INSERT(code, index, sort, ident) \ + res = sqlite3_bind_int(statement, 1, code); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_param bind of code %d failed.", code); \ + res = sqlite3_bind_int(statement, 2, index); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_param bind of index %d failed.", index); \ + res = sqlite3_bind_text(statement, 3, #sort, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of sort \"" #sort "\" failed."); \ + res = sqlite3_bind_text(statement, 4, #ident, -1, SQLITE_STATIC); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "event_type bind of ident \"" #ident "\" failed."); \ + res = sqlite3_step(statement); \ + if (res != SQLITE_DONE) \ + sqlite_error(res, db, "event_param insert of ident \"" #ident "\" for code %d failed.", code); \ + if (sqlite3_changes(db) != 0) \ + log(LOG_SOMETIMES, "Insert of event_param row for code %d, ident \"" #ident "\" affected %d rows.", code, sqlite3_changes(db)); \ + res = sqlite3_reset(statement); \ + if (res != SQLITE_OK) \ + sqlite_error(res, db, "Couldn't reset event_param insert statement."); + +#define EVENT_TYPE_INSERT_PARAMS(X, name, code, always, kind) \ + EVENT_##name##_PARAMS(EVENT_PARAM_DO_INSERT, code) + static void fillGlueTables(sqlite3 *db) { int i; @@ -609,6 +647,13 @@ static void fillGlueTables(sqlite3 *db) EVENT_LIST(EVENT_TYPE_DO_INSERT, X); finalizeStatement(db, statement); + + statement = prepareStatement(db, + "INSERT OR IGNORE INTO event_param (type, param_index, sort, ident)" + "VALUES (?, ?, ?, ?)"); + EVENT_LIST(EVENT_TYPE_INSERT_PARAMS, X); + + finalizeStatement(db, statement); } /* Populate the actual event tables. */ @@ -862,10 +907,7 @@ int main(int argc, char *argv[]) fillGlueTables(db); count = writeEventsToSQL(db); log(LOG_ALWAYS, "Imported %llu events from %s to %s, serial %lu.", - (unsigned long)count, - logFileName, - databaseName, - logSerial); + count, logFileName, databaseName, logSerial); if (runTests) { /* TODO: more unit tests in here */ From 1822b236377d1801c141b19014e751046e596a84 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Sat, 20 Oct 2012 20:34:30 +0100 Subject: [PATCH 11/14] Better table existence test. Copied from Perforce Change: 179985 ServerID: perforce.ravenbrook.com --- mps/code/eventsql.c | 127 +++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 67 deletions(-) diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index 1e995bb0f67..d7cf6a3e3f5 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -252,73 +252,6 @@ static void closeDatabase(sqlite3 *db) log(LOG_SOMETIMES, "Closed %s.", databaseName); } -/* We need to be able to test for the existence of a table. The - * SQLite3 API seems to have no way to explore metadata like this, - * unless it is compiled in a particular way (in which case the - * function sqlite3_table_column_metadata could be used). Without - * that assistance, we can use a simple SQL trick (which could also - * tell us the number of rows in the table if we cared). - * - * TODO: Update this to use the sqlite_master table: - * SELECT FROM sqlite_master WHERE type='table' AND name=? - * and fix the above comment. - */ - -static int tableExists(sqlite3* db, const char *tableName) -{ - const char *format = "SELECT SUM(1) FROM %s"; - char *sql; - int res; - - sql = malloc(strlen(format) + strlen(tableName)); - if (!sql) - error("Out of memory."); - sprintf(sql, format, tableName); - log(LOG_SELDOM, "Testing for existence of table '%s' with SQL: %s", tableName, sql); - res = sqlite3_exec(db, - sql, - NULL, /* put in a callback here if we really want to know the number of rows */ - NULL, /* callback closure */ - NULL); /* error messages handled by sqlite_error */ - free(sql); - - switch(res) { - case SQLITE_OK: - log(LOG_SELDOM, "Table '%s' exists.", tableName); - - return 1; /* table exists */ - break; - case SQLITE_ERROR: - log(LOG_SELDOM, "Table '%s' does not exist.", tableName); - return 0; /* table does not exist; we can - probably do a better test for this case. */ - break; - default: - sqlite_error(res, db, "Table test failed: %s", tableName); - } - /* UNREACHED */ - return 0; -} - -/* Unit test for tableExists() */ - -static const char *tableTests[] = { - "event_kind", - "spong", - "EVENT_SegSplit", -}; - -static void testTableExists(sqlite3 *db) -{ - int i; - for (i=0; i < (sizeof(tableTests)/sizeof(tableTests[0])); ++i) { - if (tableExists(db, tableTests[i])) - printf("Table exists: %s\n", tableTests[i]); - else - printf("Table does not exist: %s\n", tableTests[i]); - } -} - /* Utility functions for SQLite statements. */ static sqlite3_stmt *prepareStatement(sqlite3 *db, @@ -360,6 +293,66 @@ static void runStatement(sqlite3 *db, sqlite_error(res, db, "%s failed - statement %s", description, sql); } +/* Test for the existence of a table using sqlite_master table. + */ + +static int tableExists(sqlite3* db, const char *tableName) +{ + int res; + int exists; + sqlite3_stmt *statement; + + statement = prepareStatement(db, + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?"); + res = sqlite3_bind_text(statement, 1, tableName, -1, SQLITE_STATIC); + if (res != SQLITE_OK) + sqlite_error(res, db, "table existence bind of name failed."); + res = sqlite3_step(statement); + switch(res) { + case SQLITE_DONE: + exists = 0; + break; + case SQLITE_ROW: + exists = 1; + break; + default: + sqlite_error(res, db, "select from sqlite_master failed."); + } + finalizeStatement(db, statement); + return exists; +} + +/* Unit test for tableExists() */ + +static struct { + const char* name; + int exists; +} tableTests[] = { + {"event_kind", TRUE}, + {"spong", FALSE}, + {"EVENT_SegSplit", TRUE} +}; + +static void testTableExists(sqlite3 *db) +{ + int i; + int defects = 0; + int tests = 0; + for (i=0; i < (sizeof(tableTests)/sizeof(tableTests[0])); ++i) { + const char *name = tableTests[i].name; + int exists = tableExists(db, name); + if (exists) + log(LOG_OFTEN, "Table exists: %s", name); + else + log(LOG_OFTEN, "Table does not exist: %s", name); + if (exists != tableTests[i].exists) { + log(LOG_ALWAYS, "tableExists test failed on table %s", name); + ++ defects; + } + ++ tests; + } + log(LOG_ALWAYS, "%d tests, %d defects found.", tests, defects); +} /* Every time we put events from a log file into a database file, we * add the log file to the event_log table, and get a serial number From 750d51523316c37046c0c31e2a53d19b3b46fee8 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Wed, 24 Oct 2012 16:10:46 +0100 Subject: [PATCH 12/14] Further simplify the eventcnv text output format. Copied from Perforce Change: 180054 ServerID: perforce.ravenbrook.com --- mps/code/eventcnv.c | 274 ++++--------------------- mps/code/eventsql.c | 79 ++++--- mps/code/mps.xcodeproj/project.pbxproj | 113 ++++++++++ 3 files changed, 190 insertions(+), 276 deletions(-) diff --git a/mps/code/eventcnv.c b/mps/code/eventcnv.c index b1535ace624..d1c9de4d518 100644 --- a/mps/code/eventcnv.c +++ b/mps/code/eventcnv.c @@ -2,7 +2,7 @@ * Copyright (c) 2001 Ravenbrook Limited. See end of file for license. * * This is a command-line tool that converts a binary format telemetry output - * stream from the MPS into several textual formats. + * stream from the MPS into a textual format. * * The default MPS library will write a telemetry stream to a file called * "mpsio.log" when the environment variable MPS_TELEMETRY_CONTROL is set @@ -22,20 +22,16 @@ */ #include "config.h" - #include "eventdef.h" #include "eventcom.h" #include "eventpro.h" -#include "mpmtypes.h" #include "testlib.h" /* for ulongest_t and associated print formats */ #include /* for size_t */ #include /* for printf */ -#include /* for va_list */ #include /* for EXIT_FAILURE */ #include /* for assert */ #include /* for strcmp */ -#include /* for sqrt */ #include "mpstd.h" #ifdef MPS_BUILD_MV @@ -44,20 +40,12 @@ #pragma warning( disable : 4996 ) #endif - - -typedef unsigned int uint; -typedef unsigned long ulong; - +#define DEFAULT_TELEMETRY_FILENAME "mpsio.log" +#define TELEMETRY_FILENAME_ENVAR "MPS_TELEMETRY_FILENAME" static EventClock eventTime; /* current event time */ static char *prog; /* program name */ - -/* style: '\0' for human-readable, 'L' for Lisp, 'C' for CDF. */ -static char style = '\0'; - - /* everror -- error signalling */ static void everror(const char *format, ...) @@ -80,7 +68,7 @@ static void everror(const char *format, ...) static void usage(void) { fprintf(stderr, - "Usage: %s [-f logfile] [-S[LC]] [-h]\n" + "Usage: %s [-f logfile] [-h]\n" "See \"Telemetry\" in the reference manual for instructions.\n", prog); } @@ -99,7 +87,7 @@ static void usageError(void) static char *parseArgs(int argc, char *argv[]) { - char *name = "mpsio.log"; + char *name = NULL; int i = 1; if (argc >= 1) @@ -117,9 +105,6 @@ static char *parseArgs(int argc, char *argv[]) else name = argv[i]; break; - case 'S': /* style */ - style = argv[i][2]; /* '\0' for human-readable, 'L' for Lisp, */ - break; /* 'C' for CDF. */ case '?': case 'h': /* help */ usage(); exit(EXIT_SUCCESS); @@ -135,119 +120,39 @@ static char *parseArgs(int argc, char *argv[]) /* Printing routines */ +static void printHex(ulongest_t val) +{ + printf(" %"PRIXLONGEST, (ulongest_t)val); +} + +#define printParamP(p) printHex((ulongest_t)p) +#define printParamA(a) printHex((ulongest_t)a) +#define printParamU(u) printHex((ulongest_t)u) +#define printParamW(w) printHex((ulongest_t)w) +#define printParamB(b) printHex((ulongest_t)b) -/* printStr -- print an EventString */ +static void printParamD(double d) +{ + printf(" %.10G", d); +} -static void printStr(const char *str, Bool quotes) +static void printParamS(const char *str) { size_t i; - - if (quotes) putchar('"'); + putchar(' '); + putchar('"'); for (i = 0; str[i] != '\0'; ++i) { char c = str[i]; - if (quotes && (c == '"' || c == '\\')) putchar('\\'); + if (c == '"' || c == '\\') putchar('\\'); putchar(c); } - if (quotes) putchar('"'); + putchar('"'); } - -/* printAddr -- print an Addr or its label */ - -static void printAddr(EventProc proc, Addr addr) -{ - UNUSED(proc); - printf(style != 'C' ? - " %0"PRIwWORD PRIXLONGEST : - " %"PRIuLONGEST, - (ulongest_t)addr); -} - - -/* printParam* -- printing functions for event parameter types */ - -static void printParamA(EventProc proc, char *styleConv, Addr addr) -{ - if (style != 'L') { - if (style == 'C') putchar(','); - printAddr(proc, addr); - } else - printf(styleConv, (ulongest_t)addr); -} - -static void printParamP(EventProc proc, char *styleConv, void *p) -{ - UNUSED(proc); - printf(styleConv, (ulongest_t)p); -} - -static void printParamU(EventProc proc, char *styleConv, unsigned u) -{ - UNUSED(proc); - printf(styleConv, (ulongest_t)u); -} - -static void printParamW(EventProc proc, char *styleConv, Word w) -{ - UNUSED(proc); - printf(styleConv, (ulongest_t)w); -} - -static void printParamD(EventProc proc, char *styleConv, double d) -{ - UNUSED(proc); - UNUSED(styleConv); - switch (style) { - case '\0': - printf(" %#8.3g", d); break; - case 'C': - printf(", %.10G", d); break; - case 'L': - printf(" %#.10G", d); break; - } -} - -static void printParamS(EventProc proc, char *styleConv, const char *s) -{ - UNUSED(proc); - UNUSED(styleConv); - if (style == 'C') putchar(','); - putchar(' '); - printStr(s, (style == 'C' || style == 'L')); -} - -static void printParamB(EventProc proc, char *styleConv, Bool b) -{ - UNUSED(proc); - printf(styleConv, (ulongest_t)b); -} - - -/* readLog -- read and parse log - * - * This is the heart of eventcnv: It reads an event log using - * EventRead. It updates the counters. It looks up the format, - * parses the arguments, and prints a representation of the event. - * Each argument is printed using printArg (see RELATION, below), - * except for some event types that are handled specially. - */ +/* readLog -- read and parse log */ static void readLog(EventProc proc) { - char *styleConv = NULL; /* suppress uninit warning */ - - /* Init style. */ - switch (style) { - case '\0': - styleConv = " %8"PRIXLONGEST; break; - case 'C': - styleConv = ", %"PRIuLONGEST; break; - case 'L': - styleConv = " %"PRIXLONGEST; break; - default: - everror("Unknown style code '%c'", style); - } - while (TRUE) { /* loop for each event */ Event event; EventCode code; @@ -260,131 +165,25 @@ static void readLog(EventProc proc) eventTime = event->any.clock; code = event->any.code; - /* Output event. */ - { - if (style == 'L') putchar('('); + EVENT_CLOCK_PRINT(stdout, eventTime); + printf(" %X", (unsigned)code); - switch (style) { - case '\0': case 'L': - EVENT_CLOCK_PRINT(stdout, eventTime); - putchar(' '); - break; - case 'C': - EVENT_CLOCK_PRINT(stdout, eventTime); - fputs(", ", stdout); - break; - } - - switch (style) { - case '\0': case 'L': { - printf("%-19s ", EventCode2Name(code)); - } break; - case 'C': - printf("%u", (unsigned)code); - break; - } - - switch (code) { - - case EventLabelCode: - switch (style) { - case '\0': - { - const char *sym = LabelText(proc, event->Label.f1); - printf(style == '\0' ? - " %08"PRIXLONGEST" " : - ", %"PRIuLONGEST", ", - (ulongest_t)event->Label.f0); - if (sym != NULL) { - printStr(sym, 0); - } else { - printf("sym %05"PRIXLONGEST , - (ulongest_t)event->Label.f1); - } - } - break; - case 'L': - printf(" %"PRIXLONGEST" %"PRIXLONGEST, - (ulongest_t)event->Label.f0, - (ulongest_t)event->Label.f1); - break; - case 'C': - printf(", %"PRIuLONGEST", %"PRIuLONGEST, - (ulongest_t)event->Label.f0, - (ulongest_t)event->Label.f1); - break; - } - break; - - case EventMeterValuesCode: - switch (style) { - case '\0': - if (event->MeterValues.f3 == 0) { - printf(" %08"PRIXLONGEST" 0 N/A N/A N/A N/A", - (ulongest_t)event->MeterValues.f0); - } else { - double mean = event->MeterValues.f1 / (double)event->MeterValues.f3; - /* .stddev: stddev = sqrt(meanSquared - mean^2), but see */ - /* . */ - double stddev = sqrt(fabs(event->MeterValues.f2 - - (mean * mean))); - printf(" %08"PRIXLONGEST" %8u %8u %8u %#8.3g %#8.3g", - (ulongest_t)event->MeterValues.f0, (uint)event->MeterValues.f3, - (uint)event->MeterValues.f4, (uint)event->MeterValues.f5, - mean, stddev); - } - printAddr(proc, (Addr)event->MeterValues.f0); - break; - - case 'C': - putchar(','); - printAddr(proc, (Addr)event->MeterValues.f0); - printf(", %.10G, %.10G, %u, %u, %u", - event->MeterValues.f1, event->MeterValues.f2, - (uint)event->MeterValues.f3, (uint)event->MeterValues.f4, - (uint)event->MeterValues.f5); - break; - - case 'L': - printf(" %"PRIXLONGEST" %#.10G %#.10G %X %X %X", - (ulongest_t)event->MeterValues.f0, - event->MeterValues.f1, event->MeterValues.f2, - (uint)event->MeterValues.f3, (uint)event->MeterValues.f4, - (uint)event->MeterValues.f5); - break; - } - break; - - case EventPoolInitCode: /* pool, arena, class */ - printf(styleConv, (ulongest_t)event->PoolInit.f0); - printf(styleConv, (ulongest_t)event->PoolInit.f1); - /* class is a Pointer, but we label them, so call printAddr */ - if (style != 'L') { - if (style == 'C') putchar(','); - printAddr(proc, (Addr)event->PoolInit.f2); - } else - printf(styleConv, (ulongest_t)event->PoolInit.f2); - break; - - default: + switch (code) { #define EVENT_PARAM_PRINT(name, index, sort, ident) \ - printParam##sort(proc, styleConv, event->name.f##index); + printParam##sort(event->name.f##index); #define EVENT_PRINT(X, name, code, always, kind) \ case code: \ EVENT_##name##_PARAMS(EVENT_PARAM_PRINT, name) \ break; - switch (code) { EVENT_LIST(EVENT_PRINT, X) } - } - - if (style == 'L') putchar(')'); - putchar('\n'); - fflush(stdout); + EVENT_LIST(EVENT_PRINT, X) } + + putchar('\n'); + fflush(stdout); EventDestroy(proc, event); } /* while(!feof(input)) */ } - /* logReader -- reader function for a file log */ static FILE *input; @@ -416,9 +215,14 @@ int main(int argc, char *argv[]) assert(CHECKCONV(ulongest_t, Addr)); assert(CHECKCONV(ulongest_t, void *)); assert(CHECKCONV(ulongest_t, EventCode)); - assert(CHECKCONV(Addr, void *)); /* for labelled pointers */ filename = parseArgs(argc, argv); + if (!filename) { + filename = getenv(TELEMETRY_FILENAME_ENVAR + ); + if(!filename) + filename = DEFAULT_TELEMETRY_FILENAME; + } if (strcmp(filename, "-") == 0) input = stdin; diff --git a/mps/code/eventsql.c b/mps/code/eventsql.c index d7cf6a3e3f5..b8ac5d6e4f5 100644 --- a/mps/code/eventsql.c +++ b/mps/code/eventsql.c @@ -88,6 +88,8 @@ #define DATABASE_NAME_ENVAR "MPS_EVENT_DATABASE" #define DEFAULT_DATABASE_NAME "mpsevent.db" +typedef sqlite3_int64 int64; + /* we output rows of dots. One dot per SMALL_TICK events, * BIG_TICK dots per row. */ @@ -136,8 +138,8 @@ static void error(const char *format, ...) static void sqlite_error(int res, sqlite3 *db, const char *format, ...) { - log(LOG_ALWAYS, "Fatal SQL error %d", res); va_list args; + log(LOG_ALWAYS, "Fatal SQL error %d", res); va_start(args, format); vlog(LOG_ALWAYS, format, args); va_end(args); @@ -335,7 +337,7 @@ static struct { static void testTableExists(sqlite3 *db) { - int i; + size_t i; int defects = 0; int tests = 0; for (i=0; i < (sizeof(tableTests)/sizeof(tableTests[0])); ++i) { @@ -366,7 +368,7 @@ static void testTableExists(sqlite3 *db) * assume that the user is smart enough not to do that. */ -static unsigned long logSerial = 0; +static int64 logSerial = 0; static void registerLogFile(sqlite3 *db, const char *filename) @@ -374,9 +376,9 @@ static void registerLogFile(sqlite3 *db, sqlite3_stmt *statement; int res; const unsigned char *name; - unsigned long completed; - unsigned long long file_size; - unsigned long long file_modtime; + int64 completed; + int64 file_size; + int64 file_modtime; if (filename) { struct stat st; @@ -402,8 +404,8 @@ static void registerLogFile(sqlite3 *db, break; case SQLITE_ROW: name = sqlite3_column_text(statement, 0); - logSerial = sqlite3_column_int(statement, 1); - completed = sqlite3_column_int(statement, 2); + logSerial = sqlite3_column_int64(statement, 1); + completed = sqlite3_column_int64(statement, 2); log(force ? LOG_OFTEN : LOG_ALWAYS, "Log file matching '%s' already in event_log, named \"%s\" (serial %lu, completed %lu).", filename, name, logSerial, completed); if (force) { @@ -444,7 +446,7 @@ static void registerLogFile(sqlite3 *db, } static void logFileCompleted(sqlite3 *db, - unsigned long completed) + int64 completed) { sqlite3_stmt *statement; int res; @@ -515,7 +517,7 @@ EVENT_LIST(EVENT_TABLE_CREATE, X) static void makeTables(sqlite3 *db) { - int i; + size_t i; log(LOG_SOMETIMES, "Creating tables."); for (i=0; i < (sizeof(createStatements)/sizeof(createStatements[0])); ++i) { @@ -531,7 +533,7 @@ const char *glueTables[] = { static void dropGlueTables(sqlite3 *db) { - int i; + size_t i; int res; char sql[1024]; @@ -689,18 +691,16 @@ static void fillGlueTables(sqlite3 *db) EVENT_##name##_PARAMS(EVENT_PARAM_BIND, X) \ break; -static char *bind_int(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +static char *bind_int(sqlite3 *db, sqlite3_stmt *stmt, int64 count, int index, char *p) { char *q; long long val; int res; - if ((p[0] != ',') || (p[1] != ' ')) - error("event %llu field %d not preceded by \", \": %s", - count, index, p); + while(*p == ' ') + ++p; - p += 2; - val = strtoll(p, &q, 0); + val = strtoll(p, &q, 16); if (q == p) error("event %llu field %d not an integer: %s", count, index, p); @@ -711,17 +711,15 @@ static char *bind_int(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, return q; } -static char *bind_real(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +static char *bind_real(sqlite3 *db, sqlite3_stmt *stmt, int64 count, int index, char *p) { char *q; double val; int res; - if ((p[0] != ',') || (p[1] != ' ')) - error("event %llu field %d not preceded by \", \": %s", - count, index, p); + while(*p == ' ') + ++p; - p += 2; val = strtod(p, &q); if (q == p) error("event %llu field %d not a floating-point value: %s", @@ -733,16 +731,14 @@ static char *bind_real(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count return q; } -static char *bind_text(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count, int index, char *p) +static char *bind_text(sqlite3 *db, sqlite3_stmt *stmt, int64 count, int index, char *p) { char *q; int res; - if ((p[0] != ',') || (p[1] != ' ') || (p[2] != '"')) - error("event %llu string field %d not preceded by \", \\\"\": %s", - count, index, p); + while(*p == ' ') + ++p; - p += 3; q = p; while((*q != '\n') && (*q != '\0')) { ++ q; @@ -751,7 +747,7 @@ static char *bind_text(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count error("event %llu string field %d has no closing quote mark.", count, index); - res = sqlite3_bind_text(stmt, index, p, q-p-1, SQLITE_STATIC); + res = sqlite3_bind_text(stmt, index, p, (int)(q-p-1), SQLITE_STATIC); if (res != SQLITE_OK) sqlite_error(res, db, "event %llu field %d bind failed", count, index); return q; @@ -760,10 +756,10 @@ static char *bind_text(sqlite3 *db, sqlite3_stmt *stmt, unsigned long long count /* readLog -- read and parse log. Returns the number of events written. */ -static unsigned long long readLog(FILE *input, - sqlite3 *db) +static int64 readLog(FILE *input, + sqlite3 *db) { - unsigned long long eventCount = 0; + int64 eventCount = 0; /* declare statements for every event type */ EVENT_LIST(EVENT_TYPE_DECLARE_STATEMENT, X); @@ -780,8 +776,8 @@ static unsigned long long readLog(FILE *input, int last_index=0; sqlite3_stmt *statement; int res; - unsigned long long clock; - int code; + int64 clock; + long code; p = fgets(line, 1024, input); if (!p) { @@ -798,13 +794,14 @@ static unsigned long long readLog(FILE *input, error("event %llu clock field not a hex integer: %s", eventCount, p); - if ((q[0] != ',') || (q[1] != ' ')) - error("event %llu code field not preceded by \", \": %s", + if (*q != ' ') + error("event %llu code field not preceded by ' ': %s", eventCount, q); + while(*q == ' ') + ++q; - p = q + 2; - - code = strtol(p, &q, 0); + p = q; + code = strtol(p, &q, 16); if (q == p) error("event %llu code field %d not an integer: %s", eventCount, index, p); @@ -874,10 +871,10 @@ static FILE *openLog(sqlite3 *db) return input; } -static unsigned long long writeEventsToSQL(sqlite3 *db) +static int64 writeEventsToSQL(sqlite3 *db) { FILE *input; - unsigned long long count; + int64 count; input = openLog(db); count = readLog(input, db); (void)fclose(input); @@ -888,7 +885,7 @@ static unsigned long long writeEventsToSQL(sqlite3 *db) int main(int argc, char *argv[]) { sqlite3 *db; - unsigned long long count; + int64 count; parseArgs(argc, argv); diff --git a/mps/code/mps.xcodeproj/project.pbxproj b/mps/code/mps.xcodeproj/project.pbxproj index 207e50660e4..0647bc91de2 100644 --- a/mps/code/mps.xcodeproj/project.pbxproj +++ b/mps/code/mps.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ buildPhases = ( ); dependencies = ( + 2D07B9791636FCBD00DB751B /* PBXTargetDependency */, 3104AFF6156D37BC000A585A /* PBXTargetDependency */, 3114A6D5156E9839001E0AA3 /* PBXTargetDependency */, 3114A6B9156E9763001E0AA3 /* PBXTargetDependency */, @@ -51,6 +52,8 @@ /* End PBXAggregateTarget section */ /* Begin PBXBuildFile section */ + 2D07B97A1636FCCE00DB751B /* eventsql.c in Sources */ = {isa = PBXBuildFile; fileRef = 2D07B96C1636FC7200DB751B /* eventsql.c */; }; + 2D07B97C163705E400DB751B /* libsqlite3.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 2D07B97B163705E400DB751B /* libsqlite3.dylib */; }; 3104AFBF156D3591000A585A /* apss.c in Sources */ = {isa = PBXBuildFile; fileRef = 3104AFBE156D3591000A585A /* apss.c */; }; 3104AFC2156D35B2000A585A /* libmps.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 31EEABFB156AAF9D00714D05 /* libmps.a */; }; 3104AFC3156D35C3000A585A /* testlib.c in Sources */ = {isa = PBXBuildFile; fileRef = 31EEAC9E156AB73400714D05 /* testlib.c */; }; @@ -196,6 +199,13 @@ /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ + 2D07B9781636FCBD00DB751B /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 31EEABDA156AAE9E00714D05 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 2D07B9701636FC9900DB751B; + remoteInfo = eventsql; + }; 3104AFC0156D35AE000A585A /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 31EEABDA156AAE9E00714D05 /* Project object */; @@ -626,6 +636,15 @@ /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ + 2D07B96F1636FC9900DB751B /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; 3104AFB1156D357B000A585A /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -899,6 +918,9 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 2D07B96C1636FC7200DB751B /* eventsql.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = eventsql.c; sourceTree = ""; }; + 2D07B9711636FC9900DB751B /* eventsql */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = eventsql; sourceTree = BUILT_PRODUCTS_DIR; }; + 2D07B97B163705E400DB751B /* libsqlite3.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libsqlite3.dylib; path = usr/lib/libsqlite3.dylib; sourceTree = SDKROOT; }; 3104AFA5156D27E7000A585A /* ssixi6.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ssixi6.c; sourceTree = ""; }; 3104AFB3156D357B000A585A /* apss */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = apss; sourceTree = BUILT_PRODUCTS_DIR; }; 3104AFBE156D3591000A585A /* apss.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = apss.c; sourceTree = ""; }; @@ -1026,6 +1048,14 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + 2D07B96E1636FC9900DB751B /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 2D07B97C163705E400DB751B /* libsqlite3.dylib in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 3104AFB0156D357B000A585A /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -1276,6 +1306,21 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 2D07B96A1636FC4C00DB751B /* eventsql */ = { + isa = PBXGroup; + children = ( + 2D07B96C1636FC7200DB751B /* eventsql.c */, + ); + name = eventsql; + sourceTree = ""; + }; + 2D07B9731636FC9900DB751B /* eventsql */ = { + isa = PBXGroup; + children = ( + ); + path = eventsql; + sourceTree = ""; + }; 3114A647156E956C001E0AA3 /* Mysterious */ = { isa = PBXGroup; children = ( @@ -1288,6 +1333,7 @@ 3114A6D6156E9846001E0AA3 /* Tools */ = { isa = PBXGroup; children = ( + 2D07B96A1636FC4C00DB751B /* eventsql */, 3114A6D8156E9942001E0AA3 /* eventcnv */, ); name = Tools; @@ -1374,10 +1420,12 @@ 31EEABD8156AAE9E00714D05 = { isa = PBXGroup; children = ( + 2D07B97B163705E400DB751B /* libsqlite3.dylib */, 3114A6D6156E9846001E0AA3 /* Tools */, 3114A647156E956C001E0AA3 /* Mysterious */, 31A47BA8156C1E930039B1C2 /* MPS */, 3124CAB3156BE1B700753214 /* Tests */, + 2D07B9731636FC9900DB751B /* eventsql */, 31EEABEF156AAF5C00714D05 /* Products */, ); sourceTree = ""; @@ -1416,6 +1464,7 @@ 3114A695156E971B001E0AA3 /* messtest */, 3114A6AC156E9759001E0AA3 /* walkt0 */, 3114A6C6156E9815001E0AA3 /* eventcnv */, + 2D07B9711636FC9900DB751B /* eventsql */, ); name = Products; sourceTree = ""; @@ -1515,6 +1564,23 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXNativeTarget section */ + 2D07B9701636FC9900DB751B /* eventsql */ = { + isa = PBXNativeTarget; + buildConfigurationList = 2D07B9741636FC9900DB751B /* Build configuration list for PBXNativeTarget "eventsql" */; + buildPhases = ( + 2D07B96D1636FC9900DB751B /* Sources */, + 2D07B96E1636FC9900DB751B /* Frameworks */, + 2D07B96F1636FC9900DB751B /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = eventsql; + productName = eventsql; + productReference = 2D07B9711636FC9900DB751B /* eventsql */; + productType = "com.apple.product-type.tool"; + }; 3104AFB2156D357B000A585A /* apss */ = { isa = PBXNativeTarget; buildConfigurationList = 3104AFBC156D357B000A585A /* Build configuration list for PBXNativeTarget "apss" */; @@ -2124,11 +2190,20 @@ 3114A694156E971B001E0AA3 /* messtest */, 3114A6AB156E9759001E0AA3 /* walkt0 */, 3114A6C5156E9815001E0AA3 /* eventcnv */, + 2D07B9701636FC9900DB751B /* eventsql */, ); }; /* End PBXProject section */ /* Begin PBXSourcesBuildPhase section */ + 2D07B96D1636FC9900DB751B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 2D07B97A1636FCCE00DB751B /* eventsql.c in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 3104AFAF156D357B000A585A /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -2461,6 +2536,11 @@ /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ + 2D07B9791636FCBD00DB751B /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 2D07B9701636FC9900DB751B /* eventsql */; + targetProxy = 2D07B9781636FCBD00DB751B /* PBXContainerItemProxy */; + }; 3104AFC1156D35AE000A585A /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 31EEABFA156AAF9D00714D05 /* mps */; @@ -2769,6 +2849,30 @@ /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + 2D07B9751636FC9900DB751B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_C_LANGUAGE_STANDARD = c99; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 2D07B9761636FC9900DB751B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_C_LANGUAGE_STANDARD = c99; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; + 2D07B9771636FC9900DB751B /* WE */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_C_LANGUAGE_STANDARD = c99; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = WE; + }; 3104AFBA156D357B000A585A /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -3616,6 +3720,15 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + 2D07B9741636FC9900DB751B /* Build configuration list for PBXNativeTarget "eventsql" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2D07B9751636FC9900DB751B /* Debug */, + 2D07B9761636FC9900DB751B /* Release */, + 2D07B9771636FC9900DB751B /* WE */, + ); + defaultConfigurationIsVisible = 0; + }; 3104AFBC156D357B000A585A /* Build configuration list for PBXNativeTarget "apss" */ = { isa = XCConfigurationList; buildConfigurations = ( From 20ad19da97d50ab8f8d95ecd1029dcf5f9583792 Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Fri, 26 Oct 2012 13:03:44 +0100 Subject: [PATCH 13/14] Make mps_telemetry_control case-insensitive, and make it accept "all" as an event class. Copied from Perforce Change: 180098 ServerID: perforce.ravenbrook.com --- mps/code/mpsliban.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/mps/code/mpsliban.c b/mps/code/mpsliban.c index 8abab586319..f966df6f4f2 100644 --- a/mps/code/mpsliban.c +++ b/mps/code/mpsliban.c @@ -36,6 +36,7 @@ #include #include #include +#include int mps_lib_get_EOF(void) @@ -119,7 +120,9 @@ unsigned long mps_lib_telemetry_control(void) unsigned long mask; char buf[256]; char *word; + char *p; char *sep = " "; + char rowName[256]; s = getenv("MPS_TELEMETRY_CONTROL"); if (s == NULL) @@ -130,14 +133,29 @@ unsigned long mps_lib_telemetry_control(void) if (mask != 0) return mask; - /* Split the value at spaces and try to patch the words against the names - of event kinds, enabling them if there's a match. */ + /* copy the envar to a buffer so we can mess with it. */ strncpy(buf, s, sizeof(buf) - 1); buf[sizeof(buf) - 1] = '\0'; + /* downcase it */ + for (p = buf; *p != '\0'; ++p) + *p = (char)tolower(*p); + + /* Split the value at spaces and try to match the words against the names + of event kinds, enabling them if there's a match. */ for (word = strtok(buf, sep); word != NULL; word = strtok(NULL, sep)) { -#define TELEMATCH(X, rowName, rowDoc) \ - if (strcmp(word, #rowName) == 0) \ - mask |= (1ul << EventKind##rowName); + if (strcmp(word, "all") == 0) { + mask = (unsigned long)-1; + printf("All events."); + return mask; + } +#define TELEMATCH(X, name, rowDoc) \ + strncpy(rowName, #name, sizeof(rowName) - 1); \ + rowName[sizeof(rowName) - 1] = '\0'; \ + for (p = rowName; *p != '\0'; ++p) \ + *p = (char)tolower(*p); \ + if (strcmp(word, rowName) == 0) { \ + mask |= (1ul << EventKind##name); \ + printf("Events to include " rowDoc "\n"); } EventKindENUM(TELEMATCH, X) } From d48d96f15ec9f10f468e5c4a1a93f78f7ce7ad5b Mon Sep 17 00:00:00 2001 From: Nick Barnes Date: Fri, 26 Oct 2012 13:22:04 +0100 Subject: [PATCH 14/14] Add an eventinit event, giving various parameters including the event system version. Copied from Perforce Change: 180100 ServerID: perforce.ravenbrook.com --- mps/code/event.c | 2 ++ mps/code/eventdef.h | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mps/code/event.c b/mps/code/event.c index a3331d18c18..f88600df7e7 100644 --- a/mps/code/event.c +++ b/mps/code/event.c @@ -178,6 +178,8 @@ void EventInit(void) EventKindControl = (Word)mps_lib_telemetry_control(); EventInternSerial = (Serial)1; /* 0 is reserved */ (void)EventInternString(MPSVersion()); /* emit version */ + EVENT6(EventInit, EVENT_VERSION_MAJOR, EVENT_VERSION_MEDIAN, + EVENT_VERSION_MINOR, EventCodeMAX, EventNameMAX, MPS_WORD_WIDTH); } else { ++eventUserCount; } diff --git a/mps/code/eventdef.h b/mps/code/eventdef.h index df80bb6d1c2..456d230d8f8 100644 --- a/mps/code/eventdef.h +++ b/mps/code/eventdef.h @@ -38,7 +38,7 @@ #define EVENT_VERSION_MAJOR ((unsigned)1) #define EVENT_VERSION_MEDIAN ((unsigned)0) -#define EVENT_VERSION_MINOR ((unsigned)0) +#define EVENT_VERSION_MINOR ((unsigned)1) /* EVENT_LIST -- list of event types and general properties @@ -68,7 +68,7 @@ */ #define EventNameMAX ((size_t)19) -#define EventCodeMAX ((EventCode)0x0073) +#define EventCodeMAX ((EventCode)0x0074) #define EVENT_LIST(EVENT, X) \ /* 0123456789012345678 <- don't exceed without changing EventNameMAX */ \ @@ -179,7 +179,8 @@ EVENT(X, TraceFindGrey , 0x0070, TRUE, Trace) \ EVENT(X, TraceBandAdvance , 0x0071, TRUE, Trace) \ EVENT(X, AWLDeclineTotal , 0x0072, TRUE, Trace) \ - EVENT(X, AWLDeclineSeg , 0x0073, TRUE, Trace) + EVENT(X, AWLDeclineSeg , 0x0073, TRUE, Trace) \ + EVENT(X, EventInit , 0x0074, TRUE, Arena) /* Remember to update EventNameMAX and EventCodeMAX in eventcom.h! @@ -620,6 +621,14 @@ PARAM(X, 0, P, seg) /* segment declined single access */ \ PARAM(X, 1, W, singleAccesses) /* single accesses this cycle */ +#define EVENT_EventInit_PARAMS(PARAM, X) \ + PARAM(X, 0, U, major) /* EVENT_VERSION_MAJOR */ \ + PARAM(X, 1, U, median) /* EVENT_VERSION_MEDIAN */ \ + PARAM(X, 2, U, minor) /* EVENT_VERSION_MINOR */ \ + PARAM(X, 3, U, maxCode) /* EventCodeMAX */ \ + PARAM(X, 4, U, maxNameLen) /* EventNameMAX */ \ + PARAM(X, 5, U, wordWidth) /* MPS_WORD_WIDTH */ + #endif /* eventdef_h */