diff options
| -rw-r--r-- | SConscript | 1 | ||||
| -rw-r--r-- | base/misc.cc | 5 | ||||
| -rw-r--r-- | base/output.cc | 129 | ||||
| -rw-r--r-- | base/output.hh | 61 | ||||
| -rw-r--r-- | base/str.cc | 30 | ||||
| -rw-r--r-- | base/str.hh | 14 | ||||
| -rw-r--r-- | cpu/base_cpu.cc | 9 | ||||
| -rw-r--r-- | dev/etherdump.cc | 19 | ||||
| -rw-r--r-- | dev/etherdump.hh | 4 | ||||
| -rw-r--r-- | dev/simconsole.cc | 21 | ||||
| -rw-r--r-- | objects/Root.mpy | 10 | ||||
| -rw-r--r-- | sim/main.cc | 263 | ||||
| -rw-r--r-- | sim/param.cc | 5 | ||||
| -rw-r--r-- | sim/pyconfig/SConscript | 2 | ||||
| -rw-r--r-- | sim/pyconfig/m5config.py | 16 | ||||
| -rw-r--r-- | sim/serialize.cc | 7 | ||||
| -rw-r--r-- | sim/universe.cc | 98 | ||||
| -rw-r--r-- | test/genini.py | 25 | ||||
| -rwxr-xr-x | util/pbs/job.py | 183 | ||||
| -rw-r--r-- | util/pbs/jobfile.py | 83 | ||||
| -rwxr-xr-x | util/pbs/pbs.py | 176 | ||||
| -rwxr-xr-x | util/pbs/send.py | 190 |
22 files changed, 1071 insertions, 280 deletions
diff --git a/SConscript b/SConscript index 54f16fce6..4e4cb8727 100644 --- a/SConscript +++ b/SConscript @@ -64,6 +64,7 @@ base_sources = Split(''' base/intmath.cc base/match.cc base/misc.cc + base/output.cc base/pollevent.cc base/python.cc base/range.cc diff --git a/base/misc.cc b/base/misc.cc index 0c459352f..4b7c3632a 100644 --- a/base/misc.cc +++ b/base/misc.cc @@ -30,10 +30,11 @@ #include <string> #include "base/cprintf.hh" -#include "sim/host.hh" #include "base/hostinfo.hh" #include "base/misc.hh" +#include "base/output.hh" #include "base/trace.hh" +#include "sim/host.hh" #include "sim/universe.hh" using namespace std; @@ -116,7 +117,7 @@ __warn(const string &format, cp::ArgList &args, const char *func, #endif args.dump(cerr, fmt); - if (outputStream != &cerr && outputStream != &cout) + if (simout.isFile(*outputStream)) args.dump(*outputStream, fmt); delete &args; diff --git a/base/output.cc b/base/output.cc new file mode 100644 index 000000000..2b1733f21 --- /dev/null +++ b/base/output.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2005 The Regents of The University of Michigan + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <errno.h> +#include <limits.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include <fstream> + +#include "base/misc.hh" +#include "base/output.hh" + +using namespace std; + +OutputDirectory simout; + +/** + * + */ +OutputDirectory::OutputDirectory() +{} + +OutputDirectory::~OutputDirectory() +{} + +void +OutputDirectory::setDirectory(const string &d) +{ + if (!dir.empty()) + panic("Output directory already set!\n"); + + dir = d; + + if (dir != ".") { + if (mkdir(dir.c_str(), 0777) < 0 && errno != EEXIST) + panic("couldn't make output dir %s: %s\n", + dir, strerror(errno)); + } + + // guarantee that directory ends with a '/' + if (dir[dir.size() - 1] != '/') + dir += "/"; +} + +const string & +OutputDirectory::directory() +{ + if (dir.empty()) + panic("Output directory not set!"); + + return dir; +} + +string +OutputDirectory::resolve(const string &name) +{ + return (name[0] != '/') ? dir + name : name; +} + +ostream * +OutputDirectory::create(const string &name) +{ + if (name == "cerr" || name == "stderr") + return &cerr; + + if (name == "cout" || name == "stdout") + return &cout; + + ofstream *file = new ofstream(resolve(name).c_str(), ios::trunc); + if (!file->is_open()) + panic("Cannot open file %s", name); + + return file; +} + +ostream * +OutputDirectory::find(const string &name) +{ + if (name == "cerr" || name == "stderr") + return &cerr; + + if (name == "cout" || name == "stdout") + return &cout; + + string filename = resolve(name); + map_t::iterator i = files.find(filename); + if (i != files.end()) + return (*i).second; + + ofstream *file = new ofstream(filename.c_str(), ios::trunc); + if (!file->is_open()) + panic("Cannot open file %s", filename); + + files[filename] = file; + return file; +} + +bool +OutputDirectory::isFile(const std::ostream *os) +{ + return os && os != &cerr && os != &cout; +} diff --git a/base/output.hh b/base/output.hh new file mode 100644 index 000000000..3bbe73e3b --- /dev/null +++ b/base/output.hh @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2005 The Regents of The University of Michigan + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __BASE_OUTPUT_HH__ +#define __BASE_OUTPUT_HH__ + +#include <iosfwd> +#include <map> +#include <string> + +class OutputDirectory +{ + private: + typedef std::map<std::string, std::ostream *> map_t; + + map_t files; + std::string dir; + + public: + OutputDirectory(); + ~OutputDirectory(); + + void setDirectory(const std::string &dir); + const std::string &directory(); + + std::string resolve(const std::string &name); + std::ostream *create(const std::string &name); + std::ostream *find(const std::string &name); + + static bool isFile(const std::ostream *os); + static inline bool isFile(const std::ostream &os) { return isFile(&os); } +}; + +extern OutputDirectory simout; + +#endif // __BASE_OUTPUT_HH__ diff --git a/base/str.cc b/base/str.cc index dd8d80043..5357ba79f 100644 --- a/base/str.cc +++ b/base/str.cc @@ -39,6 +39,36 @@ using namespace std; +bool +split_first(const string &s, string &lhs, string &rhs, char c) +{ + string::size_type offset = s.find(c); + if (offset == string::npos) { + lhs = s; + rhs = ""; + return false; + } + + lhs = s.substr(0, offset); + rhs = s.substr(offset + 1); + return true; +} + +bool +split_last(const string &s, string &lhs, string &rhs, char c) +{ + string::size_type offset = s.rfind(c); + if (offset == string::npos) { + lhs = s; + rhs = ""; + return false; + } + + lhs = s.substr(0, offset); + rhs = s.substr(offset + 1); + return true; +} + void tokenize(vector<string>& v, const string &s, char token, bool ignore) { diff --git a/base/str.hh b/base/str.hh index 812f4d41a..41433f2bd 100644 --- a/base/str.hh +++ b/base/str.hh @@ -90,6 +90,20 @@ to_lower(const std::string &s) return lower; } +// Split the string s into lhs and rhs on the first occurence of the +// character c. +bool +split_first(const std::string &s, std::string &lhs, std::string &rhs, char c); + +// Split the string s into lhs and rhs on the last occurence of the +// character c. +bool +split_last(const std::string &s, std::string &lhs, std::string &rhs, char c); + +// Tokenize the string <s> splitting on the character <token>, and +// place the result in the string vector <vector>. If <ign> is true, +// then empty result strings (due to trailing tokens, or consecutive +// tokens) are skipped. void tokenize(std::vector<std::string> &vector, const std::string &s, char token, bool ign = true); diff --git a/cpu/base_cpu.cc b/cpu/base_cpu.cc index 19261e97b..d653baa29 100644 --- a/cpu/base_cpu.cc +++ b/cpu/base_cpu.cc @@ -26,13 +26,14 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <iostream> #include <string> #include <sstream> -#include <iostream> #include "base/cprintf.hh" #include "base/loader/symtab.hh" #include "base/misc.hh" +#include "base/output.hh" #include "cpu/base_cpu.hh" #include "cpu/exec_context.hh" #include "sim/param.hh" @@ -142,8 +143,7 @@ BaseCPU::BaseCPU(const string &_name, int _number_of_threads, bool _def_reg, functionTracingEnabled = false; if (_function_trace) { - std::string filename = csprintf("ftrace.%s", name()); - functionTraceStream = makeOutputStream(filename); + functionTraceStream = simout.find(csprintf("ftrace.%s", name())); currentFunctionStart = currentFunctionEnd = 0; functionEntryTick = _function_trace_start; @@ -167,11 +167,8 @@ BaseCPU::enableFunctionTrace() BaseCPU::~BaseCPU() { - if (functionTracingEnabled) - closeOutputStream(functionTraceStream); } - void BaseCPU::init() { diff --git a/dev/etherdump.cc b/dev/etherdump.cc index 485d5599c..3de417bdc 100644 --- a/dev/etherdump.cc +++ b/dev/etherdump.cc @@ -36,14 +36,15 @@ #include <string> #include "base/misc.hh" +#include "base/output.hh" #include "dev/etherdump.hh" #include "sim/builder.hh" #include "sim/universe.hh" using std::string; -EtherDump::EtherDump(const string &name, std::ostream *_stream, int max) - : SimObject(name), stream(_stream), maxlen(max) +EtherDump::EtherDump(const string &name, const string &file, int max) + : SimObject(name), stream(file.c_str()), maxlen(max) { } @@ -86,7 +87,7 @@ EtherDump::init() hdr.sigfigs = 0; hdr.linktype = DLT_EN10MB; - stream->write(reinterpret_cast<char *>(&hdr), sizeof(hdr)); + stream.write(reinterpret_cast<char *>(&hdr), sizeof(hdr)); /* * output an empty packet with the current time so that we know @@ -98,9 +99,9 @@ EtherDump::init() pkthdr.microseconds = 0; pkthdr.caplen = 0; pkthdr.len = 0; - stream->write(reinterpret_cast<char *>(&pkthdr), sizeof(pkthdr)); + stream.write(reinterpret_cast<char *>(&pkthdr), sizeof(pkthdr)); - stream->flush(); + stream.flush(); } void @@ -111,9 +112,9 @@ EtherDump::dumpPacket(PacketPtr &packet) pkthdr.microseconds = (curTick / us_freq) % ULL(1000000); pkthdr.caplen = std::min(packet->length, maxlen); pkthdr.len = packet->length; - stream->write(reinterpret_cast<char *>(&pkthdr), sizeof(pkthdr)); - stream->write(reinterpret_cast<char *>(packet->data), pkthdr.caplen); - stream->flush(); + stream.write(reinterpret_cast<char *>(&pkthdr), sizeof(pkthdr)); + stream.write(reinterpret_cast<char *>(packet->data), pkthdr.caplen); + stream.flush(); } BEGIN_DECLARE_SIM_OBJECT_PARAMS(EtherDump) @@ -132,7 +133,7 @@ END_INIT_SIM_OBJECT_PARAMS(EtherDump) CREATE_SIM_OBJECT(EtherDump) { - return new EtherDump(getInstanceName(), makeOutputStream(file), maxlen); + return new EtherDump(getInstanceName(), simout.resolve(file), maxlen); } REGISTER_SIM_OBJECT("EtherDump", EtherDump) diff --git a/dev/etherdump.hh b/dev/etherdump.hh index b127d05e2..ba15796c8 100644 --- a/dev/etherdump.hh +++ b/dev/etherdump.hh @@ -43,7 +43,7 @@ class EtherDump : public SimObject { private: - std::ostream *stream; + std::ofstream stream; const int maxlen; void dumpPacket(PacketPtr &packet); void init(); @@ -53,7 +53,7 @@ class EtherDump : public SimObject Tick us_freq; public: - EtherDump(const std::string &name, std::ostream *_stream, int max); + EtherDump(const std::string &name, const std::string &file, int max); inline void dump(PacketPtr &pkt) { dumpPacket(pkt); } }; diff --git a/dev/simconsole.cc b/dev/simconsole.cc index 48e5d0201..94fd9ec1f 100644 --- a/dev/simconsole.cc +++ b/dev/simconsole.cc @@ -43,6 +43,7 @@ #include <string> #include "base/misc.hh" +#include "base/output.hh" #include "base/socket.hh" #include "base/trace.hh" #include "dev/platform.hh" @@ -71,7 +72,7 @@ SimConsole::Event::process(int revent) cons->detach(); } -SimConsole::SimConsole(const string &name, std::ostream *os, int num) +SimConsole::SimConsole(const string &name, ostream *os, int num) : SimObject(name), event(NULL), number(num), in_fd(-1), out_fd(-1), listener(NULL), txbuf(16384), rxbuf(16384), outfile(os) #if TRACING_ON == 1 @@ -85,8 +86,6 @@ SimConsole::SimConsole(const string &name, std::ostream *os, int num) SimConsole::~SimConsole() { close(); - if (outfile) - closeOutputStream(outfile); } void @@ -313,18 +312,16 @@ END_INIT_SIM_OBJECT_PARAMS(SimConsole) CREATE_SIM_OBJECT(SimConsole) { - string filename; + string filename = output; + ostream *stream = NULL; - if (filename.empty()) { - filename = getInstanceName(); - } else if (append_name) { - filename = (string)output + "." + getInstanceName(); - } else { - filename = output; + if (!filename.empty()) { + if (append_name) + filename += "." + getInstanceName(); + stream = simout.find(filename); } - SimConsole *console = new SimConsole(getInstanceName(), - makeOutputStream(filename), number); + SimConsole *console = new SimConsole(getInstanceName(), stream, number); ((ConsoleListener *)listener)->add(console); return console; diff --git a/objects/Root.mpy b/objects/Root.mpy index b21396e36..0e531054b 100644 --- a/objects/Root.mpy +++ b/objects/Root.mpy @@ -1,11 +1,15 @@ from HierParams import HierParams +from Serialize import Serialize +from Statistics import Statistics +from Trace import Trace + simobj Root(SimObject): type = 'Root' frequency = Param.Tick(200000000, "tick frequency") - output_dir = Param.String('.', "directory to output data to") output_file = Param.String('cout', "file to dump simulator output to") - config_output_file = Param.String('m5config.out', - "file to dump simulator config to") full_system = Param.Bool("Full system simulation?") hier = HierParams(do_data = False, do_events = True) checkpoint = Param.String('', "Checkpoint file") + stats = Statistics() + trace = Trace() + serialize = Serialize() diff --git a/sim/main.cc b/sim/main.cc index 163c835ee..1748294af 100644 --- a/sim/main.cc +++ b/sim/main.cc @@ -42,6 +42,7 @@ #include "base/embedfile.hh" #include "base/inifile.hh" #include "base/misc.hh" +#include "base/output.hh" #include "base/pollevent.hh" #include "base/statistics.hh" #include "base/str.hh" @@ -109,25 +110,32 @@ abortHandler(int sigtype) const char *myProgName = ""; /// Show brief help message. -static void +void showBriefHelp(ostream &out) { - out << "Usage: " << myProgName - << " [-hnu] [-Dname[=def]] [-Uname] [-I[dir]] " - << "<config-spec> [<config-spec> ...]\n" - << "[] [<config file> ...]\n" - << " -h: print long help (including parameter listing)\n" - << " -u: don't quit on unreferenced parameters\n" - << " -D,-U,-I: passed to cpp for preprocessing .ini files\n" - << " <config-spec>: config file name (.ini or .py) or\n" - << " single param (--<section>:<param>=<value>)" - << endl; + char *prog = basename(myProgName); + + ccprintf(out, "Usage:\n"); + ccprintf(out, +"%s [-d <dir>] [-E <var>[=<val>]] [-I <dir>] [-P <python>]\n" +" [--<var>=<val>] <config file>\n" +"\n" +" -d set the output directory to <dir>\n" +" -E set the environment variable <var> to <val> (or 'True')\n" +" -I add the directory <dir> to python's path\n" +" -P execute <python> directly in the configuration\n" +" --var=val set the python variable <var> to '<val>'\n" +" <configfile> config file name (.py or .mpy)\n", + prog); + + ccprintf(out, "%s -X\n -X extract embedded files\n", prog); + ccprintf(out, "%s -h\n -h print long help\n", prog); } /// Show verbose help message. Includes parameter listing from /// showBriefHelp(), plus an exhaustive list of ini-file parameters /// and SimObjects (with their parameters). -static void +void showLongHelp(ostream &out) { showBriefHelp(out); @@ -152,7 +160,7 @@ showLongHelp(ostream &out) } /// Print welcome message. -static void +void sayHello(ostream &out) { extern const char *compileDate; // from date.cc @@ -176,7 +184,7 @@ sayHello(ostream &out) /// Echo the command line for posterity in such a way that it can be /// used to rerun the same simulation (given the same .ini files). /// -static void +void echoCommandLine(int argc, char **argv, ostream &out) { out << "command line: " << argv[0]; @@ -208,16 +216,20 @@ echoCommandLine(int argc, char **argv, ostream &out) out << endl << endl; } +char * +getOptionString(int &index, int argc, char **argv) +{ + char *option = argv[index] + 2; + if (*option != '\0') + return option; -/// -/// The simulator configuration database. This is the union of all -/// specified .ini files. This shouldn't need to be visible outside -/// this file, as it is passed as a parameter to all the param-parsing -/// routines. -/// -static IniFile simConfigDB; + // We didn't find an argument, it must be in the next variable. + if (++index >= argc) + panic("option string for option '%s' not found", argv[index - 1]); + + return argv[index]; +} -/// M5 entry point. int main(int argc, char **argv) { @@ -233,18 +245,9 @@ main(int argc, char **argv) sayHello(cerr); - // Initialize statistics database - Stats::InitSimStats(); - - vector<char *> cppArgs; - - // Should we quit if there are unreferenced parameters? By - // default, yes... it's a good way of catching typos in - // section/parameter names (which otherwise go by silently). Use - // -u to override. - bool quitOnUnreferenced = true; - - bool python_initialized = false; + bool configfile_found = false; + PythonConfig pyconfig; + string outdir; // Parse command-line options. // Since most of the complex options are handled through the @@ -253,12 +256,53 @@ main(int argc, char **argv) for (int i = 1; i < argc; ++i) { char *arg_str = argv[i]; - // if arg starts with '-', parse as option, - // else treat it as a configuration file name and load it - if (arg_str[0] == '-') { + // if arg starts with '--', parse as a special python option + // of the format --<python var>=<string value>, if the arg + // starts with '-', it should be a simulator option with a + // format similar to getopt. In any other case, treat the + // option as a configuration file name and load it. + if (arg_str[0] == '-' && arg_str[1] == '-') { + string str = &arg_str[2]; + string var, val; + + if (!split_first(str, var, val, '=')) + panic("Could not parse configuration argument '%s'\n" + "Expecting --<variable>=<value>\n", arg_str); + + pyconfig.setVariable(var, val); + } else if (arg_str[0] == '-') { + char *option; + string var, val; // switch on second char switch (arg_str[1]) { + case 'd': + outdir = getOptionString(i, argc, argv); + break; + + case 'h': + showLongHelp(cerr); + exit(1); + + case 'E': + option = getOptionString(i, argc, argv); + if (!split_first(option, var, val, '=')) + val = "True"; + + if (setenv(var.c_str(), val.c_str(), true) == -1) + panic("setenv: %s\n", strerror(errno)); + break; + + case 'I': + option = getOptionString(i, argc, argv); + pyconfig.addPath(option); + break; + + case 'P': + option = getOptionString(i, argc, argv); + pyconfig.writeLine(option); + break; + case 'X': { list<EmbedFile> lst; EmbedMap::all(lst); @@ -274,124 +318,58 @@ main(int argc, char **argv) return 0; } - case 'h': - // -h: show help - showLongHelp(cerr); - exit(1); - - case 'u': - // -u: don't quit on unreferenced parameters - quitOnUnreferenced = false; - break; - - case 'D': - case 'U': - // cpp options: record & pass to cpp. Note that these - // cannot have spaces, i.e., '-Dname=val' is OK, but - // '-D name=val' is not. I don't consider this a - // problem, since even though gnu cpp accepts the - // latter, other cpp implementations do not (Tru64, - // for one). - cppArgs.push_back(arg_str); - break; - - case 'I': { - // We push -I as an argument to cpp - cppArgs.push_back(arg_str); - - string arg = arg_str + 2; - eat_white(arg); - - // Send this as the python path - addPythonPath(arg); - } break; - - case 'P': - if (!python_initialized) { - initPythonConfig(); - python_initialized = true; - } - writePythonString(arg_str + 2); - writePythonString("\n"); - - case 'E': - if (putenv(arg_str + 2) == -1) - panic("putenv: %s\n", strerror(errno)); - break; - - case '-': - // command-line configuration parameter: - // '--<section>:<parameter>=<value>' - if (!simConfigDB.add(arg_str + 2)) { - // parse error - ccprintf(cerr, - "Could not parse configuration argument '%s'\n" - "Expecting --<section>:<parameter>=<value>\n", - arg_str); - exit(0); - } - break; - default: showBriefHelp(cerr); - ccprintf(cerr, "Fatal: invalid argument '%s'\n", arg_str); - exit(0); - } - } - else { - // no '-', treat as config file name - - // make STL string out of file name - string filename(arg_str); - - int ext_loc = filename.rfind("."); - - string ext = - (ext_loc != string::npos) ? filename.substr(ext_loc) : ""; - - if (ext == ".ini") { - if (!simConfigDB.loadCPP(filename, cppArgs)) { - cprintf("Error processing file %s\n", filename); - exit(1); - } - } else if (ext == ".py" || ext == ".mpy") { - if (!python_initialized) { - initPythonConfig(); - python_initialized = true; - } - loadPythonConfig(filename); - } - else { - cprintf("Config file name '%s' must end in '.py' or '.ini'.\n", - filename); - exit(1); + panic("invalid argument '%s'\n", arg_str); } + } else { + string file(arg_str); + string base, ext; + + if (!split_last(file, base, ext, '.') || + ext != "py" && ext != "mpy") + panic("Config file '%s' must end in '.py' or '.mpy'\n", file); + + pyconfig.load(file); + configfile_found = true; } } - if (python_initialized && !finishPythonConfig(simConfigDB)) { - cprintf("Error processing python code\n"); - exit(1); + if (outdir.empty()) { + char *env = getenv("OUTPUT_DIR"); + outdir = env ? env : "."; } - // The configuration database is now complete; start processing it. + simout.setDirectory(outdir); - // Parse and check all non-config-hierarchy parameters. - ParamContext::parseAllContexts(simConfigDB); - ParamContext::checkAllContexts(); + char *env = getenv("CONFIG_OUTPUT"); + if (!env) + env = "config.out"; + configStream = simout.find(env); - // Print header info into stats file. Can't do this sooner since - // the stat file name is set via a .ini param... thus it just got - // opened above during ParamContext::checkAllContexts(). + if (!configfile_found) + panic("no configuration file specified!"); + + // The configuration database is now complete; start processing it. + IniFile inifile; + if (!pyconfig.output(inifile)) + panic("Error processing python code"); + + // Initialize statistics database + Stats::InitSimStats(); // Now process the configuration hierarchy and create the SimObjects. - ConfigHierarchy configHierarchy(simConfigDB); + ConfigHierarchy configHierarchy(inifile); configHierarchy.build(); configHierarchy.createSimObjects(); + // Parse and check all non-config-hierarchy parameters. + ParamContext::parseAllContexts(inifile); + ParamContext::checkAllContexts(); + // Print hello message to stats file if it's actually a file. If // it's not (i.e. it's cout or cerr) then we already did it above. - if (outputStream != &cout && outputStream != &cerr) + if (simout.isFile(*outputStream)) sayHello(*outputStream); // Echo command line and all parameter settings to stats file as well. @@ -406,13 +384,8 @@ main(int argc, char **argv) // Done processing the configuration database. // Check for unreferenced entries. - if (simConfigDB.printUnreferenced() && quitOnUnreferenced) { - cerr << "Fatal: unreferenced .ini sections/entries." << endl - << "If this is not an error, add 'unref_section_ok=y' or " - << "'unref_entries_ok=y' to the appropriate sections " - << "to suppress this message." << endl; - exit(1); - } + if (inifile.printUnreferenced()) + panic("unreferenced sections/entries in the intermediate ini file"); SimObject::regAllStats(); diff --git a/sim/param.cc b/sim/param.cc index d16578a2d..ff023ce85 100644 --- a/sim/param.cc +++ b/sim/param.cc @@ -441,6 +441,11 @@ EnumVectorParam<Map>::parse(const string &s) { vector<string> tokens; + if (s.empty()) { + wasSet = true; + return; + } + tokenize(tokens, s, ' '); value.resize(tokens.size()); diff --git a/sim/pyconfig/SConscript b/sim/pyconfig/SConscript index 5708ac9a8..9154d3b99 100644 --- a/sim/pyconfig/SConscript +++ b/sim/pyconfig/SConscript @@ -170,7 +170,7 @@ EmbedMap %(name)s("%(fname)s", /* namespace */ } ''' -embedded_py_files = ['m5config.py'] +embedded_py_files = ['m5config.py', '../../util/pbs/jobfile.py'] objpath = os.path.join(env['SRCDIR'], 'objects') for root, dirs, files in os.walk(objpath, topdown=True): for i,dir in enumerate(dirs): diff --git a/sim/pyconfig/m5config.py b/sim/pyconfig/m5config.py index 4e3a4103c..9a48e2fa4 100644 --- a/sim/pyconfig/m5config.py +++ b/sim/pyconfig/m5config.py @@ -28,19 +28,20 @@ from __future__ import generators import os, re, sys, types noDot = False try: - import pydot + import pydot except: - noDot = True + noDot = True env = {} env.update(os.environ) def panic(*args, **kwargs): - sys.exit(*args, **kwargs) + print >>sys.stderr, 'panic:', string + sys.exit(1) def AddToPath(path): path = os.path.realpath(path) - if os.path.isdir(path): + if os.path.isdir(path) and path not in sys.path: sys.path.append(path) def Import(path): @@ -349,6 +350,13 @@ class MetaConfigNode(type): for p,v in c._values.iteritems(): if not values.has_key(p): values[p] = v + for p,v in c._params.iteritems(): + if not values.has_key(p) and hasattr(v, 'default'): + v.valid(v.default) + v = v.default + cls._setvalue(p, v) + values[p] = v + return values def _getvalue(cls, name, default = AttributeError): diff --git a/sim/serialize.cc b/sim/serialize.cc index 3a073f68d..846d191e0 100644 --- a/sim/serialize.cc +++ b/sim/serialize.cc @@ -38,6 +38,7 @@ #include "base/inifile.hh" #include "base/misc.hh" +#include "base/output.hh" #include "base/str.hh" #include "base/trace.hh" #include "sim/config_node.hh" @@ -333,11 +334,7 @@ SerializeParamContext::~SerializeParamContext() void SerializeParamContext::checkParams() { - if (serialize_dir.isValid()) { - checkpointDirBase = serialize_dir; - } else { - checkpointDirBase = outputDirectory + "cpt.%012d"; - } + checkpointDirBase = simout.resolve(serialize_dir); // guarantee that directory ends with a '/' if (checkpointDirBase[checkpointDirBase.size() - 1] != '/') diff --git a/sim/universe.cc b/sim/universe.cc index 115f6f790..9137baaf0 100644 --- a/sim/universe.cc +++ b/sim/universe.cc @@ -26,10 +26,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <sys/types.h> -#include <sys/stat.h> -#include <errno.h> - #include <cstring> #include <fstream> #include <list> @@ -37,6 +33,7 @@ #include <vector> #include "base/misc.hh" +#include "base/output.hh" #include "sim/builder.hh" #include "sim/host.hh" #include "sim/sim_object.hh" @@ -51,7 +48,7 @@ double __ticksPerUS; double __ticksPerNS; double __ticksPerPS; -string outputDirectory; +bool fullSystem; ostream *outputStream; ostream *configStream; @@ -61,81 +58,40 @@ class Root : public SimObject public: Root(const std::string &name) : SimObject(name) {} }; -Root *root = NULL; - -std::ostream * -makeOutputStream(std::string &name) -{ - if (name == "cerr" || name == "stderr") - return &std::cerr; - - if (name == "cout" || name == "stdout") - return &std::cout; - - string path = (name[0] != '/') ? outputDirectory + name : name; - - // have to dynamically allocate a stream since we're going to - // return it... though the caller can't easily free it since it - // may be cerr or cout. need GC! - ofstream *s = new ofstream(path.c_str(), ios::trunc); - - if (!s->is_open()) - fatal("Cannot open file %s", path); - - return s; -} - - -void -closeOutputStream(std::ostream *os) -{ - // can't close cerr or cout - if (os == &std::cerr || os == &std::cout) - return; - - // can only close ofstreams, not generic ostreams, so try to - // downcast and close only if the downcast succeeds - std::ofstream *ofs = dynamic_cast<std::ofstream *>(os); - if (ofs) - ofs->close(); -} - BEGIN_DECLARE_SIM_OBJECT_PARAMS(Root) Param<bool> full_system; Param<Tick> frequency; - Param<string> output_dir; Param<string> output_file; - Param<string> config_output_file; END_DECLARE_SIM_OBJECT_PARAMS(Root) BEGIN_INIT_SIM_OBJECT_PARAMS(Root) - INIT_PARAM_DFLT(full_system, "full system simulation", true), - INIT_PARAM_DFLT(frequency, "tick frequency", 200000000), - INIT_PARAM_DFLT(output_dir, "directory to output data to", "."), - INIT_PARAM_DFLT(output_file, "file to dump simulator output to", "cout"), - INIT_PARAM_DFLT(config_output_file, "file to dump simulator config to", - "m5config.out") + INIT_PARAM(full_system, "full system simulation"), + INIT_PARAM(frequency, "tick frequency"), + INIT_PARAM(output_file, "file to dump simulator output to") END_INIT_SIM_OBJECT_PARAMS(Root) CREATE_SIM_OBJECT(Root) { + static bool created = false; + if (created) + panic("only one root object allowed!"); + + created = true; + fullSystem = full_system; + #ifdef FULL_SYSTEM - if (!bool(full_system)) + if (!fullSystem) panic("FULL_SYSTEM compiled and configuration not full_system"); #else - if (bool(full_system)) + if (fullSystem) panic("FULL_SYSTEM not compiled but configuration is full_system"); #endif - if (root) - panic("only one root object allowed!"); - root = new Root(getInstanceName()); - ticksPerSecond = frequency; double freq = double(ticksPerSecond); __ticksPerMS = freq / 1.0e3; @@ -143,29 +99,9 @@ CREATE_SIM_OBJECT(Root) __ticksPerNS = freq / 1.0e9; __ticksPerPS = freq / 1.0e12; - outputDirectory = output_dir; - if (!outputDirectory.empty()) { - outputDirectory = output_dir; - - // guarantee that directory ends with a '/' - if (outputDirectory[outputDirectory.size() - 1] != '/') - outputDirectory += "/"; - - if (mkdir(outputDirectory.c_str(), 0777) < 0) { - if (errno != EEXIST) { - panic("%s\ncould not make output directory: %s\n", - strerror(errno), outputDirectory); - } - } - } - - outputStream = makeOutputStream(output_file); - configStream = outputStream; - string cof = config_output_file; - if (!cof.empty()) - configStream = makeOutputStream(cof); - - return root; + outputStream = simout.find(output_file); + + return new Root(getInstanceName()); } REGISTER_SIM_OBJECT("Root", Root) diff --git a/test/genini.py b/test/genini.py index db1a7a5b0..73e7012f6 100644 --- a/test/genini.py +++ b/test/genini.py @@ -26,26 +26,31 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import getopt, os, os.path, sys +from os.path import join as joinpath, realpath -sys.path.append('..') -sys.path.append('../configs/kernel') -sys.path.append('../sim/pyconfig') +mypath = sys.path[0] +sys.path.append(joinpath(mypath, '..')) +sys.path.append(joinpath(mypath, '../configs/kernel')) +sys.path.append(joinpath(mypath, '../util/pbs')) +sys.path.append(joinpath(mypath, '../sim/pyconfig')) from importer import mpy_exec, mpy_execfile, AddToPath from m5config import * try: - opts, args = getopt.getopt(sys.argv[1:], '-E:') - for o,a in opts: - if o == '-E': - offset = a.find('=') + opts, args = getopt.getopt(sys.argv[1:], '-E:I:') + for opt,arg in opts: + if opt == '-E': + offset = arg.find('=') if offset == -1: - name = a + name = arg value = True else: - name = a[:offset] - value = a[offset+1:] + name = arg[:offset] + value = arg[offset+1:] env[name] = value + if opt == '-I': + AddToPath(arg) except getopt.GetoptError: sys.exit('Improper Usage') diff --git a/util/pbs/job.py b/util/pbs/job.py new file mode 100755 index 000000000..5eed0cd75 --- /dev/null +++ b/util/pbs/job.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert +# Steve Reinhardt +# Ali Saidi + +import os, os.path, shutil, signal, socket, sys, time +from os import environ as env +from os.path import join as joinpath, expanduser + +class rsync: + def __init__(self): + self.sudo = False + self.rsync = 'rsync' + self.compress = False + self.archive = True + self.delete = False + self.options = '' + + def do(self, src, dst): + args = [] + if self.sudo: + args.append('sudo') + + args.append(self.rsync) + if (self.archive): + args.append('-a') + if (self.compress): + args.append('-z') + if (self.delete): + args.append('--delete') + if len(self.options): + args.append(self.options) + args.append(src) + args.append(dst) + + return os.spawnvp(os.P_WAIT, args[0], args) + +def cleandir(dir): + for root, dirs, files in os.walk(dir, False): + for name in files: + os.remove(joinpath(root, name)) + for name in dirs: + os.rmdir(joinpath(root, name)) + +def date(): + return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime()) + +def remfile(file): + if os.path.isfile(file): + os.unlink(file) + +def readval(filename): + file = open(filename, 'r') + value = file.readline().strip() + file.close() + return value + +if __name__ == '__main__': + rootdir = env.setdefault('ROOTDIR', os.getcwd()) + jobid = env['PBS_JOBID'] + jobname = env['PBS_JOBNAME'] + jobdir = joinpath(rootdir, jobname) + basedir = joinpath(rootdir, 'Base') + user = env['USER'] + + env['POOLJOB'] = 'True' + env['OUTPUT_DIR'] = jobdir + env['JOBFILE'] = joinpath(basedir, 'test.py') + env['JOBNAME'] = jobname + + def echofile(filename, string): + try: + f = file(joinpath(jobdir, filename), 'w') + print >>f, string + f.flush() + f.close() + except IOError,e: + sys.exit(e) + + if os.path.isdir("/work"): + workbase = "/work" + else: + workbase = "/tmp/" + + workdir = joinpath(workbase, '%s.%s' % (user, jobid)) + + os.umask(0022) + + echofile('.start', date()) + echofile('.jobid', jobid) + echofile('.host', socket.gethostname()) + + if os.path.isdir(workdir): + cleandir(workdir) + else: + os.mkdir(workdir) + + if os.path.isdir('/z/dist'): + sync = rsync() + sync.delete = True + sync.sudo = True + sync.do('poolfs::dist/m5/', '/z/dist/m5/') + + try: + os.chdir(workdir) + except OSError,e: + sys.exit(e) + + os.symlink(joinpath(jobdir, 'output'), 'status.out') + + args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.mpy') ] + if not len(args): + sys.exit("no arguments") + + print 'starting job... %s' % date() + print ' '.join(args) + print + sys.stdout.flush() + + childpid = os.fork() + if not childpid: + # Execute command + sys.stdin.close() + fd = os.open(joinpath(jobdir, "output"), + os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + os.dup2(fd, sys.stdout.fileno()) + os.dup2(fd, sys.stderr.fileno()) + os.execvp(args[0], args) + + def handler(signum, frame): + if childpid != 0: + os.kill(childpid, signum) + + signal.signal(signal.SIGHUP, handler) + signal.signal(signal.SIGINT, handler) + signal.signal(signal.SIGQUIT, handler) + signal.signal(signal.SIGTERM, handler) + signal.signal(signal.SIGSTOP, handler) + signal.signal(signal.SIGCONT, handler) + signal.signal(signal.SIGUSR1, handler) + signal.signal(signal.SIGUSR2, handler) + + done = 0 + while not done: + try: + thepid,ec = os.waitpid(childpid, 0) + if ec: + print 'Exit code ', ec + echofile('.failure', date()) + else: + echofile('.success', date()) + done = 1 + except OSError: + pass + + print '\njob complete... %s' % date() + echofile('.stop', date()) diff --git a/util/pbs/jobfile.py b/util/pbs/jobfile.py new file mode 100644 index 000000000..570faa61b --- /dev/null +++ b/util/pbs/jobfile.py @@ -0,0 +1,83 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +from os.path import expanduser +def crossproduct(options): + number = len(options) + indexes = [ 0 ] * number + maxes = [ len(opt) for opt in options ] + def next(): + for i in xrange(number - 1, -1, -1): + indexes[i] += 1 + if indexes[i] < maxes[i]: + return False + + indexes[i] = 0 + return True + + done = False + while not done: + result = [] + for i in xrange(number): + result.append(options[i][indexes[i]]) + yield result + done = next() + +class JobFile(object): + def __init__(self, file): + self.data = {} + execfile(expanduser(file), self.data) + self.options = self.data['options'] + self.environment = self.data['environment'] + self.jobinfo = {} + self.jobs = [] + for job in crossproduct(self.options): + jobname = '.'.join([ id[0] for id in job ]) + self.jobs.append(jobname) + list = [] + for info in job: + for item in info[1:]: + list.append(item) + self.jobinfo[jobname] = list + + def env(self, jobname): + env = {} + for key,val in self.jobinfo[jobname]: + env[key] = val + + for key,val in self.environment: + env[key] = val + return env + + def printinfo(self, jobname): + print '%s:' % jobname + for key,val in self.jobinfo[jobname]: + print ' %s = %s' % (key, val) + + for key,val in self.environment: + print ' %s = %s' % (key, val) diff --git a/util/pbs/pbs.py b/util/pbs/pbs.py new file mode 100755 index 000000000..ecacbeba2 --- /dev/null +++ b/util/pbs/pbs.py @@ -0,0 +1,176 @@ +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Nathan Binkert + +import os, popen2, re, sys + +class MyPOpen(object): + def __init__(self, cmd, input = None, output = None, bufsize = -1): + self.status = -1 + + if input is None: + p2c_read, p2c_write = os.pipe() + self.tochild = os.fdopen(p2c_write, 'w', bufsize) + else: + p2c_write = None + if isinstance(input, file): + p2c_read = input.fileno() + elif isinstance(input, str): + input = file(input, 'r') + p2c_read = input.fileno() + elif isinstance(input, int): + p2c_read = input + else: + raise AttributeError + + if output is None: + c2p_read, c2p_write = os.pipe() + self.fromchild = os.fdopen(c2p_read, 'r', bufsize) + else: + c2p_read = None + if isinstance(output, file): + c2p_write = output.fileno() + elif isinstance(output, str): + output = file(output, 'w') + c2p_write = output.fileno() + elif isinstance(output, int): + c2p_write = output + else: + raise AttributeError + + self.pid = os.fork() + if self.pid == 0: + os.dup2(p2c_read, sys.stdin.fileno()) + os.dup2(c2p_write, sys.stdout.fileno()) + os.dup2(c2p_write, sys.stderr.fileno()) + try: + os.execvp(cmd[0], cmd) + finally: + os._exit(1) + + os.close(p2c_read) + os.close(c2p_write) + + def poll(self): + if self.status < 0: + pid, status = os.waitpid(self.pid, os.WNOHANG) + if pid == self.pid: + self.status = status + return self.status + + def wait(self): + if self.status < 0: + pid, status = os.waitpid(self.pid, 0) + if pid == self.pid: + self.status = status + return self.status + +class qsub: + def __init__(self): + self.hold = False + self.join = False + self.keep_stdout = False + self.keep_stderr = False + self.node_type = '' + self.mail_abort = False + self.mail_begin = False + self.mail_end = False + self.name = '' + self.stdout = '' + self.priority = 0 + self.queue = '' + self.pbshost = '' + self.qsub = 'qsub' + self.env = {} + + def build(self, script, args = []): + self.cmd = [ self.qsub ] + + if self.env: + arg = '-v' + arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ]) + self.cmd.append(arg) + + if self.hold: + self.cmd.append('-h') + + if len(self.stdout): + self.cmd.append('-olocalhost:' + self.stdout) + + if self.keep_stdout and self.keep_stderr: + self.cmd.append('-koe') + elif self.keep_stdout: + self.cmd.append('-ko') + elif self.keep_stderr: + self.cmd.append('-ke') + else: + self.cmd.append('-kn') + + if self.join: + self.cmd.append('-joe') + + if len(self.node_type): + self.cmd.append('-lnodes=' + self.node_type) + + if self.mail_abort or self.mail_begin or self.mail_end: + flags = '' + if self.mail_abort: + flags.append('a') + if self.mail_begin: + flags.append('b') + if self.mail_end: + flags.append('e') + if len(flags): + self.cmd.append('-m ' + flags) + + if len(self.name): + self.cmd.append("-N%s" % self.name) + + if self.priority != 0: + self.cmd.append('-p' + self.priority) + + if len(self.queue): + self.cmd.append('-q' + self.queue) + + self.cmd.extend(args) + self.script = script + self.command = ' '.join(self.cmd + [ self.script ]) + + def do(self): + pbs = MyPOpen(self.cmd + [ self.script ]) + self.result = pbs.fromchild.read() + ec = pbs.wait() + + if ec != 0 and self.pbshost: + cmd = ' '.join(self.cmd + [ '-' ]) + cmd = [ 'ssh', '-x', self.pbshost, cmd ] + self.command = ' '.join(cmd) + ssh = MyPOpen(cmd, input = self.script) + self.result = ssh.fromchild.read() + ec = ssh.wait() + + return ec diff --git a/util/pbs/send.py b/util/pbs/send.py new file mode 100755 index 000000000..4daf15b45 --- /dev/null +++ b/util/pbs/send.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# Copyright (c) 2005 The Regents of The University of Michigan +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Ali Saidi +# Nathan Binkert + +import os, os.path, re, socket, sys +from os import environ as env, listdir +from os.path import basename, isdir, isfile, islink, join as joinpath +from filecmp import cmp as filecmp +from shutil import copyfile + +def nfspath(dir): + if dir.startswith('/.automount/'): + dir = '/n/%s' % dir[12:] + elif not dir.startswith('/n/'): + dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) + return dir + +progpath = nfspath(sys.path[0]) +progname = basename(sys.argv[0]) +usage = """\ +Usage: + %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp> + -c clean directory if job can be run + -e only echo pbs command info, don't actually send the job + -f force the job to run regardless of state + -q <queue> submit job to the named queue + -v be verbose + + %(progname)s -l [-v] <regexp> + -l list job names, don't submit + -v be verbose (list job parameters) + + %(progname)s -h + -h display this help +""" % locals() + +try: + import getopt + opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v') +except getopt.GetoptError: + sys.exit(usage) + +clean = False +onlyecho = False +exprs = [] +force = False +listonly = False +queue = '' +verbose = False +rootdir = nfspath(os.getcwd()) +for opt,arg in opts: + if opt == '-c': + clean = True + if opt == '-d': + rootdir = arg + if opt == '-e': + onlyecho = True + if opt == '-f': + force = True + if opt == '-h': + print usage + sys.exit(0) + if opt == '-l': + listonly = True + if opt == '-q': + queue = arg + if opt == '-v': + verbose = True + +basedir = joinpath(rootdir, 'Base') +linkdir = joinpath(rootdir, 'Link') + +for arg in args: + exprs.append(re.compile(arg)) + +if not listonly and not onlyecho and isdir(linkdir): + if verbose: + print 'Checking for outdated files in Link directory' + entries = listdir(linkdir) + for entry in entries: + link = joinpath(linkdir, entry) + if not islink(link) or not isfile(link): + continue + + base = joinpath(basedir, entry) + if not isfile(base) or not filecmp(link, base): + print 'Base/%s is different than Link/%s: copying' % (entry, entry) + copyfile(link, base) + +import job, jobfile, pbs + +test = jobfile.JobFile(joinpath(basedir, 'test.py')) + +joblist = [] +for jobname in test.jobs: + if not exprs: + joblist.append(jobname) + continue + + for expr in exprs: + if expr.match(jobname): + joblist.append(jobname) + break + +if listonly: + if verbose: + for jobname in joblist: + test.printinfo(jobname) + else: + for jobname in joblist: + print jobname + sys.exit(0) + +if not onlyecho: + jl = [] + for jobname in joblist: + jobdir = joinpath(rootdir, jobname) + if os.path.exists(jobname): + if not force: + if os.path.isfile(joinpath(jobdir, '.success')): + continue + + if os.path.isfile(joinpath(jobdir, '.start')) and \ + not os.path.isfile(joinpath(jobdir, '.stop')): + continue + + if not clean: + sys.exit('job directory not clean!') + + job.cleandir(jobdir) + else: + os.mkdir(jobdir) + jl.append(jobname) + joblist = jl + +for jobname in joblist: + jobdir = joinpath(rootdir, jobname) + + if not onlyecho and not os.path.isdir(jobdir): + sys.exit('%s is not a directory. Cannot build job' % jobdir) + + print 'Job name: %s' % jobname + print 'Job directory: %s' % jobdir + + qsub = pbs.qsub() + qsub.pbshost = 'simpool.eecs.umich.edu' + qsub.stdout = joinpath(jobdir, 'jobout') + qsub.name = jobname + qsub.join = True + qsub.node_type = 'FAST' + qsub.env['ROOTDIR'] = rootdir + if len(queue): + qsub.queue = queue + qsub.build(joinpath(progpath, 'job.py')) + + if verbose: + print 'PBS Command: %s' % qsub.command + + if not onlyecho: + ec = qsub.do() + if ec == 0: + print 'PBS Jobid: %s' % qsub.result + else: + print 'PBS Failed' |