base: Add support for changing output directories

This changeset adds support for changing the simulator output
directory. This can be useful when the simulation goes through several
stages (e.g., a warming phase, a simulation phase, and a verification
phase) since it allows the output from each stage to be located in a
different directory. Relocation is done by calling core.setOutputDir()
from Python or simout.setOutputDirectory() from C++.

This change affects several parts of the design of the gem5's output
subsystem. First, files returned by an OutputDirectory instance (e.g.,
simout) are of the type OutputStream instead of a std::ostream. This
allows us to do some more book keeping and control re-opening of files
when the output directory is changed. Second, new subdirectories are
OutputDirectory instances, which should be used to create files in
that sub-directory.

Signed-off-by: Andreas Sandberg <andreas@sandberg.pp.se>
[sascha.bischoff@arm.com: Rebased patches onto a newer gem5 version]
Signed-off-by: Sascha Bischoff <sascha.bischoff@arm.com>
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>

5 files changed, 318 insertions, 129 deletions

diff --git a/src/base/output.cc b/src/base/output.cc
index c2a37e58e..cb3c0b7f9 100644
--- a/src/base/output.cc
+++ b/src/base/output.cc
@@ -1,4 +1,17 @@
 /*
+ * Copyright (c) 2015 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Copyright (c) 2013 Andreas Sandberg
  * Copyright (c) 2005 The Regents of The University of Michigan
  * All rights reserved.
  *
@@ -27,6 +40,8 @@
  *
  * Authors: Nathan Binkert
  *          Chris Emmons
+ *          Andreas Sandberg
+ *          Sascha Bischoff
  */
 
 #include <sys/stat.h>
@@ -49,94 +64,127 @@ using namespace std;
 
 OutputDirectory simout;
 
+
+OutputStream::OutputStream(const std::string &name, std::ostream *stream)
+    : _name(name), _stream(stream)
+{
+}
+
+OutputStream::~OutputStream()
+{
+}
+
+void
+OutputStream::relocate(const OutputDirectory &dir)
+{
+}
+
+template<class StreamType>
+OutputFile<StreamType>::OutputFile(const OutputDirectory &dir,
+                                   const std::string &name,
+                                   std::ios_base::openmode mode,
+                                   bool recreateable)
+  : OutputStream(name, new stream_type_t()),
+    _mode(mode), _recreateable(recreateable),
+    _fstream(static_cast<stream_type_t *>(_stream))
+{
+    _fstream->open(dir.resolve(_name).c_str(), _mode);
+
+    assert(_fstream->is_open());
+}
+
+template<class StreamType>
+OutputFile<StreamType>::~OutputFile()
+{
+    if (_fstream->is_open())
+        _fstream->close();
+}
+
+template<class StreamType>
+void
+OutputFile<StreamType>::relocate(const OutputDirectory &dir)
+{
+    if (_recreateable) {
+        _fstream->close();
+        _fstream->open(dir.resolve(_name).c_str(), _mode);
+    }
+}
+
+OutputStream OutputDirectory::stdout("stdout", &cout);
+OutputStream OutputDirectory::stderr("stderr", &cerr);
+
 /**
  * @file This file manages creating / deleting output files for the simulator.
  */
 OutputDirectory::OutputDirectory()
 {}
 
+OutputDirectory::OutputDirectory(const std::string &name)
+{
+    setDirectory(name);
+}
+
 OutputDirectory::~OutputDirectory()
 {
-    for (map_t::iterator i = files.begin(); i != files.end(); i++) {
-        if (i->second)
-            delete i->second;
+    for (auto& f: files) {
+        if (f.second)
+            delete f.second;
     }
 }
 
-std::ostream *
-OutputDirectory::checkForStdio(const string &name) const
+OutputStream *
+OutputDirectory::checkForStdio(const string &name)
 {
     if (name == "cerr" || name == "stderr")
-        return &cerr;
+        return &stderr;
 
     if (name == "cout" || name == "stdout")
-        return &cout;
+        return &stdout;
 
     return NULL;
 }
 
-ostream *
-OutputDirectory::openFile(const string &filename,
-                          ios_base::openmode mode, bool no_gz)
-{
-    bool gz = !no_gz;
-    gz = gz && filename.find(".gz", filename.length()-3) < filename.length();
-    if (gz) {
-        gzofstream *file = new gzofstream(filename.c_str(), mode);
-        if (!file->is_open())
-            fatal("Cannot open file %s", filename);
-        assert(files.find(filename) == files.end());
-        files[filename] = file;
-        return file;
-    } else {
-        ofstream *file = new ofstream(filename.c_str(), mode);
-        if (!file->is_open())
-            fatal("Cannot open file %s", filename);
-        assert(files.find(filename) == files.end());
-        files[filename] = file;
-        return file;
-    }
-}
-
 void
-OutputDirectory::close(ostream *openStream) {
-    map_t::iterator i;
-    for (i = files.begin(); i != files.end(); i++) {
-        if (i->second != openStream)
-            continue;
-
-        ofstream *fs = dynamic_cast<ofstream*>(i->second);
-        if (fs) {
-            fs->close();
-            delete i->second;
-            break;
-        } else {
-            gzofstream *gfs = dynamic_cast<gzofstream*>(i->second);
-            if (gfs) {
-                gfs->close();
-                delete i->second;
-                break;
-            }
-        }
-    }
-
+OutputDirectory::close(OutputStream *file)
+{
+    auto i = files.find(file->name());
     if (i == files.end())
         fatal("Attempted to close an unregistred file stream");
 
     files.erase(i);
+
+    delete file;
 }
 
 void
 OutputDirectory::setDirectory(const string &d)
 {
-    if (!dir.empty())
-        panic("Output directory already set!\n");
+    const string old_dir(dir);
 
     dir = d;
 
     // guarantee that directory ends with a path separator
     if (dir[dir.size() - 1] != PATH_SEPARATOR)
         dir += PATH_SEPARATOR;
+
+    // Try to create the directory. If it already exists, that's ok;
+    // otherwise, fail if we couldn't create it.
+    if ((mkdir(dir.c_str(), 0755) != 0) && (errno != EEXIST))
+        fatal("Failed to create new output subdirectory '%s'\n", dir);
+
+    // Check if we need to recreate anything
+    if (!old_dir.empty()) {
+        // Recreate output files
+        for (file_map_t::iterator i = files.begin(); i != files.end(); ++i) {
+            i->second->relocate(*this);
+        }
+
+        // Relocate sub-directories
+        for (dir_map_t::iterator i = dirs.begin(); i != dirs.end(); ++i) {
+            i->second->setDirectory(dir + PATH_SEPARATOR + i->first);
+        }
+    }
+
 }
 
 const string &
@@ -151,43 +199,69 @@ OutputDirectory::directory() const
 string
 OutputDirectory::resolve(const string &name) const
 {
-    return (name[0] != PATH_SEPARATOR) ? dir + name : name;
+    return !isAbsolute(name) ? dir + name : name;
 }
 
-ostream *
+OutputStream *
 OutputDirectory::create(const string &name, bool binary, bool no_gz)
 {
-    ostream *file = checkForStdio(name);
+    OutputStream *file = checkForStdio(name);
     if (file)
         return file;
 
-    string filename = resolve(name);
-    ios_base::openmode mode =
-        ios::trunc | (binary ? ios::binary : (ios::openmode)0);
-    file = openFile(filename, mode, no_gz);
+    const ios_base::openmode mode(
+        ios::trunc | (binary ? ios::binary : (ios::openmode)0));
+    const bool recreateable(!isAbsolute(name));
 
-    return file;
+    return open(name, mode, recreateable, no_gz);
 }
 
-ostream *
+OutputStream *
+OutputDirectory::open(const std::string &name,
+                      ios_base::openmode mode,
+                      bool recreateable,
+                      bool no_gz)
+{
+    OutputStream *os;
+
+    if (!no_gz && name.find(".gz", name.length() - 3) < name.length()) {
+        // Although we are creating an output stream, we still need to pass the
+        // correct mode for gzofstream as this used directly to set the file
+        // mode.
+        mode |= std::ios::out;
+        os = new OutputFile<gzofstream>(*this, name, mode, recreateable);
+    } else {
+        os = new OutputFile<ofstream>(*this, name, mode, recreateable);
+    }
+
+    files[name] = os;
+
+    return os;
+}
+
+OutputStream *
 OutputDirectory::find(const string &name) const
 {
-    ostream *file = checkForStdio(name);
+    OutputStream *file = checkForStdio(name);
     if (file)
         return file;
 
-    const string filename = resolve(name);
-    map_t::const_iterator i = files.find(filename);
+    auto i = files.find(name);
     if (i != files.end())
         return (*i).second;
 
     return NULL;
 }
 
-bool
-OutputDirectory::isFile(const std::ostream *os)
+
+OutputStream *
+OutputDirectory::findOrCreate(const std::string &name, bool binary)
 {
-    return os && os != &cerr && os != &cout;
+    OutputStream *os(find(name));
+    if (os)
+        return os;
+    else
+        return create(name, binary);
 }
 
 bool
@@ -201,18 +275,17 @@ OutputDirectory::isFile(const string &name) const
     return (st == 0) && S_ISREG(st_buf.st_mode);
 }
 
-string
-OutputDirectory::createSubdirectory(const string &name) const
+OutputDirectory *
+OutputDirectory::createSubdirectory(const string &name)
 {
     const string new_dir = resolve(name);
     if (new_dir.find(directory()) == string::npos)
         fatal("Attempting to create subdirectory not in m5 output dir\n");
 
-    // if it already exists, that's ok; otherwise, fail if we couldn't create
-    if ((mkdir(new_dir.c_str(), 0755) != 0) && (errno != EEXIST))
-        fatal("Failed to create new output subdirectory '%s'\n", new_dir);
+    OutputDirectory *dir(new OutputDirectory(new_dir));
+    dirs[name] = dir;
 
-    return name + PATH_SEPARATOR;
+    return dir;
 }
 
 void
@@ -225,10 +298,10 @@ OutputDirectory::remove(const string &name, bool recursive)
 
     if (isFile(fname)) {
         // close and release file if we have it open
-        map_t::iterator itr = files.find(fname);
-        if (itr != files.end()) {
-            delete itr->second;
-            files.erase(itr);
+        auto i = files.find(fname);
+        if (i != files.end()) {
+            delete i->second;
+            files.erase(i);
         }
 
         if (::remove(fname.c_str()) != 0)
@@ -246,7 +319,7 @@ OutputDirectory::remove(const string &name, bool recursive)
             if (!subdir) {
                 perror("opendir");
                 fatal("Error opening directory for recursive removal '%s'\n",
-                    fname);
+                      fname);
             }
 
             struct dirent *de = readdir(subdir);
diff --git a/src/base/output.hh b/src/base/output.hh
index 67e6ecb18..a0b1d0124 100644
--- a/src/base/output.hh
+++ b/src/base/output.hh
@@ -1,4 +1,17 @@
 /*
+ * Copyright (c) 2015 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Copyright (c) 2013 Andreas Sandberg
  * Copyright (c) 2005 The Regents of The University of Michigan
  * All rights reserved.
  *
@@ -27,6 +40,8 @@
  *
  * Authors: Nathan Binkert
  *          Chris Emmons
+ *          Andreas Sandberg
+ *          Sascha Bischoff
  */
 
 #ifndef __BASE_OUTPUT_HH__
@@ -36,15 +51,105 @@
 #include <map>
 #include <string>
 
+#include "base/compiler.hh"
+
+class OutputDirectory;
+
+class OutputStream
+{
+  public:
+    virtual ~OutputStream();
+
+    /** Get the output underlying output stream */
+    std::ostream *stream() const { return _stream; };
+
+    /**
+     * Can the file be recreated if the output directory is moved?
+     *
+     * @return true if the file will be created in the new location,
+     * false otherwise.
+     */
+    virtual bool recreateable() const { return false; }
+
+    /** Get the file name in the output directory */
+    const std::string &name() const { return _name; }
+
+  protected:
+    friend class OutputDirectory;
+
+    /** Wrap an existing stream */
+    OutputStream(const std::string &name,
+                 std::ostream *stream);
+
+    /* Prevent copying */
+    OutputStream(const OutputStream &f);
+
+    /** Re-create the in a new location if recreateable. */
+    virtual void relocate(const OutputDirectory &dir);
+
+    /** Name in output directory */
+    const std::string _name;
+
+    /** Underlying output stream */
+    std::ostream *const _stream;
+};
+
+template<class StreamType>
+class OutputFile
+    : public OutputStream
+{
+  public:
+    typedef StreamType stream_type_t;
+
+    virtual ~OutputFile();
+
+    /**
+     * Can the file be recreated if the output directory is moved?
+     *
+     * @return true if the file will be created in the new location,
+     * false otherwise.
+     */
+    bool recreateable() const override { return _recreateable; }
+
+  protected:
+    friend class OutputDirectory;
+
+    OutputFile(const OutputDirectory &dir,
+               const std::string &name,
+               std::ios_base::openmode mode,
+               bool recreateable);
+
+    /* Prevent copying */
+    OutputFile(const OutputFile<StreamType> &f);
+
+    /** Re-create the file in a new location if it is relocatable. */
+    void relocate(const OutputDirectory &dir) override;
+
+    /** File mode when opened */
+    const std::ios_base::openmode _mode;
+
+    /** Can the file be recreated in a new location? */
+    const bool _recreateable;
+
+    /** Pointer to the file stream */
+    stream_type_t *const _fstream;
+};
+
 /** Interface for creating files in a gem5 output directory. */
 class OutputDirectory
 {
   private:
     /** File names and associated stream handles */
-    typedef std::map<std::string, std::ostream *> map_t;
+    typedef std::map<std::string, OutputStream *> file_map_t;
+
+    /** Output subdirectories */
+    typedef std::map<std::string, OutputDirectory *> dir_map_t;
 
     /** Open file streams within this directory */
-    map_t files;
+    file_map_t files;
+
+    /** Output sub-directories */
+    dir_map_t dirs;
 
     /** Name of this directory */
     std::string dir;
@@ -52,6 +157,9 @@ class OutputDirectory
     /** System-specific path separator character */
     static const char PATH_SEPARATOR = '/';
 
+    static OutputStream stdout;
+    static OutputStream stderr;
+
   protected:
     /**
      * Determines whether given file name corresponds to standard output
@@ -61,12 +169,15 @@ class OutputDirectory
      * @return output stream for standard output or error stream if name
      *         corresponds to one or the other; NULL otherwise
      */
-    std::ostream *checkForStdio(const std::string &name) const;
+    static OutputStream *checkForStdio(const std::string &name);
 
   public:
     /** Constructor. */
     OutputDirectory();
 
+    /** Constructor. */
+    OutputDirectory(const std::string &name);
+
     /** Destructor. */
     ~OutputDirectory();
 
@@ -80,20 +191,6 @@ class OutputDirectory
      */
     std::string resolve(const std::string &name) const;
 
-    /** Opens a file (optionally compressed).
-     *
-     * Will open a file as a compressed stream if filename ends in .gz.
-     *
-     * @param filename file to open
-     * @param mode attributes to open file with
-     * @param no_gz true to disable opening the file as a gzip compressed output
-     *     stream; false otherwise
-     * @return stream pointer to opened file; will cause sim fail on error
-     */
-    std::ostream *openFile(const std::string &filename,
-                        std::ios_base::openmode mode = std::ios::trunc,
-                        bool no_gz = false);
-
     /**
      * Sets name of this directory.
      * @param dir name of this directory
@@ -109,40 +206,64 @@ class OutputDirectory
     /**
      * Creates a file in this directory (optionally compressed).
      *
-     * Will open a file as a compressed stream if filename ends in .gz.
+     * Will open a file as a compressed stream if filename ends in .gz, unless
+     * explicitly disabled.
+     *
+     * Relative output paths will result in the creation of a
+     * recreateable (see OutputFile) output file in the current output
+     * directory. Files created with an absolute path will not be
+     * recreateable.
      *
      * @param name name of file to create (without this directory's name
      *          leading it)
      * @param binary true to create a binary file; false otherwise
-     * @param no_gz true to disable creating a gzip compressed output stream;
-     *     false otherwise
-     * @return stream to the opened file
+     * @param no_gz true to disable opening the file as a gzip compressed output
+     *     stream; false otherwise
+     * @return OutputStream instance representing the created file
      */
-    std::ostream *create(const std::string &name, bool binary = false,
+    OutputStream *create(const std::string &name,
+                         bool binary = false,
                          bool no_gz = false);
 
     /**
-     * Closes a file stream.
+     * Open a file in this directory (optionally compressed).
+     *
+     * Will open a file as a compressed stream if filename ends in .gz, unless
+     * explicitly disabled.
+     *
+     * @param filename file to open
+     * @param mode attributes to open file with
+     * @param recreateable Set to true if the file can be recreated in a new
+     *     location.
+     * @param no_gz true to disable opening the file as a gzip compressed output
+     *     stream; false otherwise
+     * @return OutputStream instance representing the opened file
+     */
+    OutputStream *open(const std::string &name,
+                       std::ios_base::openmode mode,
+                       bool recreateable = true,
+                       bool no_gz = false);
+
+    /**
+     * Closes an output file and free the corresponding OutputFile.
      *
-     * Stream must have been opened through this interface, or sim will fail.
+     * The output file must have been opened by the same
+     * OutputDirectory instance as the one closing it, or sim will
+     * fail.
      *
-     * @param openStream open stream to close
+     * @param file OutputStream instance in this OutputDirectory.
      */
-    void close(std::ostream *openStream);
+    void close(OutputStream *file);
 
     /**
-     * Finds stream associated with a file.
+     * Finds stream associated with an open file or stdout/stderr.
+     *
      * @param name of file
      * @return stream to specified file or NULL if file does not exist
      */
-    std::ostream *find(const std::string &name) const;
+    OutputStream *find(const std::string &name) const;
 
-    /**
-     * Returns true if stream is open and not standard output or error.
-     * @param os output stream to evaluate
-     * @return true if os is non-NULL and not cout or cerr
-     */
-    static bool isFile(const std::ostream *os);
+    OutputStream *findOrCreate(const std::string &name, bool binary = false);
 
     /**
      * Determines whether a file name corresponds to a file in this directory.
@@ -153,12 +274,10 @@ class OutputDirectory
     bool isFile(const std::string &name) const;
 
     /**
-     * Returns true if stream is open and not standard output or error.
-     * @param os output stream to evaluate
-     * @return true if os is non-NULL and not cout or cerr
+     * Test if a path is absolute.
      */
-    static inline bool isFile(const std::ostream &os) {
-        return isFile(&os);
+    static inline bool isAbsolute(const std::string &name) {
+        return name[0] == PATH_SEPARATOR;
     }
 
     /**
@@ -166,7 +285,7 @@ class OutputDirectory
      * @param name name of subdirectory
      * @return the new subdirectory's name suffixed with a path separator
      */
-    std::string createSubdirectory(const std::string &name) const;
+    OutputDirectory *createSubdirectory(const std::string &name);
 
     /**
      * Removes a specified file or subdirectory.
diff --git a/src/base/stats/text.cc b/src/base/stats/text.cc
index efe43c602..05ce33cdf 100644
--- a/src/base/stats/text.cc
+++ b/src/base/stats/text.cc
@@ -740,11 +740,7 @@ initText(const string &filename, bool desc)
     static bool connected = false;
 
     if (!connected) {
-        ostream *os = simout.find(filename);
-        if (!os)
-            os = simout.create(filename);
-
-        text.open(*os);
+        text.open(*simout.findOrCreate(filename)->stream());
         text.descriptions = desc;
         connected = true;
     }
diff --git a/src/base/vnc/vncinput.cc b/src/base/vnc/vncinput.cc
index 017fc3876..d97306f03 100644
--- a/src/base/vnc/vncinput.cc
+++ b/src/base/vnc/vncinput.cc
@@ -123,10 +123,9 @@ VncInput::captureFrameBuffer()
     const string frameFilename(frameFilenameBuffer);
 
     // create the compressed framebuffer file
-    ostream *fb_out = simout.create(captureOutputDirectory + frameFilename,
-                                    true);
-    captureBitmap->write(*fb_out);
-    simout.close(fb_out);
+    OutputStream *fb_out(captureOutputDirectory->create(frameFilename, true));
+    captureBitmap->write(*fb_out->stream());
+    captureOutputDirectory->close(fb_out);
 
     ++captureCurrentFrame;
 }
diff --git a/src/base/vnc/vncinput.hh b/src/base/vnc/vncinput.hh
index 96235fec7..15ddc5c58 100644
--- a/src/base/vnc/vncinput.hh
+++ b/src/base/vnc/vncinput.hh
@@ -52,6 +52,8 @@
 #include "params/VncInput.hh"
 #include "sim/sim_object.hh"
 
+class OutputDirectory;
+
 /**
  * A device that expects to receive input from the vnc server should derrive
  * (through mulitple inheritence if necessary from VncKeyboard or VncMouse
@@ -219,7 +221,7 @@ class VncInput : public SimObject
     int captureCurrentFrame;
 
     /** Directory to store captured frames to */
-    std::string captureOutputDirectory;
+    OutputDirectory *captureOutputDirectory;
 
     /** Computed hash of the last captured frame */
     uint64_t captureLastHash;