base: Add wrapped protobuf output streams

This patch adds support for outputting protobuf messages through a
ProtoOutputStream which hides the internal streams used by the
library. The stream is created based on the name of an output file and
optionally includes compression using gzip.

The output stream will start by putting a magic number in the file,
and then for every message that is serialized prepend the size such
that the stream can be written and read incrementally. At this point
this merely serves as a proof of concept.

4 files changed, 272 insertions, 0 deletions

diff --git a/src/proto/SConscript b/src/proto/SConscript
new file mode 100644
index 000000000..ade891737
--- /dev/null
+++ b/src/proto/SConscript
@@ -0,0 +1,44 @@
+# -*- mode:python -*-
+
+# Copyright (c) 2012 ARM Limited
+# All rights reserved.
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder.  You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Hansson
+
+Import('*')
+
+# Only build if we have protobuf support
+if env['HAVE_PROTOBUF']:
+    Source('protoio.cc')
diff --git a/src/proto/protoio.cc b/src/proto/protoio.cc
new file mode 100644
index 000000000..852ba56fa
--- /dev/null
+++ b/src/proto/protoio.cc
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2012 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Andreas Hansson
+ */
+
+#include "base/misc.hh"
+#include "proto/protoio.hh"
+
+using namespace std;
+using namespace google::protobuf;
+
+ProtoOutputStream::ProtoOutputStream(const string& filename) :
+    fileStream(filename.c_str(), ios::out | ios::binary | ios::trunc),
+    zeroCopyStream(NULL), gzipStream(NULL), codedStream(NULL)
+{
+    if (!fileStream.good())
+        panic("Could not open %s for writing\n", filename);
+
+    // Wrap the output file in a zero copy stream, that in turn is
+    // wrapped in a gzip stream if the filename ends with .gz. The
+    // latter stream is in turn wrapped in a coded stream
+    zeroCopyStream = new io::OstreamOutputStream(&fileStream);
+    if (filename.find_last_of('.') != string::npos &&
+        filename.substr(filename.find_last_of('.') + 1) == "gz") {
+        gzipStream = new io::GzipOutputStream(zeroCopyStream);
+        codedStream = new io::CodedOutputStream(gzipStream);
+    } else {
+        codedStream = new io::CodedOutputStream(zeroCopyStream);
+    }
+
+    // Use the ASCII characters gem5 as our magic number and write it
+    // to the file
+    const uint32_t magic_number = 0x356d6567;
+    codedStream->WriteLittleEndian32(magic_number);
+
+    // Note that each type of stream (packet, instruction etc) should
+    // add its own header and perform the appropriate checks
+}
+
+ProtoOutputStream::~ProtoOutputStream()
+{
+    delete codedStream;
+    // As the compression is optional, see if the stream exists
+    if (gzipStream != NULL)
+        delete gzipStream;
+    delete zeroCopyStream;
+    fileStream.close();
+}
+
+void
+ProtoOutputStream::write(const Message& msg)
+{
+    // Write the size of the message to the stream
+    codedStream->WriteVarint32(msg.ByteSize());
+
+    // Write the message itself to the stream
+    if (!msg.SerializeToCodedStream(codedStream))
+        panic("Unable to write message to coded stream\n");
+}
diff --git a/src/proto/protoio.hh b/src/proto/protoio.hh
new file mode 100644
index 000000000..bb11bc7e9
--- /dev/null
+++ b/src/proto/protoio.hh
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2012 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Andreas Hansson
+ */
+
+
+/**
+ * @file
+ * Declaration of a wrapper for protobuf output streams.
+ */
+
+#ifndef __PROTO_PROTOIO_HH__
+#define __PROTO_PROTOIO_HH__
+
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/gzip_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/message.h>
+
+#include <fstream>
+
+/**
+ * A ProtoOutputStream wraps a coded stream, potentially with
+ * compression, based on looking at the file name. Writing to the
+ * stream is done to enable interaction with the file on a per-message
+ * basis to avoid having to deal with huge data structures. The latter
+ * is made possible by encoding the length of each message in the
+ * stream.
+ */
+class ProtoOutputStream
+{
+
+  public:
+
+    /**
+     * Create an output stream for a given file name. If the filename
+     * ends with .gz then the file will be compressed accordinly.
+     *
+     * @param filename Path to the file to create or truncate
+     */
+    ProtoOutputStream(const std::string& filename);
+
+    /**
+     * Destruct the output stream, and also flush and close the
+     * underlying file streams and coded streams.
+     */
+    ~ProtoOutputStream();
+
+    /**
+     * Write a message to the stream, preprending it with the message
+     * size.
+     *
+     * @param msg Message to write to the stream
+     */
+    void write(const google::protobuf::Message& msg);
+
+  private:
+
+    /// Underlying file output stream
+    std::ofstream fileStream;
+
+    /// Zero Copy stream wrapping the STL output stream
+    google::protobuf::io::OstreamOutputStream* zeroCopyStream;
+
+    /// Optional Gzip stream to wrap the Zero Copy stream
+    google::protobuf::io::GzipOutputStream* gzipStream;
+
+    /// Top-level coded stream that messages are written to
+    google::protobuf::io::CodedOutputStream* codedStream;
+
+};
+
+#endif //__PROTO_PROTOIO_HH
diff --git a/src/sim/init.cc b/src/sim/init.cc
index 788d125c3..0cebc417d 100644
--- a/src/sim/init.cc
+++ b/src/sim/init.cc
@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2012 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 2000-2005 The Regents of The University of Michigan
  * Copyright (c) 2008 The Hewlett-Packard Development Company
  * All rights reserved.
@@ -42,10 +54,15 @@
 #include "base/cprintf.hh"
 #include "base/misc.hh"
 #include "base/types.hh"
+#include "config/have_protobuf.hh"
 #include "sim/async.hh"
 #include "sim/core.hh"
 #include "sim/init.hh"
 
+#if HAVE_PROTOBUF
+#include <google/protobuf/stubs/common.h>
+#endif
+
 using namespace std;
 
 /// Stats signal handler.
@@ -238,6 +255,13 @@ const char * __attribute__((weak)) m5MainCommands[] = {
 int
 m5Main(int argc, char **argv)
 {
+#if HAVE_PROTOBUF
+    // Verify that the version of the protobuf library that we linked
+    // against is compatible with the version of the headers we
+    // compiled against.
+    GOOGLE_PROTOBUF_VERIFY_VERSION;
+#endif
+
     PySys_SetArgv(argc, argv);
 
     // We have to set things up in the special __main__ module
@@ -263,6 +287,10 @@ m5Main(int argc, char **argv)
         command++;
     }
 
+#if HAVE_PROTOBUF
+    google::protobuf::ShutdownProtobufLibrary();
+#endif
+
     return 0;
 }