syscall-emul: Rewrite system call exit code

The changeset does a major refactor on the exit, exit_group, and
futex system calls regarding exit functionality.

A FutexMap class and related structures are added into a new
file. This increases code clarity by encapsulating the futex
operations and the futex state into an object.

Several exit conditions were added to allow the simulator to end
processes under certain conditions. Also, the simulation only
exits now when all processes have finished executing.

Change-Id: I1ee244caa9b5586fe7375e5b9b50fd3959b9655e
Reviewed-on: https://gem5-review.googlesource.com/2269
Maintainer: Jason Lowe-Power <jason@lowepower.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>

6 files changed, 282 insertions, 114 deletions

diff --git a/src/arch/riscv/registers.hh b/src/arch/riscv/registers.hh
index d8977030d..0793398fb 100644
--- a/src/arch/riscv/registers.hh
+++ b/src/arch/riscv/registers.hh
@@ -50,9 +50,9 @@
 #include <map>
 #include <string>
 
+#include "arch/isa_traits.hh"
 #include "arch/riscv/generated/max_inst_regs.hh"
 #include "base/types.hh"
-#include "sim/system.hh"
 
 namespace RiscvISA {
 
diff --git a/src/sim/futex_map.hh b/src/sim/futex_map.hh
new file mode 100644
index 000000000..998f8d481
--- /dev/null
+++ b/src/sim/futex_map.hh
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2017 Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Brandon Potter
+ *          Steve Reinhardt
+ *          Alexandru Dutu
+ */
+
+#ifndef __FUTEX_MAP_HH__
+#define __FUTEX_MAP_HH__
+
+#include <unordered_map>
+
+#include <cpu/thread_context.hh>
+
+/**
+ * FutexKey class defines an unique identifier for a particular futex in the
+ * system. The tgid and an address are the unique values needed as the key.
+ */
+class FutexKey {
+  public:
+    uint64_t addr;
+    uint64_t tgid;
+
+    FutexKey(uint64_t addr_in, uint64_t tgid_in)
+        : addr(addr_in), tgid(tgid_in)
+    {
+    }
+
+    bool
+    operator==(const FutexKey &in) const
+    {
+        return addr == in.addr && tgid == in.tgid;
+    }
+};
+
+namespace std {
+    /**
+     * The unordered_map structure needs the parenthesis operator defined for
+     * std::hash if a user defined key is used. Our key is is user defined
+     * so we need to provide the hash functor.
+     */
+    template <>
+    struct hash<FutexKey>
+    {
+        size_t operator()(const FutexKey& in) const
+        {
+            size_t hash = 65521;
+            for (int i = 0; i < sizeof(uint64_t) / sizeof(size_t); i++) {
+                hash ^= (size_t)(in.addr >> sizeof(size_t) * i) ^
+                        (size_t)(in.tgid >> sizeof(size_t) * i);
+            }
+            return hash;
+        }
+    };
+}
+
+typedef std::list<ThreadContext *> ThreadContextList;
+
+/**
+ * FutexMap class holds a map of all futexes used in the system
+ */
+class FutexMap : public std::unordered_map<FutexKey, ThreadContextList>
+{
+  public:
+    /** Inserts a futex into the map with one waiting TC */
+    void
+    suspend(Addr addr, uint64_t tgid, ThreadContext *tc)
+    {
+        FutexKey key(addr, tgid);
+        auto it = find(key);
+
+        if (it == end()) {
+            ThreadContextList tcList {tc};
+            insert({key, tcList});
+        } else {
+            it->second.push_back(tc);
+        }
+
+        /** Suspend the thread context */
+        tc->suspend();
+    }
+
+    /** Wakes up at most count waiting threads on a futex */
+    int
+    wakeup(Addr addr, uint64_t tgid, int count)
+    {
+        FutexKey key(addr, tgid);
+        auto it = find(key);
+
+        if (it == end())
+            return 0;
+
+        int woken_up = 0;
+        auto &tcList = it->second;
+
+        while (!tcList.empty() && woken_up < count) {
+            tcList.front()->activate();
+            tcList.pop_front();
+            woken_up++;
+        }
+
+        if (tcList.empty())
+            erase(it);
+
+        return woken_up;
+    }
+
+};
+
+#endif // __FUTEX_MAP_HH__
diff --git a/src/sim/process.cc b/src/sim/process.cc
index 71f398893..406b38679 100644
--- a/src/sim/process.cc
+++ b/src/sim/process.cc
@@ -50,6 +50,7 @@
 #include <unistd.h>
 
 #include <array>
+#include <csignal>
 #include <map>
 #include <string>
 #include <vector>
@@ -71,20 +72,27 @@
 
 #if THE_ISA == ALPHA_ISA
 #include "arch/alpha/linux/process.hh"
+
 #elif THE_ISA == SPARC_ISA
 #include "arch/sparc/linux/process.hh"
 #include "arch/sparc/solaris/process.hh"
+
 #elif THE_ISA == MIPS_ISA
 #include "arch/mips/linux/process.hh"
+
 #elif THE_ISA == ARM_ISA
-#include "arch/arm/linux/process.hh"
 #include "arch/arm/freebsd/process.hh"
+#include "arch/arm/linux/process.hh"
+
 #elif THE_ISA == X86_ISA
 #include "arch/x86/linux/process.hh"
+
 #elif THE_ISA == POWER_ISA
 #include "arch/power/linux/process.hh"
+
 #elif THE_ISA == RISCV_ISA
 #include "arch/riscv/linux/process.hh"
+
 #else
 #error "THE_ISA not set"
 #endif
diff --git a/src/sim/syscall_emul.cc b/src/sim/syscall_emul.cc
index 1441592a8..4cf17a266 100644
--- a/src/sim/syscall_emul.cc
+++ b/src/sim/syscall_emul.cc
@@ -34,6 +34,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 
+#include <csignal>
 #include <iostream>
 #include <string>
 
@@ -75,67 +76,116 @@ ignoreFunc(SyscallDesc *desc, int callnum, Process *process,
 }
 
 static void
-exitFutexWake(ThreadContext *tc, uint64_t uaddr)
-{
-    std::map<uint64_t, std::list<ThreadContext *> * >
-        &futex_map = tc->getSystemPtr()->futexMap;
-
-    int wokenUp = 0;
-    std::list<ThreadContext *> * tcWaitList;
-    if (futex_map.count(uaddr)) {
-        tcWaitList = futex_map.find(uaddr)->second;
-        if (tcWaitList->size() > 0) {
-            tcWaitList->front()->activate();
-            tcWaitList->pop_front();
-            wokenUp++;
+exitFutexWake(ThreadContext *tc, Addr addr, uint64_t tgid)
+{
+    // Clear value at address pointed to by thread's childClearTID field.
+    BufferArg ctidBuf(addr, sizeof(long));
+    long *ctid = (long *)ctidBuf.bufferPtr();
+    *ctid = 0;
+    ctidBuf.copyOut(tc->getMemProxy());
+
+    FutexMap &futex_map = tc->getSystemPtr()->futexMap;
+    // Wake one of the waiting threads.
+    futex_map.wakeup(addr, tgid, 1);
+}
+
+static SyscallReturn
+exitImpl(SyscallDesc *desc, int callnum, Process *p, ThreadContext *tc,
+         bool group)
+{
+    int index = 0;
+    int status = p->getSyscallArg(tc, index);
+
+    System *sys = tc->getSystemPtr();
+
+    int activeContexts = 0;
+    for (auto &system: sys->systemList)
+        activeContexts += system->numRunningContexts();
+    if (activeContexts == 1) {
+        exitSimLoop("exiting with last active thread context", status & 0xff);
+        return status;
+    }
+
+    if (group)
+        *p->exitGroup = true;
+
+    if (p->childClearTID)
+        exitFutexWake(tc, p->childClearTID, p->tgid());
+
+    bool last_thread = true;
+    Process *parent = nullptr, *tg_lead = nullptr;
+    for (int i = 0; last_thread && i < sys->numContexts(); i++) {
+        Process *walk;
+        if (!(walk = sys->threadContexts[i]->getProcessPtr()))
+            continue;
+
+        /**
+         * Threads in a thread group require special handing. For instance,
+         * we send the SIGCHLD signal so that it appears that it came from
+         * the head of the group. We also only delete file descriptors if
+         * we are the last thread in the thread group.
+         */
+        if (walk->pid() == p->tgid())
+            tg_lead = walk;
+
+        if ((sys->threadContexts[i]->status() != ThreadContext::Halted)
+            && (walk != p)) {
+            /**
+             * Check if we share thread group with the pointer; this denotes
+             * that we are not the last thread active in the thread group.
+             * Note that setting this to false also prevents further
+             * iterations of the loop.
+             */
+            if (walk->tgid() == p->tgid())
+                last_thread = false;
+
+            /**
+             * A corner case exists which involves execve(). After execve(),
+             * the execve will enable SIGCHLD in the process. The problem
+             * occurs when the exiting process is the root process in the
+             * system; there is no parent to receive the signal. We obviate
+             * this problem by setting the root process' ppid to zero in the
+             * Python configuration files. We really should handle the
+             * root/execve specific case more gracefully.
+             */
+            if (*p->sigchld && (p->ppid() != 0) && (walk->pid() == p->ppid()))
+                parent = walk;
         }
-        if (tcWaitList->empty()) {
-            futex_map.erase(uaddr);
-            delete tcWaitList;
+    }
+
+    if (last_thread) {
+        if (parent) {
+            assert(tg_lead);
+            sys->signalList.push_back(BasicSignal(tg_lead, parent, SIGCHLD));
+        }
+
+        /**
+         * Run though FD array of the exiting process and close all file
+         * descriptors except for the standard file descriptors.
+         * (The standard file descriptors are shared with gem5.)
+         */
+        for (int i = 0; i < p->fds->getSize(); i++) {
+            if ((*p->fds)[i])
+                p->fds->closeFDEntry(i);
         }
     }
-    DPRINTF(SyscallVerbose, "exit: FUTEX_WAKE, activated %d waiting "
-                            "thread contexts\n", wokenUp);
+
+    tc->halt();
+    return status;
 }
 
 SyscallReturn
 exitFunc(SyscallDesc *desc, int callnum, Process *p, ThreadContext *tc)
 {
-    if (p->system->numRunningContexts() == 1 && !p->childClearTID) {
-        // Last running free-parent context; exit simulator.
-        int index = 0;
-        exitSimLoop("target called exit()",
-                    p->getSyscallArg(tc, index) & 0xff);
-    } else {
-        if (p->childClearTID)
-            exitFutexWake(tc, p->childClearTID);
-        tc->halt();
-    }
-
-    return 1;
+    return exitImpl(desc, callnum, p, tc, false);
 }
 
-
 SyscallReturn
-exitGroupFunc(SyscallDesc *desc, int callnum, Process *process,
-              ThreadContext *tc)
+exitGroupFunc(SyscallDesc *desc, int callnum, Process *p, ThreadContext *tc)
 {
-    // halt all threads belonging to this process
-    for (auto i: process->contextIds) {
-        process->system->getThreadContext(i)->halt();
-    }
-
-    if (!process->system->numRunningContexts()) {
-        // all threads belonged to this process... exit simulator
-        int index = 0;
-        exitSimLoop("target called exit()",
-                    process->getSyscallArg(tc, index) & 0xff);
-    }
-
-    return 1;
+    return exitImpl(desc, callnum, p, tc, true);
 }
 
-
 SyscallReturn
 getpagesizeFunc(SyscallDesc *desc, int num, Process *p, ThreadContext *tc)
 {
diff --git a/src/sim/syscall_emul.hh b/src/sim/syscall_emul.hh
index 2380e4779..a0fb7e4ed 100644
--- a/src/sim/syscall_emul.hh
+++ b/src/sim/syscall_emul.hh
@@ -93,6 +93,7 @@
 #include "mem/page_table.hh"
 #include "params/Process.hh"
 #include "sim/emul_driver.hh"
+#include "sim/futex_map.hh"
 #include "sim/process.hh"
 #include "sim/syscall_debug_macros.hh"
 #include "sim/syscall_desc.hh"
@@ -304,79 +305,44 @@ SyscallReturn
 futexFunc(SyscallDesc *desc, int callnum, Process *process,
           ThreadContext *tc)
 {
-    int index_uaddr = 0;
-    int index_op = 1;
-    int index_val = 2;
-    int index_timeout = 3;
+    using namespace std;
 
-    uint64_t uaddr = process->getSyscallArg(tc, index_uaddr);
-    int op = process->getSyscallArg(tc, index_op);
-    int val = process->getSyscallArg(tc, index_val);
-    uint64_t timeout = process->getSyscallArg(tc, index_timeout);
-
-    std::map<uint64_t, std::list<ThreadContext *> * >
-        &futex_map = tc->getSystemPtr()->futexMap;
-
-    DPRINTF(SyscallVerbose, "futex: Address=%llx, op=%d, val=%d\n",
-            uaddr, op, val);
+    int index = 0;
+    Addr uaddr = process->getSyscallArg(tc, index);
+    int op = process->getSyscallArg(tc, index);
+    int val = process->getSyscallArg(tc, index);
 
+    /*
+     * Unsupported option that does not affect the correctness of the
+     * application. This is a performance optimization utilized by Linux.
+     */
     op &= ~OS::TGT_FUTEX_PRIVATE_FLAG;
 
-    if (op == OS::TGT_FUTEX_WAIT) {
-        if (timeout != 0) {
-            warn("futex: FUTEX_WAIT with non-null timeout unimplemented;"
-                 "we'll wait indefinitely");
-        }
+    FutexMap &futex_map = tc->getSystemPtr()->futexMap;
 
-        uint8_t *buf = new uint8_t[sizeof(int)];
-        tc->getMemProxy().readBlob((Addr)uaddr, buf, (int)sizeof(int));
-        int mem_val = *((int *)buf);
-        delete[] buf;
+    if (OS::TGT_FUTEX_WAIT == op) {
+        // Ensure futex system call accessed atomically.
+        BufferArg buf(uaddr, sizeof(int));
+        buf.copyIn(tc->getMemProxy());
+        int mem_val = *(int*)buf.bufferPtr();
 
-        if (val != mem_val) {
-            DPRINTF(SyscallVerbose, "futex: FUTEX_WAKE, read: %d, "
-                                    "expected: %d\n", mem_val, val);
+        /*
+         * The value in memory at uaddr is not equal with the expected val
+         * (a different thread must have changed it before the system call was
+         * invoked). In this case, we need to throw an error.
+         */
+        if (val != mem_val)
             return -OS::TGT_EWOULDBLOCK;
-        }
 
-        // Queue the thread context
-        std::list<ThreadContext *> * tcWaitList;
-        if (futex_map.count(uaddr)) {
-            tcWaitList = futex_map.find(uaddr)->second;
-        } else {
-            tcWaitList = new std::list<ThreadContext *>();
-            futex_map.insert(std::pair< uint64_t,
-                            std::list<ThreadContext *> * >(uaddr, tcWaitList));
-        }
-        tcWaitList->push_back(tc);
-        DPRINTF(SyscallVerbose, "futex: FUTEX_WAIT, suspending calling thread "
-                                "context on address 0x%lx\n", uaddr);
-        tc->suspend();
-        return 0;
-    } else if (op == OS::TGT_FUTEX_WAKE){
-        int wokenUp = 0;
-        std::list<ThreadContext *> * tcWaitList;
-        if (futex_map.count(uaddr)) {
-            tcWaitList = futex_map.find(uaddr)->second;
-            while (tcWaitList->size() > 0 && wokenUp < val) {
-                tcWaitList->front()->activate();
-                tcWaitList->pop_front();
-                wokenUp++;
-            }
-            if (tcWaitList->empty()) {
-                futex_map.erase(uaddr);
-                delete tcWaitList;
-            }
-        }
-        DPRINTF(SyscallVerbose, "futex: FUTEX_WAKE, activated %d waiting "
-                                "thread context on address 0x%lx\n",
-                                wokenUp, uaddr);
-        return wokenUp;
-    } else {
-        warn("futex: op %d is not implemented, just returning...", op);
+        futex_map.suspend(uaddr, process->tgid(), tc);
+
         return 0;
+    } else if (OS::TGT_FUTEX_WAKE == op) {
+        return futex_map.wakeup(uaddr, process->tgid(), val);
     }
 
+    warn("futex: op %d not implemented; ignoring.", op);
+    return -ENOSYS;
 }
 
 
@@ -1319,6 +1285,13 @@ cloneFunc(SyscallDesc *desc, int callnum, Process *p, ThreadContext *tc)
     cp->initState();
     p->clone(tc, ctc, cp, flags);
 
+    if (flags & OS::TGT_CLONE_THREAD) {
+        delete cp->sigchld;
+        cp->sigchld = p->sigchld;
+    } else if (flags & OS::TGT_SIGCHLD) {
+        *cp->sigchld = true;
+    }
+
     if (flags & OS::TGT_CLONE_CHILD_SETTID) {
         BufferArg ctidBuf(ctidPtr, sizeof(long));
         long *ctid = (long *)ctidBuf.bufferPtr();
diff --git a/src/sim/system.hh b/src/sim/system.hh
index 7584cebc9..c3c178dbd 100644
--- a/src/sim/system.hh
+++ b/src/sim/system.hh
@@ -48,6 +48,7 @@
 #define __SYSTEM_HH__
 
 #include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -57,10 +58,11 @@
 #include "config/the_isa.hh"
 #include "enums/MemoryMode.hh"
 #include "mem/mem_object.hh"
+#include "mem/physical.hh"
 #include "mem/port.hh"
 #include "mem/port_proxy.hh"
-#include "mem/physical.hh"
 #include "params/System.hh"
+#include "sim/futex_map.hh"
 #include "sim/se_signal.hh"
 
 /**
@@ -69,6 +71,7 @@
  */
 #if THE_ISA != NULL_ISA
 #include "cpu/pc_event.hh"
+
 #endif
 
 class BaseRemoteGDB;
@@ -549,8 +552,7 @@ class System : public MemObject
 
     static void printSystems();
 
-    // For futex system call
-    std::map<uint64_t, std::list<ThreadContext *> * > futexMap;
+    FutexMap futexMap;
 
     static const int maxPID = 32768;