scons: Switch from "guards" to "tags" on source files.

Tags are just arbitrary strings which are attached to source files
which mark them as having some property. By default, all source files
have the "gem5 lib" tag added to them which marks them as part of the
gem5 library, the primary component of the gem5 binary but also a
seperable component for use in, for example, system C.

The tags can be completely overridden by setting the "tags" parameter
on Source, etc., functions, and can be augmented by setting "add_tags"
which are tags that will be added, or alternatively additional tags.
It's possible to specify both, in which case the tags will be set to
the union of tags and add_tags. add_tags is supposed to be a way to
add extra tags to the default without actually overriding the default.
Both tags and add_tags can be a list/tuple/etc of tags, or a single
string which will be converted into a set internally.

Other existing tags include:
1. "python" for files that need or are used with python and are
   excluded when the --without-python option is set
2. "main" for the file(s) which implement the gem5 binary's main
   function.
3. The name of a unit test to group its files together.
4. Tags which group source files for partial linking.

By grouping the "tags" into a single parameter instead of taking all
extra parameters as tags, the extra parameters can, in the future, be
passed to the underlying scons environment. Also, the tags are either
present or not. With guards, they could be present and True, present
and False, or not present at all.

Change-Id: I6d0404211a393968df66f7eddfe019897b6573a2
Reviewed-on: https://gem5-review.googlesource.com/5822
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Maintainer: Andreas Sandberg <andreas.sandberg@arm.com>

4 files changed, 94 insertions, 101 deletions

diff --git a/src/SConscript b/src/SConscript
index 6c3f22019..da8476d6c 100755
--- a/src/SConscript
+++ b/src/SConscript
@@ -59,56 +59,59 @@ from m5.util import code_formatter, compareVersions
 ########################################################################
 # Code for adding source files of various types
 #
-# When specifying a source file of some type, a set of guards can be
-# specified for that file.  When get() is used to find the files, if
-# get specifies a set of filters, only files that match those filters
-# will be accepted (unspecified filters on files are assumed to be
-# false).  Current filters are:
-#     main -- specifies the gem5 main() function
-#     skip_lib -- do not put this file into the gem5 library
-#     skip_no_python -- do not put this file into a no_python library
-#       as it embeds compiled Python
-#     <unittest> -- unit tests use filters based on the unit test name
-#
-# A parent can now be specified for a source file and default filter
-# values will be retrieved recursively from parents (children override
-# parents).
-#
-def guarded_source_iterator(sources, **guards):
-    '''Iterate over a set of sources, gated by a set of guards.'''
-    for src in sources:
-        for flag,value in guards.iteritems():
-            # if the flag is found and has a different value, skip
-            # this file
-            if src.all_guards.get(flag, False) != value:
-                break
-        else:
-            yield src
+# When specifying a source file of some type, a set of tags can be
+# specified for that file.
+
+class SourceList(list):
+    def with_tags_that(self, predicate):
+        '''Return a list of sources with tags that satisfy a predicate.'''
+        def match(source):
+            return predicate(source.tags)
+        return SourceList(filter(match, self))
+
+    def with_any_tags(self, *tags):
+        '''Return a list of sources with any of the supplied tags.'''
+        return self.with_tags_that(lambda stags: len(tags & stags) > 0)
+
+    def with_all_tags(self, *tags):
+        '''Return a list of sources with all of the supplied tags.'''
+        return self.with_tags_that(lambda stags: tags <= stags)
+
+    def with_tag(self, tag):
+        '''Return a list of sources with the supplied tag.'''
+        return self.with_tags_that(lambda stags: tag in stags)
+
+    def without_tags(self, *tags):
+        '''Return a list of sources without any of the supplied tags.'''
+        return self.with_tags_that(lambda stags: len(tags & stags) == 0)
+
+    def without_tag(self, tag):
+        '''Return a list of sources with the supplied tag.'''
+        return self.with_tags_that(lambda stags: tag not in stags)
 
 class SourceMeta(type):
     '''Meta class for source files that keeps track of all files of a
-    particular type and has a get function for finding all functions
-    of a certain type that match a set of guards'''
+    particular type.'''
     def __init__(cls, name, bases, dict):
         super(SourceMeta, cls).__init__(name, bases, dict)
-        cls.all = []
-
-    def get(cls, **guards):
-        '''Find all files that match the specified guards.  If a source
-        file does not specify a flag, the default is False'''
-        for s in guarded_source_iterator(cls.all, **guards):
-            yield s
+        cls.all = SourceList()
 
 class SourceFile(object):
     '''Base object that encapsulates the notion of a source file.
     This includes, the source node, target node, various manipulations
-    of those.  A source file also specifies a set of guards which
-    describing which builds the source file applies to.  A parent can
-    also be specified to get default guards from'''
+    of those.  A source file also specifies a set of tags which
+    describing arbitrary properties of the source file.'''
     __metaclass__ = SourceMeta
-    def __init__(self, source, parent=None, **guards):
-        self.guards = guards
-        self.parent = parent
+    def __init__(self, source, tags=None, add_tags=None):
+        if tags is None:
+            tags='gem5 lib'
+        if isinstance(tags, basestring):
+            tags = set([tags])
+        if isinstance(add_tags, basestring):
+            add_tags = set([add_tags])
+        if add_tags:
+            tags = tags | add_tags
+        self.tags = set(tags)
 
         tnode = source
         if not isinstance(source, SCons.Node.FS.File):
@@ -142,16 +145,6 @@ class SourceFile(object):
 
         return self.basename[:index], self.basename[index+1:]
 
-    @property
-    def all_guards(self):
-        '''find all guards for this object getting default values
-        recursively from its parents'''
-        guards = {}
-        if self.parent:
-            guards.update(self.parent.guards)
-        guards.update(self.guards)
-        return guards
-
     def __lt__(self, other): return self.filename < other.filename
     def __le__(self, other): return self.filename <= other.filename
     def __gt__(self, other): return self.filename > other.filename
@@ -168,24 +161,31 @@ class SourceFile(object):
 
 
 class Source(SourceFile):
-    current_group = None
-    source_groups = { None : [] }
+    ungrouped_tag = 'No link group'
+    source_groups = set()
+
+    _current_group_tag = ungrouped_tag
+
+    @staticmethod
+    def link_group_tag(group):
+        return 'link group: %s' % group
 
     @classmethod
     def set_group(cls, group):
-        if not group in Source.source_groups:
-            Source.source_groups[group] = []
-        Source.current_group = group
+        new_tag = Source.link_group_tag(group)
+        Source._current_group_tag = new_tag
+        Source.source_groups.add(group)
 
-    '''Add a c/c++ source file to the build'''
-    def __init__(self, source, Werror=True, **guards):
-        '''specify the source file, and any guards'''
-        super(Source, self).__init__(source, **guards)
+    def _add_link_group_tag(self):
+        self.tags.add(Source._current_group_tag)
 
+    '''Add a c/c++ source file to the build'''
+    def __init__(self, source, tags=None, add_tags=None, Werror=True):
+        '''specify the source file, and any tags'''
+        super(Source, self).__init__(source, tags, add_tags)
+        self._add_link_group_tag()
         self.Werror = Werror
 
-        Source.source_groups[Source.current_group].append(self)
-
 class PySource(SourceFile):
     '''Add a python source file to the named package'''
     invalid_sym_char = re.compile('[^A-z0-9_]')
@@ -193,9 +193,9 @@ class PySource(SourceFile):
     tnodes = {}
     symnames = {}
 
-    def __init__(self, package, source, **guards):
-        '''specify the python package, the source file, and any guards'''
-        super(PySource, self).__init__(source, **guards)
+    def __init__(self, package, source, tags=None, add_tags=None):
+        '''specify the python package, the source file, and any tags'''
+        super(PySource, self).__init__(source, tags, add_tags)
 
         modname,ext = self.extname
         assert ext == 'py'
@@ -235,10 +235,10 @@ class SimObject(PySource):
     fixed = False
     modnames = []
 
-    def __init__(self, source, **guards):
-        '''Specify the source file and any guards (automatically in
+    def __init__(self, source, tags=None, add_tags=None):
+        '''Specify the source file and any tags (automatically in
         the m5.objects package)'''
-        super(SimObject, self).__init__('m5.objects', source, **guards)
+        super(SimObject, self).__init__('m5.objects', source, tags, add_tags)
         if self.fixed:
             raise AttributeError, "Too late to call SimObject now."
 
@@ -247,9 +247,9 @@ class SimObject(PySource):
 class ProtoBuf(SourceFile):
     '''Add a Protocol Buffer to build'''
 
-    def __init__(self, source, **guards):
-        '''Specify the source file, and any guards'''
-        super(ProtoBuf, self).__init__(source, **guards)
+    def __init__(self, source, tags=None, add_tags=None):
+        '''Specify the source file, and any tags'''
+        super(ProtoBuf, self).__init__(source, tags, add_tags)
 
         # Get the file name and the extension
         modname,ext = self.extname
@@ -267,14 +267,12 @@ class UnitTest(object):
     def __init__(self, target, *sources, **kwargs):
         '''Specify the target name and any sources.  Sources that are
         not SourceFiles are evalued with Source().  All files are
-        guarded with a guard of the same name as the UnitTest
-        target.'''
+        tagged with the name of the UnitTest target.'''
 
-        srcs = []
+        srcs = SourceList()
         for src in sources:
             if not isinstance(src, SourceFile):
-                src = Source(src, skip_lib=True)
-            src.guards[target] = True
+                src = Source(src, tags=str(target))
             srcs.append(src)
 
         self.sources = srcs
@@ -770,7 +768,7 @@ if env['HAVE_PROTOBUF']:
                                Transform("PROTOC")))
 
         # Add the C++ source file
-        Source(proto.cc_file, **proto.guards)
+        Source(proto.cc_file, tags=proto.tags)
 elif ProtoBuf.all:
     print 'Got protobuf to build, but lacks support!'
     Exit(1)
@@ -936,7 +934,7 @@ EmbeddedPython embedded_${sym}(
 for source in PySource.all:
     env.Command(source.cpp, source.tnode,
                 MakeAction(embedPyFile, Transform("EMBED PY")))
-    Source(source.cpp, skip_no_python=True)
+    Source(source.cpp, tags=source.tags, add_tags='python')
 
 ########################################################################
 #
@@ -945,7 +943,7 @@ for source in PySource.all:
 #
 
 # List of constructed environments to pass back to SConstruct
-date_source = Source('base/date.cc', skip_lib=True)
+date_source = Source('base/date.cc', tags=[])
 
 # Function to create a new build environment as clone of current
 # environment 'env' with modified object suffix and optional stripped
@@ -1028,28 +1026,24 @@ def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
 
         return obj
 
-    lib_guards = {'main': False, 'skip_lib': False}
+    lib_sources = Source.all.with_tag('gem5 lib')
 
     # Without Python, leave out all Python content from the library
     # builds.  The option doesn't affect gem5 built as a program
     if GetOption('without_python'):
-        lib_guards['skip_no_python'] = False
+        lib_sources = lib_sources.without_tag('python')
 
     static_objs = []
     shared_objs = []
-    for s in guarded_source_iterator(Source.source_groups[None], **lib_guards):
+
+    for s in lib_sources.with_tag(Source.ungrouped_tag):
         static_objs.append(make_obj(s, True))
         shared_objs.append(make_obj(s, False))
 
     partial_objs = []
-    for group, all_srcs in Source.source_groups.iteritems():
-        # If these are the ungrouped source files, skip them.
-        if not group:
-            continue
 
-        # Get a list of the source files compatible with the current guards.
-        srcs = [ s for s in guarded_source_iterator(all_srcs, **lib_guards) ]
-        # If there aren't any left, skip this group.
+    for group in Source.source_groups:
+        srcs = lib_sources.with_tag(Source.link_group_tag(group))
         if not srcs:
             continue
 
@@ -1087,11 +1081,10 @@ def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
     shared_lib = new_env.SharedLibrary(libname, shared_objs)
 
     # Now link a stub with main() and the static library.
-    main_objs = [ make_obj(s, True) for s in Source.get(main=True) ]
+    main_objs = [ make_obj(s, True) for s in Source.all.with_tag('main') ]
 
     for test in UnitTest.all:
-        flags = { test.target : True }
-        test_sources = Source.get(**flags)
+        test_sources = Source.all.with_tag(str(test.target))
         test_objs = [ make_obj(s, static=True) for s in test_sources ]
         if test.main:
             test_objs += main_objs
diff --git a/src/python/SConscript b/src/python/SConscript
index 1e7746930..cfd2afeff 100644
--- a/src/python/SConscript
+++ b/src/python/SConscript
@@ -64,8 +64,8 @@ PySource('m5.util', 'm5/util/pybind.py')
 PySource('m5.internal', 'm5/internal/__init__.py')
 PySource('m5.internal', 'm5/internal/params.py')
 
-Source('pybind11/core.cc', skip_no_python=True)
-Source('pybind11/debug.cc', skip_no_python=True)
-Source('pybind11/event.cc', skip_no_python=True)
-Source('pybind11/pyobject.cc', skip_no_python=True)
-Source('pybind11/stats.cc', skip_no_python=True)
+Source('pybind11/core.cc', add_tags='python')
+Source('pybind11/debug.cc', add_tags='python')
+Source('pybind11/event.cc', add_tags='python')
+Source('pybind11/pyobject.cc', add_tags='python')
+Source('pybind11/stats.cc', add_tags='python')
diff --git a/src/sim/SConscript b/src/sim/SConscript
index a3d5464e8..996a3324a 100644
--- a/src/sim/SConscript
+++ b/src/sim/SConscript
@@ -48,12 +48,12 @@ Source('cxx_config.cc')
 Source('cxx_manager.cc')
 Source('cxx_config_ini.cc')
 Source('debug.cc')
-Source('py_interact.cc', skip_no_python=True)
+Source('py_interact.cc', add_tags='python')
 Source('eventq.cc')
 Source('global_event.cc')
-Source('init.cc', skip_no_python=True)
+Source('init.cc', add_tags='python')
 Source('init_signals.cc')
-Source('main.cc', main=True, skip_lib=True)
+Source('main.cc', tags='main')
 Source('root.cc')
 Source('serialize.cc')
 Source('drain.cc')
@@ -63,7 +63,7 @@ Source('sub_system.cc')
 Source('ticked_object.cc')
 Source('simulate.cc')
 Source('stat_control.cc')
-Source('stat_register.cc', skip_no_python=True)
+Source('stat_register.cc', add_tags='python')
 Source('clock_domain.cc')
 Source('voltage_domain.cc')
 Source('se_signal.cc')
diff --git a/src/unittest/SConscript b/src/unittest/SConscript
index 1f723ed2a..70e3c2f95 100644
--- a/src/unittest/SConscript
+++ b/src/unittest/SConscript
@@ -45,7 +45,7 @@ UnitTest('refcnttest', 'refcnttest.cc')
 UnitTest('strnumtest', 'strnumtest.cc')
 UnitTest('trietest', 'trietest.cc')
 
-stattest_py = PySource('m5', 'stattestmain.py', skip_lib=True)
+stattest_py = PySource('m5', 'stattestmain.py', tags='stattest')
 UnitTest('stattest', 'stattest.cc', stattest_py, main=True)
 
 UnitTest('symtest', 'symtest.cc')