From 3e438d5f6a079cf045f7fff73e41431abe8e7b0c Mon Sep 17 00:00:00 2001 From: Nikos Nikoleris Date: Wed, 29 May 2019 11:50:02 +0100 Subject: tests: Add support for downloaded archive fixtures This changes add support for specifying fixtures that download archives and uncompress them to the desired directory. Change-Id: Ib3f6ee111b8d6130200507cbd170ecaf9fb39445 Signed-off-by: Nikos Nikoleris Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18988 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power --- tests/gem5/cpu_tests/test.py | 11 +++-- tests/gem5/fixture.py | 78 ++++++++++++++++++++++++------------ tests/gem5/hello_se/test_hello_se.py | 9 +++-- tests/gem5/m5_util/test_exit.py | 9 +++-- 4 files changed, 72 insertions(+), 35 deletions(-) (limited to 'tests') diff --git a/tests/gem5/cpu_tests/test.py b/tests/gem5/cpu_tests/test.py index f34b23d07..58925de25 100644 --- a/tests/gem5/cpu_tests/test.py +++ b/tests/gem5/cpu_tests/test.py @@ -42,23 +42,26 @@ valid_isas = { } +base_path = joinpath(absdirpath(__file__), 'benchmarks', 'bin') +base_url = 'http://gem5.org/dist/current/gem5/cpu_tests/benchmarks/bin/' for isa in valid_isas: - bm_dir = joinpath('gem5/cpu_tests/benchmarks/bin/', isa) + path = joinpath(base_path, isa) for workload in workloads: ref_path = joinpath(getcwd(), 'ref', workload) verifiers = ( verifier.MatchStdout(ref_path), ) - workload_binary = DownloadedProgram(bm_dir, workload) - workload_path = workload_binary.path + url = base_url + isa + '/' + workload + workload_binary = DownloadedProgram(url, path, workload) + binary = joinpath(workload_binary.path, workload) for cpu in valid_isas[isa]: gem5_verify_config( name='cpu_test_{}_{}'.format(cpu,workload), verifiers=verifiers, config=joinpath(getcwd(), 'run.py'), - config_args=['--cpu={}'.format(cpu), workload_path], + config_args=['--cpu={}'.format(cpu), binary], valid_isas=(isa.upper(),), fixtures=[workload_binary] ) diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py index e84b89f69..d39296efd 100644 --- a/tests/gem5/fixture.py +++ b/tests/gem5/fixture.py @@ -37,11 +37,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Authors: Sean Wilson +# Nikos Nikoleris import os import tempfile import shutil import threading +import urllib +import urllib2 from testlib.fixture import Fixture from testlib.config import config, constants @@ -239,44 +242,43 @@ class TestProgram(MakeTarget): elif not os.path.exists(self.path): super(MakeTarget, self).setup() -class DownloadedProgram(Fixture): +class DownloadedProgram(UniqueFixture): """ Like TestProgram, but checks the version in the gem5 binary repository and downloads an updated version if it is needed. """ - urlbase = "http://gem5.org/dist/current/" - def __init__(self, path, program, **kwargs): + def __new__(cls, url, path, filename): + target = joinpath(path, filename) + return super(DownloadedProgram, cls).__new__(cls, target) + + def _init(self, url, path, filename, **kwargs): """ + url: string + The url of the archive path: string - The path to the directory containing the binary relative to - $GEM5_BASE/tests - program: string - The name of the binary file + The absolute path of the directory containing the archive + filename: string + The name of the archive """ - super(DownloadedProgram, self).__init__("download-" + program, - build_once=True, **kwargs) - self.program_dir = path - relative_path = joinpath(self.program_dir, program) - self.url = self.urlbase + relative_path - self.path = os.path.realpath( - joinpath(absdirpath(__file__), '../', relative_path) - ) + self.url = url + self.path = path + self.filename = joinpath(path, filename) + self.name = "Downloaded:" + self.filename def _download(self): - import urllib import errno log.test_log.debug("Downloading " + self.url + " to " + self.path) - if not os.path.exists(self.program_dir): + if not os.path.exists(self.path): try: - os.makedirs(self.program_dir) + os.makedirs(self.path) except OSError as e: if e.errno != errno.EEXIST: raise - urllib.urlretrieve(self.url, self.path) + urllib.urlretrieve(self.url, self.filename) def _getremotetime(self): - import urllib2, datetime, time + import datetime, time import _strptime # Needed for python threading bug u = urllib2.urlopen(self.url) @@ -284,18 +286,44 @@ class DownloadedProgram(Fixture): u.info().getheaders("Last-Modified")[0], "%a, %d %b %Y %X GMT").timetuple()) - def setup(self, testitem): - import urllib2 + def _setup(self, testitem): # Check to see if there is a file downloaded - if not os.path.exists(self.path): + if not os.path.exists(self.filename): + self._download() + else: + try: + t = self._getremotetime() + except urllib2.URLError: + # Problem checking the server, use the old files. + log.test_log.debug("Could not contact server. Binaries may be old.") + return + # If the server version is more recent, download it + if t > os.path.getmtime(self.filename): + self._download() + +class DownloadedArchive(DownloadedProgram): + """ Like TestProgram, but checks the version in the gem5 binary repository + and downloads an updated version if it is needed. + """ + + def _extract(self): + import tarfile + with tarfile.open(self.filename) as tf: + tf.extractall(self.path) + + def _setup(self, testitem): + # Check to see if there is a file downloaded + if not os.path.exists(self.filename): self._download() + self._extract() else: try: t = self._getremotetime() except urllib2.URLError: # Problem checking the server, use the old files. - log.debug("Could not contact server. Binaries may be old.") + log.test_log.debug("Could not contact server. Binaries may be old.") return # If the server version is more recent, download it - if t > os.path.getmtime(self.path): + if t > os.path.getmtime(self.filename): self._download() + self._extract() diff --git a/tests/gem5/hello_se/test_hello_se.py b/tests/gem5/hello_se/test_hello_se.py index 501796207..cc62efb71 100644 --- a/tests/gem5/hello_se/test_hello_se.py +++ b/tests/gem5/hello_se/test_hello_se.py @@ -36,11 +36,14 @@ test_progs = { 'arm': ('hello64-static', 'hello32-static'), } +urlbase = 'http://gem5.org/dist/current/test-progs/hello/bin/' for isa in test_progs: for binary in test_progs[isa]: import os - path = os.path.join('test-progs', 'hello', 'bin', isa, 'linux') - hello_program = DownloadedProgram(path, binary) + url = urlbase + isa + '/linux/' + binary + path = joinpath(absdirpath(__file__), '..', 'test-progs', 'hello', + 'bin', isa, 'linux') + hello_program = DownloadedProgram(url, path, binary) ref_path = joinpath(getcwd(), 'ref') @@ -53,6 +56,6 @@ for isa in test_progs: fixtures=(hello_program,), verifiers=verifiers, config=joinpath(config.base_dir, 'configs', 'example','se.py'), - config_args=['--cmd', hello_program.path], + config_args=['--cmd', joinpath(path, binary)], valid_isas=(isa.upper(),), ) diff --git a/tests/gem5/m5_util/test_exit.py b/tests/gem5/m5_util/test_exit.py index a766db420..3c9951254 100644 --- a/tests/gem5/m5_util/test_exit.py +++ b/tests/gem5/m5_util/test_exit.py @@ -37,8 +37,11 @@ m5_exit_regex = re.compile( r'Exiting @ tick \d* because m5_exit instruction encountered' ) -test_program = DownloadedProgram('test-progs/m5-exit/bin/x86/linux/',\ - 'm5_exit') +path = joinpath(absdirpath(__file__), '..', + 'test-progs', 'hello', 'bin', 'x86', 'linux') +filename = 'm5_exit' +url = 'http://gem5.org/dist/current/test-progs/m5-exit/bin/x86/linux/m5_exit' +test_program = DownloadedProgram(url, path, filename) a = verifier.MatchRegex(m5_exit_regex) gem5_verify_config( @@ -46,6 +49,6 @@ gem5_verify_config( verifiers=[a], fixtures=(test_program,), config=os.path.join(config.base_dir, 'configs', 'example','se.py'), - config_args=['--cmd', test_program.path], + config_args=['--cmd', joinpath(test_program.path, filename)], valid_isas=('X86',) ) -- cgit v1.2.3