diff options
Diffstat (limited to 'tools/coverage/coverage_report.py')
| -rwxr-xr-x | tools/coverage/coverage_report.py | 390 |
1 files changed, 0 insertions, 390 deletions
diff --git a/tools/coverage/coverage_report.py b/tools/coverage/coverage_report.py deleted file mode 100755 index bb2d4e1396..0000000000 --- a/tools/coverage/coverage_report.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017 The PDFium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Generates a coverage report for given binaries using llvm-gcov & lcov. - -Requires llvm-cov 3.5 or later. -Requires lcov 1.11 or later. -Requires that 'use_coverage = true' is set in args.gn. -""" - -import argparse -from collections import namedtuple -import pprint -import os -import re -import subprocess -import sys - - -# 'binary' is the file that is to be run for the test. -# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus -# requires special handling. -TestSpec = namedtuple('TestSpec', 'binary, use_test_runner') - -# All of the coverage tests that the script knows how to run. -COVERAGE_TESTS = { - 'pdfium_unittests': TestSpec('pdfium_unittests', False), - 'pdfium_embeddertests': TestSpec('pdfium_embeddertests', False), - 'corpus_tests': TestSpec('run_corpus_tests.py', True), - 'javascript_tests': TestSpec('run_javascript_tests.py', True), - 'pixel_tests': TestSpec('run_pixel_tests.py', True), -} - -# Coverage tests that are known to take a long time to run, so are not in the -# default set. The user must either explicitly invoke these tests or pass in -# --slow. -SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests'] - -class CoverageExecutor(object): - - def __init__(self, parser, args): - """Initialize executor based on the current script environment - - Args: - parser: argparse.ArgumentParser for handling improper inputs. - args: Dictionary of arguments passed into the calling script. - """ - self.dry_run = args['dry_run'] - self.verbose = args['verbose'] - - llvm_cov = self.determine_proper_llvm_cov() - if not llvm_cov: - print 'Unable to find appropriate llvm-cov to use' - sys.exit(1) - self.lcov_env = os.environ - self.lcov_env['LLVM_COV_BIN'] = llvm_cov - - self.lcov = self.determine_proper_lcov() - if not self.lcov: - print 'Unable to find appropriate lcov to use' - sys.exit(1) - - self.coverage_files = set() - self.source_directory = args['source_directory'] - if not os.path.isdir(self.source_directory): - parser.error("'%s' needs to be a directory" % self.source_directory) - - self.build_directory = args['build_directory'] - if not os.path.isdir(self.build_directory): - parser.error("'%s' needs to be a directory" % self.build_directory) - - self.coverage_tests = self.calculate_coverage_tests(args) - if not self.coverage_tests: - parser.error( - 'No valid tests in set to be run. This is likely due to bad command ' - 'line arguments') - - if not self.boolean_gn_arg('use_coverage'): - parser.error( - 'use_coverage does not appear to be set to true for build, but is ' - 'needed') - - self.use_goma = self.boolean_gn_arg('use_goma') - - self.output_directory = args['output_directory'] - if not os.path.exists(self.output_directory): - if not self.dry_run: - os.makedirs(self.output_directory) - elif not os.path.isdir(self.output_directory): - parser.error('%s exists, but is not a directory' % self.output_directory) - self.coverage_totals_path = os.path.join(self.output_directory, - 'pdfium_totals.info') - - def boolean_gn_arg(self, arg): - """Extract the value of a boolean flag in args.gn""" - cwd = os.getcwd() - os.chdir(self.build_directory) - gn_args_output = self.check_output( - ['gn', 'args', '.', '--list=%s' % arg, '--short']) - os.chdir(cwd) - arg_match_output = re.match('%s = (.*)' % arg, gn_args_output).group(1) - if self.verbose: - print "Found '%s' for value of %s" % (arg_match_output, arg) - return arg_match_output == 'true' - - def check_output(self, args, dry_run=False, env=None): - """Dry run aware wrapper of subprocess.check_output()""" - if dry_run: - print "Would have run '%s'" % ' '.join(args) - return '' - - output = subprocess.check_output(args, env=env) - - if self.verbose: - print "check_output(%s) returned '%s'" % (args, output) - return output - - def call(self, args, dry_run=False, env=None): - """Dry run aware wrapper of subprocess.call()""" - if dry_run: - print "Would have run '%s'" % ' '.join(args) - return 0 - - output = subprocess.call(args, env=env) - - if self.verbose: - print 'call(%s) returned %s' % (args, output) - return output - - def call_lcov(self, args, dry_run=False, needs_directory=True): - """Wrapper to call lcov that adds appropriate arguments as needed.""" - lcov_args = [ - self.lcov, '--config-file', - os.path.join(self.source_directory, 'tools', 'coverage', 'lcovrc'), - '--gcov-tool', - os.path.join(self.source_directory, 'tools', 'coverage', 'llvm-gcov') - ] - if needs_directory: - lcov_args.extend(['--directory', self.source_directory]) - if not self.verbose: - lcov_args.append('--quiet') - lcov_args.extend(args) - return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env) - - def calculate_coverage_tests(self, args): - """Determine which tests should be run.""" - testing_tools_directory = os.path.join(self.source_directory, 'testing', - 'tools') - coverage_tests = {} - for name in COVERAGE_TESTS.keys(): - test_spec = COVERAGE_TESTS[name] - if test_spec.use_test_runner: - binary_path = os.path.join(testing_tools_directory, test_spec.binary) - else: - binary_path = os.path.join(self.build_directory, test_spec.binary) - coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner) - - if args['tests']: - return {name: spec - for name, spec in coverage_tests.iteritems() if name in args['tests']} - elif not args['slow']: - return {name: spec - for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS} - else: - return coverage_tests - - def find_acceptable_binary(self, binary_name, version_regex, - min_major_version, min_minor_version): - """Find the newest version of binary that meets the min version.""" - min_version = (min_major_version, min_minor_version) - parsed_versions = {} - # When calling Bash builtins like this the command and arguments must be - # passed in as a single string instead of as separate list members. - potential_binaries = self.check_output( - ['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines() - for binary in potential_binaries: - if self.verbose: - print 'Testing llvm-cov binary, %s' % binary - # Assuming that scripts that don't respond to --version correctly are not - # valid binaries and just happened to get globbed in. This is true for - # lcov and llvm-cov - try: - version_output = self.check_output([binary, '--version']).splitlines() - except subprocess.CalledProcessError: - if self.verbose: - print '--version returned failure status 1, so ignoring' - continue - - for line in version_output: - matcher = re.match(version_regex, line) - if matcher: - parsed_version = (int(matcher.group(1)), int(matcher.group(2))) - if parsed_version >= min_version: - parsed_versions[parsed_version] = binary - break - - if not parsed_versions: - return None - return parsed_versions[max(parsed_versions)] - - def determine_proper_llvm_cov(self): - """Find a version of llvm_cov that will work with the script.""" - version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*') - return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5) - - def determine_proper_lcov(self): - """Find a version of lcov that will work with the script.""" - version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*') - return self.find_acceptable_binary('lcov', version_regex, 1, 11) - - def build_binaries(self): - """Build all the binaries that are going to be needed for coverage - generation.""" - call_args = ['ninja'] - if self.use_goma: - call_args.extend(['-j', '250']) - call_args.extend(['-C', self.build_directory]) - return self.call(call_args, dry_run=self.dry_run) == 0 - - def generate_coverage(self, name, spec): - """Generate the coverage data for a test - - Args: - name: Name associated with the test to be run. This is used as a label - in the coverage data, so should be unique across all of the tests - being run. - spec: Tuple containing the path to the binary to run, and if this test - uses test_runner.py. - """ - if self.verbose: - print "Generating coverage for test '%s', using data '%s'" % (name, spec) - if not os.path.exists(spec.binary): - print('Unable to generate coverage for %s, since it appears to not exist' - ' @ %s') % (name, spec.binary) - return False - - if self.call_lcov(['--zerocounters'], dry_run=self.dry_run): - print 'Unable to clear counters for %s' % name - return False - - binary_args = [spec.binary] - if spec.use_test_runner: - # Test runner performs multi-threading in the wrapper script, not the test - # binary, so need -j 1, otherwise multiple processes will be writing to - # the code coverage files, invalidating results. - # TODO(pdfium:811): Rewrite how test runner tests work, so that they can - # be run in multi-threaded mode. - binary_args.extend(['-j', '1', '--build-dir', self.build_directory]) - if self.call(binary_args, dry_run=self.dry_run) and self.verbose: - print('Running %s appears to have failed, which might affect ' - 'results') % spec.binary - - output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name) - if self.call_lcov( - ['--capture', '--test-name', name, '--output-file', output_raw_path], - dry_run=self.dry_run): - print 'Unable to capture coverage data for %s' % name - return False - - output_filtered_path = os.path.join(self.output_directory, - '%s_filtered.info' % name) - output_filters = [ - '/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp', - '*_embeddertest.cpp' - ] - if self.call_lcov( - ['--remove', output_raw_path] + output_filters + - ['--output-file', output_filtered_path], - dry_run=self.dry_run, - needs_directory=False): - print 'Unable to filter coverage data for %s' % name - return False - - self.coverage_files.add(output_filtered_path) - return True - - def merge_coverage(self): - """Merge all of the coverage data sets into one for report generation.""" - merge_args = [] - for coverage_file in self.coverage_files: - merge_args.extend(['--add-tracefile', coverage_file]) - - merge_args.extend(['--output-file', self.coverage_totals_path]) - return self.call_lcov( - merge_args, dry_run=self.dry_run, needs_directory=False) == 0 - - def generate_report(self): - """Produce HTML coverage report based on combined coverage data set.""" - config_file = os.path.join( - self.source_directory, 'tools', 'coverage', 'lcovrc') - - lcov_args = ['genhtml', - '--config-file', config_file, - '--legend', - '--demangle-cpp', - '--show-details', - '--prefix', self.source_directory, - '--ignore-errors', - 'source', self.coverage_totals_path, - '--output-directory', self.output_directory] - return self.call(lcov_args, dry_run=self.dry_run) == 0 - - def run(self): - """Setup environment, execute the tests and generate coverage report""" - if not self.build_binaries(): - print 'Failed to successfully build binaries' - return False - - for name in self.coverage_tests.keys(): - if not self.generate_coverage(name, self.coverage_tests[name]): - print 'Failed to successfully generate coverage data' - return False - - if not self.merge_coverage(): - print 'Failed to successfully merge generated coverage data' - return False - - if not self.generate_report(): - print 'Failed to successfully generated coverage report' - return False - - return True - - -def main(): - parser = argparse.ArgumentParser() - parser.formatter_class = argparse.RawDescriptionHelpFormatter - parser.description = ('Generates a coverage report for given binaries using ' - 'llvm-cov & lcov.\n\n' - 'Requires llvm-cov 3.5 or later.\n' - 'Requires lcov 1.11 or later.\n\n' - 'By default runs pdfium_unittests and ' - 'pdfium_embeddertests. If --slow is passed in then all ' - 'tests will be run. If any of the tests are specified ' - 'on the command line, then only those will be run.') - parser.add_argument( - '-s', - '--source_directory', - help='Location of PDFium source directory, defaults to CWD', - default=os.getcwd()) - build_default = os.path.join('out', 'Coverage') - parser.add_argument( - '-b', - '--build_directory', - help= - 'Location of PDFium build directory with coverage enabled, defaults to ' - '%s under CWD' % build_default, - default=os.path.join(os.getcwd(), build_default)) - output_default = 'coverage_report' - parser.add_argument( - '-o', - '--output_directory', - help='Location to write out coverage report to, defaults to %s under CWD ' - % output_default, - default=os.path.join(os.getcwd(), output_default)) - parser.add_argument( - '-n', - '--dry-run', - help='Output commands instead of executing them', - action='store_true') - parser.add_argument( - '-v', - '--verbose', - help='Output additional diagnostic information', - action='store_true') - parser.add_argument( - '--slow', - help='Run all tests, even those known to take a long time. Ignored if ' - 'specific tests are passed in.', - action='store_true') - parser.add_argument( - 'tests', - help='Tests to be run, defaults to all. Valid entries are %s' % - COVERAGE_TESTS.keys(), - nargs='*') - - args = vars(parser.parse_args()) - if args['verbose']: - pprint.pprint(args) - - executor = CoverageExecutor(parser, args) - if executor.run(): - return 0 - return 1 - - -if __name__ == '__main__': - sys.exit(main()) |