summaryrefslogtreecommitdiff
path: root/testing/tools/coverage/coverage_report.py
diff options
context:
space:
mode:
authorHenrique Nakashima <hnakashima@chromium.org>2017-08-09 14:38:04 -0400
committerChromium commit bot <commit-bot@chromium.org>2017-08-09 19:10:13 +0000
commit44f777901732fbf1bdb152e6ff4d1ba41ce75bba (patch)
tree5c02c76719dc15ee0594c89d6d7013587734c5dd /testing/tools/coverage/coverage_report.py
parentbe63ab97e0385b4024ef84fda79fc84dc111ab23 (diff)
downloadpdfium-44f777901732fbf1bdb152e6ff4d1ba41ce75bba.tar.xz
Move tools/coverage to testing/tools/coverage
Change-Id: Icb1d85167a234c9862a4011dea5dfaa09299c71c Reviewed-on: https://pdfium-review.googlesource.com/10431 Reviewed-by: Ryan Harrison <rharrison@chromium.org> Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Diffstat (limited to 'testing/tools/coverage/coverage_report.py')
-rwxr-xr-xtesting/tools/coverage/coverage_report.py392
1 files changed, 392 insertions, 0 deletions
diff --git a/testing/tools/coverage/coverage_report.py b/testing/tools/coverage/coverage_report.py
new file mode 100755
index 0000000000..a4222558e4
--- /dev/null
+++ b/testing/tools/coverage/coverage_report.py
@@ -0,0 +1,392 @@
+#!/usr/bin/env python
+# Copyright 2017 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Generates a coverage report for given binaries using llvm-gcov & lcov.
+
+Requires llvm-cov 3.5 or later.
+Requires lcov 1.11 or later.
+Requires that 'use_coverage = true' is set in args.gn.
+"""
+
+import argparse
+from collections import namedtuple
+import pprint
+import os
+import re
+import subprocess
+import sys
+
+
+# 'binary' is the file that is to be run for the test.
+# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus
+# requires special handling.
+TestSpec = namedtuple('TestSpec', 'binary, use_test_runner')
+
+# All of the coverage tests that the script knows how to run.
+COVERAGE_TESTS = {
+ 'pdfium_unittests': TestSpec('pdfium_unittests', False),
+ 'pdfium_embeddertests': TestSpec('pdfium_embeddertests', False),
+ 'corpus_tests': TestSpec('run_corpus_tests.py', True),
+ 'javascript_tests': TestSpec('run_javascript_tests.py', True),
+ 'pixel_tests': TestSpec('run_pixel_tests.py', True),
+}
+
+# Coverage tests that are known to take a long time to run, so are not in the
+# default set. The user must either explicitly invoke these tests or pass in
+# --slow.
+SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests']
+
+class CoverageExecutor(object):
+
+ def __init__(self, parser, args):
+ """Initialize executor based on the current script environment
+
+ Args:
+ parser: argparse.ArgumentParser for handling improper inputs.
+ args: Dictionary of arguments passed into the calling script.
+ """
+ self.dry_run = args['dry_run']
+ self.verbose = args['verbose']
+
+ llvm_cov = self.determine_proper_llvm_cov()
+ if not llvm_cov:
+ print 'Unable to find appropriate llvm-cov to use'
+ sys.exit(1)
+ self.lcov_env = os.environ
+ self.lcov_env['LLVM_COV_BIN'] = llvm_cov
+
+ self.lcov = self.determine_proper_lcov()
+ if not self.lcov:
+ print 'Unable to find appropriate lcov to use'
+ sys.exit(1)
+
+ self.coverage_files = set()
+ self.source_directory = args['source_directory']
+ if not os.path.isdir(self.source_directory):
+ parser.error("'%s' needs to be a directory" % self.source_directory)
+
+ self.build_directory = args['build_directory']
+ if not os.path.isdir(self.build_directory):
+ parser.error("'%s' needs to be a directory" % self.build_directory)
+
+ self.coverage_tests = self.calculate_coverage_tests(args)
+ if not self.coverage_tests:
+ parser.error(
+ 'No valid tests in set to be run. This is likely due to bad command '
+ 'line arguments')
+
+ if not self.boolean_gn_arg('use_coverage'):
+ parser.error(
+ 'use_coverage does not appear to be set to true for build, but is '
+ 'needed')
+
+ self.use_goma = self.boolean_gn_arg('use_goma')
+
+ self.output_directory = args['output_directory']
+ if not os.path.exists(self.output_directory):
+ if not self.dry_run:
+ os.makedirs(self.output_directory)
+ elif not os.path.isdir(self.output_directory):
+ parser.error('%s exists, but is not a directory' % self.output_directory)
+ self.coverage_totals_path = os.path.join(self.output_directory,
+ 'pdfium_totals.info')
+
+ def boolean_gn_arg(self, arg):
+ """Extract the value of a boolean flag in args.gn"""
+ cwd = os.getcwd()
+ os.chdir(self.build_directory)
+ gn_args_output = self.check_output(
+ ['gn', 'args', '.', '--list=%s' % arg, '--short'])
+ os.chdir(cwd)
+ arg_match_output = re.match('%s = (.*)' % arg, gn_args_output).group(1)
+ if self.verbose:
+ print "Found '%s' for value of %s" % (arg_match_output, arg)
+ return arg_match_output == 'true'
+
+ def check_output(self, args, dry_run=False, env=None):
+ """Dry run aware wrapper of subprocess.check_output()"""
+ if dry_run:
+ print "Would have run '%s'" % ' '.join(args)
+ return ''
+
+ output = subprocess.check_output(args, env=env)
+
+ if self.verbose:
+ print "check_output(%s) returned '%s'" % (args, output)
+ return output
+
+ def call(self, args, dry_run=False, env=None):
+ """Dry run aware wrapper of subprocess.call()"""
+ if dry_run:
+ print "Would have run '%s'" % ' '.join(args)
+ return 0
+
+ output = subprocess.call(args, env=env)
+
+ if self.verbose:
+ print 'call(%s) returned %s' % (args, output)
+ return output
+
+ def call_lcov(self, args, dry_run=False, needs_directory=True):
+ """Wrapper to call lcov that adds appropriate arguments as needed."""
+ lcov_args = [
+ self.lcov, '--config-file',
+ os.path.join(self.source_directory, 'testing', 'tools', 'coverage',
+ 'lcovrc'),
+ '--gcov-tool',
+ os.path.join(self.source_directory, 'testing', 'tools', 'coverage',
+ 'llvm-gcov')
+ ]
+ if needs_directory:
+ lcov_args.extend(['--directory', self.source_directory])
+ if not self.verbose:
+ lcov_args.append('--quiet')
+ lcov_args.extend(args)
+ return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env)
+
+ def calculate_coverage_tests(self, args):
+ """Determine which tests should be run."""
+ testing_tools_directory = os.path.join(self.source_directory, 'testing',
+ 'tools')
+ coverage_tests = {}
+ for name in COVERAGE_TESTS.keys():
+ test_spec = COVERAGE_TESTS[name]
+ if test_spec.use_test_runner:
+ binary_path = os.path.join(testing_tools_directory, test_spec.binary)
+ else:
+ binary_path = os.path.join(self.build_directory, test_spec.binary)
+ coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner)
+
+ if args['tests']:
+ return {name: spec
+ for name, spec in coverage_tests.iteritems() if name in args['tests']}
+ elif not args['slow']:
+ return {name: spec
+ for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS}
+ else:
+ return coverage_tests
+
+ def find_acceptable_binary(self, binary_name, version_regex,
+ min_major_version, min_minor_version):
+ """Find the newest version of binary that meets the min version."""
+ min_version = (min_major_version, min_minor_version)
+ parsed_versions = {}
+ # When calling Bash builtins like this the command and arguments must be
+ # passed in as a single string instead of as separate list members.
+ potential_binaries = self.check_output(
+ ['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines()
+ for binary in potential_binaries:
+ if self.verbose:
+ print 'Testing llvm-cov binary, %s' % binary
+ # Assuming that scripts that don't respond to --version correctly are not
+ # valid binaries and just happened to get globbed in. This is true for
+ # lcov and llvm-cov
+ try:
+ version_output = self.check_output([binary, '--version']).splitlines()
+ except subprocess.CalledProcessError:
+ if self.verbose:
+ print '--version returned failure status 1, so ignoring'
+ continue
+
+ for line in version_output:
+ matcher = re.match(version_regex, line)
+ if matcher:
+ parsed_version = (int(matcher.group(1)), int(matcher.group(2)))
+ if parsed_version >= min_version:
+ parsed_versions[parsed_version] = binary
+ break
+
+ if not parsed_versions:
+ return None
+ return parsed_versions[max(parsed_versions)]
+
+ def determine_proper_llvm_cov(self):
+ """Find a version of llvm_cov that will work with the script."""
+ version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*')
+ return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5)
+
+ def determine_proper_lcov(self):
+ """Find a version of lcov that will work with the script."""
+ version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*')
+ return self.find_acceptable_binary('lcov', version_regex, 1, 11)
+
+ def build_binaries(self):
+ """Build all the binaries that are going to be needed for coverage
+ generation."""
+ call_args = ['ninja']
+ if self.use_goma:
+ call_args.extend(['-j', '250'])
+ call_args.extend(['-C', self.build_directory])
+ return self.call(call_args, dry_run=self.dry_run) == 0
+
+ def generate_coverage(self, name, spec):
+ """Generate the coverage data for a test
+
+ Args:
+ name: Name associated with the test to be run. This is used as a label
+ in the coverage data, so should be unique across all of the tests
+ being run.
+ spec: Tuple containing the path to the binary to run, and if this test
+ uses test_runner.py.
+ """
+ if self.verbose:
+ print "Generating coverage for test '%s', using data '%s'" % (name, spec)
+ if not os.path.exists(spec.binary):
+ print('Unable to generate coverage for %s, since it appears to not exist'
+ ' @ %s') % (name, spec.binary)
+ return False
+
+ if self.call_lcov(['--zerocounters'], dry_run=self.dry_run):
+ print 'Unable to clear counters for %s' % name
+ return False
+
+ binary_args = [spec.binary]
+ if spec.use_test_runner:
+ # Test runner performs multi-threading in the wrapper script, not the test
+ # binary, so need -j 1, otherwise multiple processes will be writing to
+ # the code coverage files, invalidating results.
+ # TODO(pdfium:811): Rewrite how test runner tests work, so that they can
+ # be run in multi-threaded mode.
+ binary_args.extend(['-j', '1', '--build-dir', self.build_directory])
+ if self.call(binary_args, dry_run=self.dry_run) and self.verbose:
+ print('Running %s appears to have failed, which might affect '
+ 'results') % spec.binary
+
+ output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name)
+ if self.call_lcov(
+ ['--capture', '--test-name', name, '--output-file', output_raw_path],
+ dry_run=self.dry_run):
+ print 'Unable to capture coverage data for %s' % name
+ return False
+
+ output_filtered_path = os.path.join(self.output_directory,
+ '%s_filtered.info' % name)
+ output_filters = [
+ '/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp',
+ '*_embeddertest.cpp'
+ ]
+ if self.call_lcov(
+ ['--remove', output_raw_path] + output_filters +
+ ['--output-file', output_filtered_path],
+ dry_run=self.dry_run,
+ needs_directory=False):
+ print 'Unable to filter coverage data for %s' % name
+ return False
+
+ self.coverage_files.add(output_filtered_path)
+ return True
+
+ def merge_coverage(self):
+ """Merge all of the coverage data sets into one for report generation."""
+ merge_args = []
+ for coverage_file in self.coverage_files:
+ merge_args.extend(['--add-tracefile', coverage_file])
+
+ merge_args.extend(['--output-file', self.coverage_totals_path])
+ return self.call_lcov(
+ merge_args, dry_run=self.dry_run, needs_directory=False) == 0
+
+ def generate_report(self):
+ """Produce HTML coverage report based on combined coverage data set."""
+ config_file = os.path.join(
+ self.source_directory, 'tools', 'coverage', 'lcovrc')
+
+ lcov_args = ['genhtml',
+ '--config-file', config_file,
+ '--legend',
+ '--demangle-cpp',
+ '--show-details',
+ '--prefix', self.source_directory,
+ '--ignore-errors',
+ 'source', self.coverage_totals_path,
+ '--output-directory', self.output_directory]
+ return self.call(lcov_args, dry_run=self.dry_run) == 0
+
+ def run(self):
+ """Setup environment, execute the tests and generate coverage report"""
+ if not self.build_binaries():
+ print 'Failed to successfully build binaries'
+ return False
+
+ for name in self.coverage_tests.keys():
+ if not self.generate_coverage(name, self.coverage_tests[name]):
+ print 'Failed to successfully generate coverage data'
+ return False
+
+ if not self.merge_coverage():
+ print 'Failed to successfully merge generated coverage data'
+ return False
+
+ if not self.generate_report():
+ print 'Failed to successfully generated coverage report'
+ return False
+
+ return True
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.formatter_class = argparse.RawDescriptionHelpFormatter
+ parser.description = ('Generates a coverage report for given binaries using '
+ 'llvm-cov & lcov.\n\n'
+ 'Requires llvm-cov 3.5 or later.\n'
+ 'Requires lcov 1.11 or later.\n\n'
+ 'By default runs pdfium_unittests and '
+ 'pdfium_embeddertests. If --slow is passed in then all '
+ 'tests will be run. If any of the tests are specified '
+ 'on the command line, then only those will be run.')
+ parser.add_argument(
+ '-s',
+ '--source_directory',
+ help='Location of PDFium source directory, defaults to CWD',
+ default=os.getcwd())
+ build_default = os.path.join('out', 'Coverage')
+ parser.add_argument(
+ '-b',
+ '--build_directory',
+ help=
+ 'Location of PDFium build directory with coverage enabled, defaults to '
+ '%s under CWD' % build_default,
+ default=os.path.join(os.getcwd(), build_default))
+ output_default = 'coverage_report'
+ parser.add_argument(
+ '-o',
+ '--output_directory',
+ help='Location to write out coverage report to, defaults to %s under CWD '
+ % output_default,
+ default=os.path.join(os.getcwd(), output_default))
+ parser.add_argument(
+ '-n',
+ '--dry-run',
+ help='Output commands instead of executing them',
+ action='store_true')
+ parser.add_argument(
+ '-v',
+ '--verbose',
+ help='Output additional diagnostic information',
+ action='store_true')
+ parser.add_argument(
+ '--slow',
+ help='Run all tests, even those known to take a long time. Ignored if '
+ 'specific tests are passed in.',
+ action='store_true')
+ parser.add_argument(
+ 'tests',
+ help='Tests to be run, defaults to all. Valid entries are %s' %
+ COVERAGE_TESTS.keys(),
+ nargs='*')
+
+ args = vars(parser.parse_args())
+ if args['verbose']:
+ pprint.pprint(args)
+
+ executor = CoverageExecutor(parser, args)
+ if executor.run():
+ return 0
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())