Migrate coverage_report.py to use upstream Chromium scripts

This adds tools/code_coverage from Chromium to DEPS and converts our
existing coverage_report.py to use it instead of gcov & lcov. This
generates a different format of HTML report, but the content appears
to be the same. Some of the coverage numbers changed a bit, due to
differences in how llvm-cov and gcov calculate executable lines, but
drilling down into the reports I think llvm-cov is more accurate
overall and there are no major discrepancies.

Large portions of the existing script are left as is and just the
report generation has been changed. I plan in follow up CLs to remove
the duplication of functionality in the PDFium scripts and modularlize
the upstream code better.

BUG=pdfium:1069

Change-Id: I009bfb8aac8f1a878e01ff70923e19bbb4774a9c
Reviewed-on: https://pdfium-review.googlesource.com/32894
Commit-Queue: Ryan Harrison <rharrison@chromium.org>
Reviewed-by: dsinclair <dsinclair@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>

1 files changed, 114 insertions, 180 deletions

diff --git a/testing/tools/coverage/coverage_report.py b/testing/tools/coverage/coverage_report.py
index 95c88bf612..db8d75a2c3 100755
--- a/testing/tools/coverage/coverage_report.py
+++ b/testing/tools/coverage/coverage_report.py
@@ -2,35 +2,29 @@
 # Copyright 2017 The PDFium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+"""Generates a coverage report for given tests.
 
-"""Generates a coverage report for given binaries using llvm-gcov & lcov.
-
-Requires llvm-cov 3.5 or later.
-Requires lcov 1.11 or later.
-Requires that 'use_coverage = true' is set in args.gn.
+Requires that 'use_clang_coverage = true' is set in args.gn.
+Prefers that 'is_component_build = false' is set in args.gn.
 """
 
 import argparse
 from collections import namedtuple
+import fnmatch
 import os
 import pprint
-import re
 import subprocess
 import sys
 
-
 # Add src dir to path to avoid having to set PYTHONPATH.
 sys.path.append(
     os.path.abspath(
-       os.path.join(
-          os.path.dirname(__file__),
-          os.path.pardir,
-          os.path.pardir,
-          os.path.pardir)))
+        os.path.join(
+            os.path.dirname(__file__), os.path.pardir, os.path.pardir,
+            os.path.pardir)))
 
 from testing.tools.common import GetBooleanGnArg
 
-
 # 'binary' is the file that is to be run for the test.
 # 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus
 # requires special handling.
@@ -45,10 +39,6 @@ COVERAGE_TESTS = {
     'pixel_tests': TestSpec('run_pixel_tests.py', True),
 }
 
-# Coverage tests that are known to take a long time to run, so are not in the
-# default set. The user must either explicitly invoke these tests or pass in
-# --slow.
-SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests']
 
 class CoverageExecutor(object):
 
@@ -62,36 +52,31 @@ class CoverageExecutor(object):
     self.dry_run = args['dry_run']
     self.verbose = args['verbose']
 
-    llvm_cov = self.determine_proper_llvm_cov()
-    if not llvm_cov:
-      print 'Unable to find appropriate llvm-cov to use'
-      sys.exit(1)
-    self.lcov_env = os.environ
-    self.lcov_env['LLVM_COV_BIN'] = llvm_cov
-
-    self.lcov = self.determine_proper_lcov()
-    if not self.lcov:
-      print 'Unable to find appropriate lcov to use'
-      sys.exit(1)
-
-    self.coverage_files = set()
     self.source_directory = args['source_directory']
     if not os.path.isdir(self.source_directory):
       parser.error("'%s' needs to be a directory" % self.source_directory)
 
+    self.llvm_directory = os.path.join(self.source_directory, 'third_party',
+                                       'llvm-build', 'Release+Asserts', 'bin')
+    if not os.path.isdir(self.llvm_directory):
+      parser.error("Cannot find LLVM bin directory , expected it to be in '%s'"
+                   % self.llvm_directory)
+
     self.build_directory = args['build_directory']
     if not os.path.isdir(self.build_directory):
       parser.error("'%s' needs to be a directory" % self.build_directory)
 
-    self.coverage_tests = self.calculate_coverage_tests(args)
+    (self.coverage_tests,
+     self.build_targets) = self.calculate_coverage_tests(args)
     if not self.coverage_tests:
       parser.error(
           'No valid tests in set to be run. This is likely due to bad command '
           'line arguments')
 
-    if not GetBooleanGnArg('use_coverage', self.build_directory, self.verbose):
+    if not GetBooleanGnArg('use_clang_coverage', self.build_directory,
+                           self.verbose):
       parser.error(
-          'use_coverage does not appear to be set to true for build, but is '
+          'use_clang_coverage does not appear to be set to true for build, but is '
           'needed')
 
     self.use_goma = GetBooleanGnArg('use_goma', self.build_directory,
@@ -103,8 +88,11 @@ class CoverageExecutor(object):
         os.makedirs(self.output_directory)
     elif not os.path.isdir(self.output_directory):
       parser.error('%s exists, but is not a directory' % self.output_directory)
-    self.coverage_totals_path = os.path.join(self.output_directory,
-                                             'pdfium_totals.info')
+    elif len(os.listdir(self.output_directory)) > 0:
+      parser.error('%s is not empty, cowardly refusing to continue' %
+                   self.output_directory)
+
+    self.prof_data = os.path.join(self.output_directory, 'pdfium.profdata')
 
   def check_output(self, args, dry_run=False, env=None):
     """Dry run aware wrapper of subprocess.check_output()"""
@@ -130,96 +118,48 @@ class CoverageExecutor(object):
       print 'call(%s) returned %s' % (args, output)
     return output
 
-  def call_lcov(self, args, dry_run=False, needs_directory=True):
-    """Wrapper to call lcov that adds appropriate arguments as needed."""
-    lcov_args = [
-        self.lcov, '--config-file',
-        os.path.join(self.source_directory, 'testing', 'tools', 'coverage',
-                     'lcovrc'),
-        '--gcov-tool',
-        os.path.join(self.source_directory, 'testing', 'tools', 'coverage',
-                     'llvm-gcov')
-    ]
-    if needs_directory:
-      lcov_args.extend(['--directory', self.source_directory])
-    if not self.verbose:
-      lcov_args.append('--quiet')
-    lcov_args.extend(args)
-    return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env)
+  def call_silent(self, args, dry_run=False, env=None):
+    """Dry run aware wrapper of subprocess.call() that eats output from call"""
+    if dry_run:
+      print "Would have run '%s'" % ' '.join(args)
+      return 0
+
+    with open(os.devnull, 'w') as f:
+      output = subprocess.call(args, env=env, stdout=f)
+
+    if self.verbose:
+      print 'call_silent(%s) returned %s' % (args, output)
+    return output
 
   def calculate_coverage_tests(self, args):
     """Determine which tests should be run."""
     testing_tools_directory = os.path.join(self.source_directory, 'testing',
                                            'tools')
+    tests = args['tests'] if args['tests'] else COVERAGE_TESTS.keys()
     coverage_tests = {}
-    for name in COVERAGE_TESTS.keys():
+    build_targets = set()
+    for name in tests:
       test_spec = COVERAGE_TESTS[name]
       if test_spec.use_test_runner:
         binary_path = os.path.join(testing_tools_directory, test_spec.binary)
+        build_targets.add('pdfium_test')
       else:
         binary_path = os.path.join(self.build_directory, test_spec.binary)
+        build_targets.add(name)
       coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner)
 
-    if args['tests']:
-      return {name: spec
-        for name, spec in coverage_tests.iteritems() if name in args['tests']}
-    elif not args['slow']:
-      return {name: spec
-        for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS}
-    else:
-      return coverage_tests
-
-  def find_acceptable_binary(self, binary_name, version_regex,
-                             min_major_version, min_minor_version):
-    """Find the newest version of binary that meets the min version."""
-    min_version = (min_major_version, min_minor_version)
-    parsed_versions = {}
-    # When calling Bash builtins like this the command and arguments must be
-    # passed in as a single string instead of as separate list members.
-    potential_binaries = self.check_output(
-        ['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines()
-    for binary in potential_binaries:
-      if self.verbose:
-        print 'Testing llvm-cov binary, %s' % binary
-      # Assuming that scripts that don't respond to --version correctly are not
-      # valid binaries and just happened to get globbed in. This is true for
-      # lcov and llvm-cov
-      try:
-        version_output = self.check_output([binary, '--version']).splitlines()
-      except subprocess.CalledProcessError:
-        if self.verbose:
-          print '--version returned failure status 1, so ignoring'
-        continue
-
-      for line in version_output:
-        matcher = re.match(version_regex, line)
-        if matcher:
-          parsed_version = (int(matcher.group(1)), int(matcher.group(2)))
-          if parsed_version >= min_version:
-            parsed_versions[parsed_version] = binary
-          break
-
-    if not parsed_versions:
-      return None
-    return parsed_versions[max(parsed_versions)]
-
-  def determine_proper_llvm_cov(self):
-    """Find a version of llvm_cov that will work with the script."""
-    version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*')
-    return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5)
-
-  def determine_proper_lcov(self):
-    """Find a version of lcov that will work with the script."""
-    version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*')
-    return self.find_acceptable_binary('lcov', version_regex, 1, 11)
+    build_targets = list(build_targets)
+
+    return coverage_tests, build_targets
 
   def build_binaries(self):
     """Build all the binaries that are going to be needed for coverage
     generation."""
     call_args = ['ninja']
     if self.use_goma:
-      call_args.extend(['-j', '250'])
-    call_args.extend(['-C', self.build_directory])
+      call_args += ['-j', '250']
+    call_args += ['-C', self.build_directory]
+    call_args += self.build_targets
     return self.call(call_args, dry_run=self.dry_run) == 0
 
   def generate_coverage(self, name, spec):
@@ -239,74 +179,73 @@ class CoverageExecutor(object):
             ' @ %s') % (name, spec.binary)
       return False
 
-    if self.call_lcov(['--zerocounters'], dry_run=self.dry_run):
-      print 'Unable to clear counters for %s' % name
-      return False
-
     binary_args = [spec.binary]
+    profile_pattern_string = '%8m'
+    expected_profraw_file = '%s.%s.profraw' % (name, profile_pattern_string)
+    expected_profraw_path = os.path.join(self.output_directory,
+                                         expected_profraw_file)
+
+    env = {
+        'LLVM_PROFILE_FILE': expected_profraw_path,
+        'PATH': os.getenv('PATH') + os.pathsep + self.llvm_directory
+    }
+
     if spec.use_test_runner:
       # Test runner performs multi-threading in the wrapper script, not the test
-      # binary, so need -j 1, otherwise multiple processes will be writing to
-      # the code coverage files, invalidating results.
-      # TODO(pdfium:811): Rewrite how test runner tests work, so that they can
-      # be run in multi-threaded mode.
-      binary_args.extend(['-j', '1', '--build-dir', self.build_directory])
-    if self.call(binary_args, dry_run=self.dry_run) and self.verbose:
+      # binary, so need to limit the number of instances of the binary being run
+      # to the max value in LLVM_PROFILE_FILE, which is 8.
+      binary_args.extend(['-j', '8', '--build-dir', self.build_directory])
+    if self.call(binary_args, dry_run=self.dry_run, env=env) and self.verbose:
       print('Running %s appears to have failed, which might affect '
             'results') % spec.binary
 
-    output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name)
-    if self.call_lcov(
-        ['--capture', '--test-name', name, '--output-file', output_raw_path],
-        dry_run=self.dry_run):
-      print 'Unable to capture coverage data for %s' % name
-      return False
-
-    output_filtered_path = os.path.join(self.output_directory,
-                                        '%s_filtered.info' % name)
-    output_filters = [
-        '/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp',
-        '*_embeddertest.cpp'
-    ]
-    if self.call_lcov(
-        ['--remove', output_raw_path] + output_filters +
-        ['--output-file', output_filtered_path],
-        dry_run=self.dry_run,
-        needs_directory=False):
-      print 'Unable to filter coverage data for %s' % name
-      return False
-
-    self.coverage_files.add(output_filtered_path)
     return True
 
-  def merge_coverage(self):
-    """Merge all of the coverage data sets into one for report generation."""
-    merge_args = []
-    for coverage_file in self.coverage_files:
-      merge_args.extend(['--add-tracefile', coverage_file])
-
-    merge_args.extend(['--output-file', self.coverage_totals_path])
-    return self.call_lcov(
-        merge_args, dry_run=self.dry_run, needs_directory=False) == 0
-
-  def generate_report(self):
-    """Produce HTML coverage report based on combined coverage data set."""
-    config_file = os.path.join(
-        self.source_directory, 'testing', 'tools', 'coverage', 'lcovrc')
-
-    lcov_args = ['genhtml',
-      '--config-file', config_file,
-      '--legend',
-      '--demangle-cpp',
-      '--show-details',
-      '--prefix', self.source_directory,
-      '--ignore-errors',
-      'source', self.coverage_totals_path,
-      '--output-directory', self.output_directory]
-    return self.call(lcov_args, dry_run=self.dry_run) == 0
+  def merge_raw_coverage_results(self):
+    """Merge raw coverage data sets into one one file for report generation."""
+    llvm_profdata_bin = os.path.join(self.llvm_directory, 'llvm-profdata')
+
+    raw_data = []
+    raw_data_pattern = '*.profraw'
+    for file_name in os.listdir(self.output_directory):
+      if fnmatch.fnmatch(file_name, raw_data_pattern):
+        raw_data.append(os.path.join(self.output_directory, file_name))
+
+    return self.call(
+        [llvm_profdata_bin, 'merge', '-o', self.prof_data, '-sparse=true'] +
+        raw_data) == 0
+
+  def generate_html_report(self):
+    """Generate HTML report by calling upstream coverage.py"""
+    coverage_bin = os.path.join(self.source_directory, 'tools', 'code_coverage',
+                                'coverage.py')
+    report_directory = os.path.join(self.output_directory, 'HTML')
+
+    coverage_args = ['-p', self.prof_data]
+    coverage_args += ['-b', self.build_directory]
+    coverage_args += ['-o', report_directory]
+    coverage_args += self.build_targets
+
+    # Whitelist the directories of interest
+    coverage_args += ['-f', 'core']
+    coverage_args += ['-f', 'fpdfsdk']
+    coverage_args += ['-f', 'fxbarcode']
+    coverage_args += ['-f', 'fxjs']
+    coverage_args += ['-f', 'public']
+    coverage_args += ['-f', 'samples']
+    coverage_args += ['-f', 'xfa']
+
+    # Blacklist test files
+    coverage_args += ['-i', '.*test*.']
+
+    return self.call([coverage_bin] + coverage_args) == 0
 
   def run(self):
     """Setup environment, execute the tests and generate coverage report"""
+    if not self.fetch_profiling_tools():
+      print 'Unable to fetch profiling tools'
+      return False
+
     if not self.build_binaries():
       print 'Failed to successfully build binaries'
       return False
@@ -316,28 +255,28 @@ class CoverageExecutor(object):
         print 'Failed to successfully generate coverage data'
         return False
 
-    if not self.merge_coverage():
-      print 'Failed to successfully merge generated coverage data'
+    if not self.merge_raw_coverage_results():
+      print 'Failed to successfully merge raw coverage results'
       return False
 
-    if not self.generate_report():
-      print 'Failed to successfully generated coverage report'
+    if not self.generate_html_report():
+      print 'Failed to successfully generate HTML report'
       return False
 
     return True
 
+  def fetch_profiling_tools(self):
+    """Call coverage.py with no args to ensure profiling tools are present."""
+    return self.call_silent(
+        os.path.join(self.source_directory, 'tools', 'code_coverage',
+                     'coverage.py')) == 0
+
 
 def main():
   parser = argparse.ArgumentParser()
   parser.formatter_class = argparse.RawDescriptionHelpFormatter
-  parser.description = ('Generates a coverage report for given binaries using '
-                        'llvm-cov & lcov.\n\n'
-                        'Requires llvm-cov 3.5 or later.\n'
-                        'Requires lcov 1.11 or later.\n\n'
-                        'By default runs pdfium_unittests and '
-                        'pdfium_embeddertests. If --slow is passed in then all '
-                        'tests will be run. If any of the tests are specified '
-                        'on the command line, then only those will be run.')
+  parser.description = 'Generates a coverage report for given tests.'
+
   parser.add_argument(
       '-s',
       '--source_directory',
@@ -369,11 +308,6 @@ def main():
       help='Output additional diagnostic information',
       action='store_true')
   parser.add_argument(
-      '--slow',
-      help='Run all tests, even those known to take a long time. Ignored if '
-      'specific tests are passed in.',
-      action='store_true')
-  parser.add_argument(
       'tests',
       help='Tests to be run, defaults to all. Valid entries are %s' %
       COVERAGE_TESTS.keys(),