Use visual studio toolchain from depot_tools for PDFium compilation

Change to use visual studio toolchain from depot_tools by default.
Setting DEPOT_TOOLS_WIN_TOOLCHAIN=0 allows compilation using system
toolchain as before.

Using toolchain from depot_tools unifies the compilation
environment, and brings the benefits of automated update, bug fixes
etc.

Review-Url: https://codereview.chromium.org/1897523002

5 files changed, 460 insertions, 2 deletions

diff --git a/.gitignore b/.gitignore
index 2ecaf97ecd..ccdfb546f4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 /build
+/build_gyp/win_toolchain.json
 /buildtools
 /out
 /testing/corpus
diff --git a/DEPS b/DEPS
index 7fb3a32b6e..1a08ef31eb 100644
--- a/DEPS
+++ b/DEPS
@@ -169,4 +169,10 @@ hooks = [
                '--if-needed'
     ],
   },
+  {
+    # Update the Windows toolchain if necessary.
+    'name': 'win_toolchain',
+    'pattern': '.',
+    'action': ['python', 'pdfium/build_gyp/vs_toolchain.py', 'update'],
+  },
 ]
diff --git a/build_gyp/find_depot_tools.py b/build_gyp/find_depot_tools.py
new file mode 100644
index 0000000000..6ec83b3d5c
--- /dev/null
+++ b/build_gyp/find_depot_tools.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# Copyright 2016 PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Small utility function to find depot_tools and add it to the python path.
+
+Will throw an ImportError exception if depot_tools can't be found since it
+imports breakpad.
+
+This can also be used as a standalone script to print out the depot_tools
+directory location.
+"""
+
+import os
+import sys
+
+
+def IsRealDepotTools(path):
+  return os.path.isfile(os.path.join(path, 'gclient.py'))
+
+
+def add_depot_tools_to_path():
+  """Search for depot_tools and add it to sys.path."""
+  # First look if depot_tools is already in PYTHONPATH.
+  for i in sys.path:
+    if i.rstrip(os.sep).endswith('depot_tools') and IsRealDepotTools(i):
+      return i
+  # Then look if depot_tools is in PATH, common case.
+  for i in os.environ['PATH'].split(os.pathsep):
+    if IsRealDepotTools(i):
+      sys.path.append(i.rstrip(os.sep))
+      return i
+  # Rare case, it's not even in PATH, look upward up to root.
+  root_dir = os.path.dirname(os.path.abspath(__file__))
+  previous_dir = os.path.abspath(__file__)
+  while root_dir and root_dir != previous_dir:
+    i = os.path.join(root_dir, 'depot_tools')
+    if IsRealDepotTools(i):
+      sys.path.append(i)
+      return i
+    previous_dir = root_dir
+    root_dir = os.path.dirname(root_dir)
+  print >> sys.stderr, 'Failed to find depot_tools'
+  return None
+
+DEPOT_TOOLS_PATH = add_depot_tools_to_path()
+
+# pylint: disable=W0611
+import breakpad
+
+
+def main():
+  if DEPOT_TOOLS_PATH is None:
+    return 1
+  print DEPOT_TOOLS_PATH
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/build_gyp/gyp_pdfium b/build_gyp/gyp_pdfium
index c79593b990..23b1c719c2 100755
--- a/build_gyp/gyp_pdfium
+++ b/build_gyp/gyp_pdfium
@@ -13,13 +13,23 @@ import sys
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 pdfium_root = os.path.abspath(os.path.join(script_dir, os.pardir))
+output_rel_dir = 'out'
 
 sys.path.insert(0, os.path.join(pdfium_root, 'tools', 'gyp', 'pylib'))
 import gyp
-
+# vs_toolchain needs to be after gyp path setting since it also uses gyp.
+import vs_toolchain
 
 def run_gyp(args):
   rc = gyp.main(args)
+
+  # Copy Windows toolchain DLLs.
+  vs_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
+  if vs_runtime_dll_dirs:
+    x64_runtime, x86_runtime = vs_runtime_dll_dirs
+    vs_toolchain.CopyVsRuntimeDlls(
+      os.path.join(pdfium_root, output_rel_dir), (x86_runtime, x64_runtime))
+
   if rc != 0:
     print 'Error running GYP'
     sys.exit(rc)
@@ -36,7 +46,7 @@ def main():
   args.append('-I')
   args.append(os.path.join(pdfium_root, 'build_gyp', 'standalone.gypi'))
 
-  args.extend(['-D', 'gyp_output_dir=out'])
+  args.extend(['-D', 'gyp_output_dir=' + output_rel_dir])
 
   # Set the GYP DEPTH variable to the root of the PDFium project.
   args.append('--depth=' + os.path.relpath(pdfium_root))
@@ -46,6 +56,9 @@ def main():
     print 'GYP_GENERATORS is not set, defaulting to ninja'
     os.environ['GYP_GENERATORS'] = 'ninja'
 
+  # Set up the environment variables.
+  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
+
   print 'Updating projects from gyp files...'
   sys.stdout.flush()
 
diff --git a/build_gyp/vs_toolchain.py b/build_gyp/vs_toolchain.py
new file mode 100644
index 0000000000..3424650913
--- /dev/null
+++ b/build_gyp/vs_toolchain.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python
+# Copyright 2016 PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import glob
+import json
+import os
+import pipes
+import shutil
+import subprocess
+import sys
+
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+src_root = os.path.abspath(os.path.join(script_dir, os.pardir))
+SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# Add gyp path which is needed to import gyp.
+sys.path.insert(0, os.path.join(src_root, 'tools', 'gyp', 'pylib'))
+json_data_file = os.path.join(script_dir, 'win_toolchain.json')
+
+
+import gyp
+
+
+# Use MSVS2015 as the default toolchain.
+CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2015'
+
+
+def SetEnvironmentAndGetRuntimeDllDirs():
+  """Sets up os.environ to use the depot_tools VS toolchain with gyp, and
+  returns the location of the VS runtime DLLs so they can be copied into
+  the output directory after gyp generation.
+  """
+  vs_runtime_dll_dirs = None
+  depot_tools_win_toolchain = \
+      bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')))
+  # When running on a non-Windows host, only do this if the SDK has explicitly
+  # been downloaded before (in which case json_data_file will exist).
+  if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file))
+      and depot_tools_win_toolchain):
+    if ShouldUpdateToolchain():
+      Update()
+    with open(json_data_file, 'r') as tempf:
+      toolchain_data = json.load(tempf)
+
+    toolchain = toolchain_data['path']
+    version = toolchain_data['version']
+    win_sdk = toolchain_data.get('win_sdk')
+    if not win_sdk:
+      win_sdk = toolchain_data['win8sdk']
+    wdk = toolchain_data['wdk']
+    # TODO(scottmg): The order unfortunately matters in these. They should be
+    # split into separate keys for x86 and x64. (See CopyVsRuntimeDlls call
+    # below). http://crbug.com/345992
+    vs_runtime_dll_dirs = toolchain_data['runtime_dirs']
+
+    os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain
+    os.environ['GYP_MSVS_VERSION'] = version
+    # We need to make sure windows_sdk_path is set to the automated
+    # toolchain values in GYP_DEFINES, but don't want to override any
+    # otheroptions.express
+    # values there.
+    gyp_defines_dict = gyp.NameValueListToDict(gyp.ShlexEnv('GYP_DEFINES'))
+    gyp_defines_dict['windows_sdk_path'] = win_sdk
+    os.environ['GYP_DEFINES'] = ' '.join('%s=%s' % (k, pipes.quote(str(v)))
+        for k, v in gyp_defines_dict.iteritems())
+    os.environ['WINDOWSSDKDIR'] = win_sdk
+    os.environ['WDK_DIR'] = wdk
+    # Include the VS runtime in the PATH in case it's not machine-installed.
+    runtime_path = os.path.pathsep.join(vs_runtime_dll_dirs)
+    os.environ['PATH'] = runtime_path + os.path.pathsep + os.environ['PATH']
+  elif sys.platform == 'win32' and not depot_tools_win_toolchain:
+    if not 'GYP_MSVS_OVERRIDE_PATH' in os.environ:
+      os.environ['GYP_MSVS_OVERRIDE_PATH'] = DetectVisualStudioPath()
+    if not 'GYP_MSVS_VERSION' in os.environ:
+      os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion()
+
+  return vs_runtime_dll_dirs
+
+
+def _RegistryGetValueUsingWinReg(key, value):
+  """Use the _winreg module to obtain the value of a registry key.
+
+  Args:
+    key: The registry key.
+    value: The particular registry value to read.
+  Return:
+    contents of the registry key's value, or None on failure.  Throws
+    ImportError if _winreg is unavailable.
+  """
+  import _winreg
+  try:
+    root, subkey = key.split('\\', 1)
+    assert root == 'HKLM'  # Only need HKLM for now.
+    with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, subkey) as hkey:
+      return _winreg.QueryValueEx(hkey, value)[0]
+  except WindowsError:
+    return None
+
+
+def _RegistryGetValue(key, value):
+  try:
+    return _RegistryGetValueUsingWinReg(key, value)
+  except ImportError:
+    raise Exception('The python library _winreg not found.')
+
+
+def GetVisualStudioVersion():
+  """Return GYP_MSVS_VERSION of Visual Studio.
+  """
+  return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION)
+
+
+def DetectVisualStudioPath():
+  """Return path to the GYP_MSVS_VERSION of Visual Studio.
+  """
+
+  # Note that this code is used from
+  # build/toolchain/win/setup_toolchain.py as well.
+  version_as_year = GetVisualStudioVersion()
+  year_to_version = {
+      '2013': '12.0',
+      '2015': '14.0',
+  }
+  if version_as_year not in year_to_version:
+    raise Exception(('Visual Studio version %s (from GYP_MSVS_VERSION)'
+                     ' not supported. Supported versions are: %s') % (
+                       version_as_year, ', '.join(year_to_version.keys())))
+  version = year_to_version[version_as_year]
+  keys = [r'HKLM\Software\Microsoft\VisualStudio\%s' % version,
+          r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\%s' % version]
+  for key in keys:
+    path = _RegistryGetValue(key, 'InstallDir')
+    if not path:
+      continue
+    path = os.path.normpath(os.path.join(path, '..', '..'))
+    return path
+
+  raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)'
+                   ' not found.') % (version_as_year))
+
+
+def _VersionNumber():
+  """Gets the standard version number ('120', '140', etc.) based on
+  GYP_MSVS_VERSION."""
+  vs_version = GetVisualStudioVersion()
+  if vs_version == '2013':
+    return '120'
+  elif vs_version == '2015':
+    return '140'
+  else:
+    raise ValueError('Unexpected GYP_MSVS_VERSION')
+
+
+def _CopyRuntimeImpl(target, source, verbose=True):
+  """Copy |source| to |target| if it doesn't already exist or if it
+  needs to be updated.
+  """
+  if (os.path.isdir(os.path.dirname(target)) and
+      (not os.path.isfile(target) or
+      os.stat(target).st_mtime != os.stat(source).st_mtime)):
+    if verbose:
+      print 'Copying %s to %s...' % (source, target)
+    if os.path.exists(target):
+      os.unlink(target)
+    shutil.copy2(source, target)
+
+
+def _CopyRuntime2013(target_dir, source_dir, dll_pattern):
+  """Copy both the msvcr and msvcp runtime DLLs, only if the target doesn't
+  exist, but the target directory does exist."""
+  for file_part in ('p', 'r'):
+    dll = dll_pattern % file_part
+    target = os.path.join(target_dir, dll)
+    source = os.path.join(source_dir, dll)
+    _CopyRuntimeImpl(target, source)
+
+
+def _CopyRuntime2015(target_dir, source_dir, dll_pattern, suffix):
+  """Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't
+  exist, but the target directory does exist."""
+  for file_part in ('msvcp', 'vccorlib', 'vcruntime'):
+    dll = dll_pattern % file_part
+    target = os.path.join(target_dir, dll)
+    source = os.path.join(source_dir, dll)
+    _CopyRuntimeImpl(target, source)
+  ucrt_src_dir = os.path.join(source_dir, 'api-ms-win-*.dll')
+  print 'Copying %s to %s...' % (ucrt_src_dir, target_dir)
+  for ucrt_src_file in glob.glob(ucrt_src_dir):
+    file_part = os.path.basename(ucrt_src_file)
+    ucrt_dst_file = os.path.join(target_dir, file_part)
+    _CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False)
+  _CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix),
+                    os.path.join(source_dir, 'ucrtbase' + suffix))
+
+
+def _CopyRuntime(target_dir, source_dir, target_cpu, debug):
+  """Copy the VS runtime DLLs, only if the target doesn't exist, but the target
+  directory does exist. Handles VS 2013 and VS 2015."""
+  suffix = "d.dll" if debug else ".dll"
+  if GetVisualStudioVersion() == '2015':
+    _CopyRuntime2015(target_dir, source_dir, '%s140' + suffix, suffix)
+  else:
+    _CopyRuntime2013(target_dir, source_dir, 'msvc%s120' + suffix)
+
+  # Copy the PGO runtime library to the release directories.
+  if not debug and os.environ.get('GYP_MSVS_OVERRIDE_PATH'):
+    pgo_x86_runtime_dir = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'),
+                                        'VC', 'bin')
+    pgo_x64_runtime_dir = os.path.join(pgo_x86_runtime_dir, 'amd64')
+    pgo_runtime_dll = 'pgort' + _VersionNumber() + '.dll'
+    if target_cpu == "x86":
+      source_x86 = os.path.join(pgo_x86_runtime_dir, pgo_runtime_dll)
+      if os.path.exists(source_x86):
+        _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll), source_x86)
+    elif target_cpu == "x64":
+      source_x64 = os.path.join(pgo_x64_runtime_dir, pgo_runtime_dll)
+      if os.path.exists(source_x64):
+        _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll),
+                          source_x64)
+    else:
+      raise NotImplementedError("Unexpected target_cpu value:" + target_cpu)
+
+
+def CopyVsRuntimeDlls(output_dir, runtime_dirs):
+  """Copies the VS runtime DLLs from the given |runtime_dirs| to the output
+  directory so that even if not system-installed, built binaries are likely to
+  be able to run.
+
+  This needs to be run after gyp has been run so that the expected target
+  output directories are already created.
+
+  This is used for the GYP build and gclient runhooks.
+  """
+  x86, x64 = runtime_dirs
+  out_debug = os.path.join(output_dir, 'Debug')
+  out_debug_nacl64 = os.path.join(output_dir, 'Debug', 'x64')
+  out_release = os.path.join(output_dir, 'Release')
+  out_release_nacl64 = os.path.join(output_dir, 'Release', 'x64')
+  out_debug_x64 = os.path.join(output_dir, 'Debug_x64')
+  out_release_x64 = os.path.join(output_dir, 'Release_x64')
+
+  if os.path.exists(out_debug) and not os.path.exists(out_debug_nacl64):
+    os.makedirs(out_debug_nacl64)
+  if os.path.exists(out_release) and not os.path.exists(out_release_nacl64):
+    os.makedirs(out_release_nacl64)
+  _CopyRuntime(out_debug,          x86, "x86", debug=True)
+  _CopyRuntime(out_release,        x86, "x86", debug=False)
+  _CopyRuntime(out_debug_x64,      x64, "x64", debug=True)
+  _CopyRuntime(out_release_x64,    x64, "x64", debug=False)
+  _CopyRuntime(out_debug_nacl64,   x64, "x64", debug=True)
+  _CopyRuntime(out_release_nacl64, x64, "x64", debug=False)
+
+
+def CopyDlls(target_dir, configuration, target_cpu):
+  """Copy the VS runtime DLLs into the requested directory as needed.
+
+  configuration is one of 'Debug' or 'Release'.
+  target_cpu is one of 'x86' or 'x64'.
+
+  The debug configuration gets both the debug and release DLLs; the
+  release config only the latter.
+
+  This is used for the GN build.
+  """
+  vs_runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
+  if not vs_runtime_dll_dirs:
+    return
+
+  x64_runtime, x86_runtime = vs_runtime_dll_dirs
+  runtime_dir = x64_runtime if target_cpu == 'x64' else x86_runtime
+  _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False)
+  if configuration == 'Debug':
+    _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True)
+
+
+def _GetDesiredVsToolchainHashes():
+  """Load a list of SHA1s corresponding to the toolchains that we want installed
+  to build with."""
+  if GetVisualStudioVersion() == '2015':
+    # Update 2.
+    return ['95ddda401ec5678f15eeed01d2bee08fcbc5ee97']
+  else:
+    return ['4087e065abebdca6dbd0caca2910c6718d2ec67f']
+
+
+def ShouldUpdateToolchain():
+  """Check if the toolchain should be upgraded."""
+  if not os.path.exists(json_data_file):
+    return True
+  with open(json_data_file, 'r') as tempf:
+    toolchain_data = json.load(tempf)
+  version = toolchain_data['version']
+  env_version = GetVisualStudioVersion()
+  # If there's a mismatch between the version set in the environment and the one
+  # in the json file then the toolchain should be updated.
+  return version != env_version
+
+
+def Update(force=False):
+  """Requests an update of the toolchain to the specific hashes we have at
+  this revision. The update outputs a .json of the various configuration
+  information required to pass to gyp which we use in |GetToolchainDir()|.
+  """
+  if force != False and force != '--force':
+    print >>sys.stderr, 'Unknown parameter "%s"' % force
+    return 1
+  if force == '--force' or os.path.exists(json_data_file):
+    force = True
+
+  depot_tools_win_toolchain = \
+      bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')))
+  if ((sys.platform in ('win32', 'cygwin') or force) and
+        depot_tools_win_toolchain):
+    import find_depot_tools
+    depot_tools_path = find_depot_tools.add_depot_tools_to_path()
+    # Necessary so that get_toolchain_if_necessary.py will put the VS toolkit
+    # in the correct directory.
+    os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion()
+    get_toolchain_args = [
+        sys.executable,
+        os.path.join(depot_tools_path,
+                    'win_toolchain',
+                    'get_toolchain_if_necessary.py'),
+        '--output-json', json_data_file,
+      ] + _GetDesiredVsToolchainHashes()
+    if force:
+      get_toolchain_args.append('--force')
+    subprocess.check_call(get_toolchain_args)
+
+  return 0
+
+
+def NormalizePath(path):
+  while path.endswith("\\"):
+    path = path[:-1]
+  return path
+
+
+def GetToolchainDir():
+  """Gets location information about the current toolchain (must have been
+  previously updated by 'update'). This is used for the GN build."""
+  runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
+
+  # If WINDOWSSDKDIR is not set, search the default SDK path and set it.
+  if not 'WINDOWSSDKDIR' in os.environ:
+    default_sdk_path = 'C:\\Program Files (x86)\\Windows Kits\\10'
+    if os.path.isdir(default_sdk_path):
+      os.environ['WINDOWSSDKDIR'] = default_sdk_path
+
+  print '''vs_path = "%s"
+sdk_path = "%s"
+vs_version = "%s"
+wdk_dir = "%s"
+runtime_dirs = "%s"
+''' % (
+      NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH']),
+      NormalizePath(os.environ['WINDOWSSDKDIR']),
+      GetVisualStudioVersion(),
+      NormalizePath(os.environ.get('WDK_DIR', '')),
+      os.path.pathsep.join(runtime_dll_dirs or ['None']))
+
+
+def main():
+  commands = {
+      'update': Update,
+      'get_toolchain_dir': GetToolchainDir,
+      'copy_dlls': CopyDlls,
+  }
+  if len(sys.argv) < 2 or sys.argv[1] not in commands:
+    print >>sys.stderr, 'Expected one of: %s' % ', '.join(commands)
+    return 1
+  return commands[sys.argv[1]](*sys.argv[2:])
+
+
+if __name__ == '__main__':
+  sys.exit(main())