From cc8b2fa83e8cf02beb27c390b4dcfb7564becac0 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Wed, 21 Oct 2015 18:00:31 -0400 Subject: [PATCH] Update gyp to b3cef02. Includes two patches for AIX. Adds support for both 32-bit and 64-bit files[0] and uses -RPf for cp instead of -af (which is unsupported)[1] [0] https://codereview.chromium.org/1319663007 [1] https://codereview.chromium.org/1368133002 PR-URL: https://github.com/nodejs/node-gyp/pull/781 Reviewed-By: Ben Noordhuis --- gyp/PRESUBMIT.py | 10 +- gyp/buildbot/buildbot_run.py | 119 +----- gyp/buildbot/commit_queue/cq_config.json | 1 - gyp/gyp_main.py | 12 +- gyp/pylib/gyp/MSVSSettings.py | 5 +- gyp/pylib/gyp/MSVSSettings_test.py | 2 +- gyp/pylib/gyp/MSVSVersion.py | 5 +- gyp/pylib/gyp/__init__.py | 16 +- gyp/pylib/gyp/common.py | 11 +- gyp/pylib/gyp/generator/analyzer.py | 53 ++- gyp/pylib/gyp/generator/cmake.py | 384 +++++++++++------- .../gyp/generator/dump_dependency_json.py | 20 +- gyp/pylib/gyp/generator/make.py | 97 +++-- gyp/pylib/gyp/generator/msvs.py | 31 +- gyp/pylib/gyp/generator/ninja.py | 23 +- gyp/pylib/gyp/input.py | 55 ++- gyp/pylib/gyp/mac_tool.py | 3 +- gyp/pylib/gyp/msvs_emulation.py | 10 + gyp/pylib/gyp/win_tool.py | 4 +- gyp/pylib/gyp/xcode_emulation.py | 30 +- 20 files changed, 509 insertions(+), 382 deletions(-) diff --git a/gyp/PRESUBMIT.py b/gyp/PRESUBMIT.py index abec27b3e3..dde025383c 100644 --- a/gyp/PRESUBMIT.py +++ b/gyp/PRESUBMIT.py @@ -125,15 +125,13 @@ def CheckChangeOnCommit(input_api, output_api): TRYBOTS = [ - 'gyp-win32', - 'gyp-win64', - 'gyp-linux', - 'gyp-mac', - 'gyp-android' + 'linux_try', + 'mac_try', + 'win_try', ] def GetPreferredTryMasters(_, change): return { - 'tryserver.nacl': { t: set(['defaulttests']) for t in TRYBOTS }, + 'client.gyp': { t: set(['defaulttests']) for t in TRYBOTS }, } diff --git a/gyp/buildbot/buildbot_run.py b/gyp/buildbot/buildbot_run.py index f46ab1822f..9a2b71f1b3 100755 --- a/gyp/buildbot/buildbot_run.py +++ b/gyp/buildbot/buildbot_run.py @@ -3,27 +3,17 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Argument-less script to select what to run on the buildbots.""" - -import filecmp import os import shutil import subprocess import sys -if sys.platform in ['win32', 'cygwin']: - EXE_SUFFIX = '.exe' -else: - EXE_SUFFIX = '' - - BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__)) TRUNK_DIR = os.path.dirname(BUILDBOT_DIR) ROOT_DIR = os.path.dirname(TRUNK_DIR) -ANDROID_DIR = os.path.join(ROOT_DIR, 'android') CMAKE_DIR = os.path.join(ROOT_DIR, 'cmake') CMAKE_BIN_DIR = os.path.join(CMAKE_DIR, 'bin') OUT_DIR = os.path.join(TRUNK_DIR, 'out') @@ -71,95 +61,6 @@ def PrepareCmake(): CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR) -_ANDROID_SETUP = 'source build/envsetup.sh && lunch full-eng' - - -def PrepareAndroidTree(): - """Prepare an Android tree to run 'android' format tests.""" - if os.environ['BUILDBOT_CLOBBER'] == '1': - print '@@@BUILD_STEP Clobber Android checkout@@@' - shutil.rmtree(ANDROID_DIR) - - # (Re)create the directory so that the following steps will succeed. - if not os.path.isdir(ANDROID_DIR): - os.mkdir(ANDROID_DIR) - - # We use a manifest from the gyp project listing pinned revisions of AOSP to - # use, to ensure that we test against a stable target. This needs to be - # updated to pick up new build system changes sometimes, so we must test if - # it has changed. - manifest_filename = 'aosp_manifest.xml' - gyp_manifest = os.path.join(BUILDBOT_DIR, manifest_filename) - android_manifest = os.path.join(ANDROID_DIR, '.repo', 'manifests', - manifest_filename) - manifest_is_current = (os.path.isfile(android_manifest) and - filecmp.cmp(gyp_manifest, android_manifest)) - if not manifest_is_current: - # It's safe to repeat these steps, so just do them again to make sure we are - # in a good state. - print '@@@BUILD_STEP Initialize Android checkout@@@' - CallSubProcess( - ['repo', 'init', - '-u', 'https://android.googlesource.com/platform/manifest', - '-b', 'master', - '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'], - cwd=ANDROID_DIR) - shutil.copy(gyp_manifest, android_manifest) - - print '@@@BUILD_STEP Sync Android@@@' - CallSubProcess(['repo', 'sync', '-j4', '-m', manifest_filename], - cwd=ANDROID_DIR) - - # If we already built the system image successfully and didn't sync to a new - # version of the source, skip running the build again as it's expensive even - # when there's nothing to do. - system_img = os.path.join(ANDROID_DIR, 'out', 'target', 'product', 'generic', - 'system.img') - if manifest_is_current and os.path.isfile(system_img): - return - - print '@@@BUILD_STEP Build Android@@@' - CallSubProcess( - ['/bin/bash', - '-c', '%s && make -j4' % _ANDROID_SETUP], - cwd=ANDROID_DIR) - - -def StartAndroidEmulator(): - """Start an android emulator from the built android tree.""" - print '@@@BUILD_STEP Start Android emulator@@@' - - CallSubProcess(['/bin/bash', '-c', - '%s && adb kill-server ' % _ANDROID_SETUP], - cwd=ANDROID_DIR) - - # If taskset is available, use it to force adbd to run only on one core, as, - # sadly, it improves its reliability (see crbug.com/268450). - adbd_wrapper = '' - with open(os.devnull, 'w') as devnull_fd: - if subprocess.call(['which', 'taskset'], stdout=devnull_fd) == 0: - adbd_wrapper = 'taskset -c 0' - CallSubProcess(['/bin/bash', '-c', - '%s && %s adb start-server ' % (_ANDROID_SETUP, adbd_wrapper)], - cwd=ANDROID_DIR) - - subprocess.Popen( - ['/bin/bash', '-c', - '%s && emulator -no-window' % _ANDROID_SETUP], - cwd=ANDROID_DIR) - CallSubProcess( - ['/bin/bash', '-c', - '%s && adb wait-for-device' % _ANDROID_SETUP], - cwd=ANDROID_DIR) - - -def StopAndroidEmulator(): - """Stop all android emulators.""" - print '@@@BUILD_STEP Stop Android emulator@@@' - # If this fails, it's because there is no emulator running. - subprocess.call(['pkill', 'emulator.*']) - - def GypTestFormat(title, format=None, msvs_version=None, tests=[]): """Run the gyp tests for a given format, emitting annotator tags. @@ -185,15 +86,7 @@ def GypTestFormat(title, format=None, msvs_version=None, tests=[]): '--format', format, '--path', CMAKE_BIN_DIR, '--chdir', 'gyp'] + tests) - if format == 'android': - # gyptest needs the environment setup from envsetup/lunch in order to build - # using the 'android' backend, so this is done in a single shell. - retcode = subprocess.call( - ['/bin/bash', - '-c', '%s && cd %s && %s' % (_ANDROID_SETUP, ROOT_DIR, command)], - cwd=ANDROID_DIR, env=env) - else: - retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True) + retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True) if retcode: # Emit failure tag, and keep going. print '@@@STEP_FAILURE@@@' @@ -209,15 +102,7 @@ def GypBuild(): print 'Done.' retcode = 0 - # The Android gyp bot runs on linux so this must be tested first. - if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android': - PrepareAndroidTree() - StartAndroidEmulator() - try: - retcode += GypTestFormat('android') - finally: - StopAndroidEmulator() - elif sys.platform.startswith('linux'): + if sys.platform.startswith('linux'): retcode += GypTestFormat('ninja') retcode += GypTestFormat('make') PrepareCmake() diff --git a/gyp/buildbot/commit_queue/cq_config.json b/gyp/buildbot/commit_queue/cq_config.json index bbf20e394f..656c21e54f 100644 --- a/gyp/buildbot/commit_queue/cq_config.json +++ b/gyp/buildbot/commit_queue/cq_config.json @@ -3,7 +3,6 @@ "launched": { "tryserver.nacl": { "gyp-presubmit": ["defaulttests"], - "gyp-android": ["defaulttests"], "gyp-linux": ["defaulttests"], "gyp-mac": ["defaulttests"], "gyp-win32": ["defaulttests"], diff --git a/gyp/gyp_main.py b/gyp/gyp_main.py index 4ec872f0f9..25a6eba94a 100755 --- a/gyp/gyp_main.py +++ b/gyp/gyp_main.py @@ -4,15 +4,13 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import os import sys -# TODO(mark): sys.path manipulation is some temporary testing stuff. -try: - import gyp -except ImportError, e: - import os.path - sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), 'pylib')) - import gyp +# Make sure we're using the version of pylib in this repo, not one installed +# elsewhere on the system. +sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), 'pylib')) +import gyp if __name__ == '__main__': sys.exit(gyp.script_main()) diff --git a/gyp/pylib/gyp/MSVSSettings.py b/gyp/pylib/gyp/MSVSSettings.py index dde0e07092..4985756bdd 100644 --- a/gyp/pylib/gyp/MSVSSettings.py +++ b/gyp/pylib/gyp/MSVSSettings.py @@ -708,10 +708,7 @@ def _ValidateSettings(validators, settings, stderr): _MSBuildOnly(_compile, 'BuildingInIDE', _boolean) _MSBuildOnly(_compile, 'CompileAsManaged', _Enumeration([], new=['false', - 'true', # /clr - 'Pure', # /clr:pure - 'Safe', # /clr:safe - 'OldSyntax'])) # /clr:oldSyntax + 'true'])) # /clr _MSBuildOnly(_compile, 'CreateHotpatchableImage', _boolean) # /hotpatch _MSBuildOnly(_compile, 'MultiProcessorCompilation', _boolean) # /MP _MSBuildOnly(_compile, 'PreprocessOutputPath', _string) # /Fi diff --git a/gyp/pylib/gyp/MSVSSettings_test.py b/gyp/pylib/gyp/MSVSSettings_test.py index d24dcac4d5..bf6ea6b802 100755 --- a/gyp/pylib/gyp/MSVSSettings_test.py +++ b/gyp/pylib/gyp/MSVSSettings_test.py @@ -296,7 +296,7 @@ def testValidateMSBuildSettings_settings(self): 'BuildingInIDE': 'true', 'CallingConvention': 'Cdecl', 'CompileAs': 'CompileAsC', - 'CompileAsManaged': 'Pure', + 'CompileAsManaged': 'true', 'CreateHotpatchableImage': 'true', 'DebugInformationFormat': 'ProgramDatabase', 'DisableLanguageExtensions': 'true', diff --git a/gyp/pylib/gyp/MSVSVersion.py b/gyp/pylib/gyp/MSVSVersion.py index 92e583fd6e..d9bfa684fa 100644 --- a/gyp/pylib/gyp/MSVSVersion.py +++ b/gyp/pylib/gyp/MSVSVersion.py @@ -84,10 +84,11 @@ def SetupScript(self, target_arch): # vcvars32, which it can only find if VS??COMNTOOLS is set, which it # isn't always. if target_arch == 'x86': - if self.short_name == '2013' and ( + if self.short_name >= '2013' and self.short_name[-1] != 'e' and ( os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'): - # VS2013 non-Express has a x64-x86 cross that we want to prefer. + # VS2013 and later, non-Express have a x64-x86 cross that we want + # to prefer. return [os.path.normpath( os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86'] # Otherwise, the standard x86 compiler. diff --git a/gyp/pylib/gyp/__init__.py b/gyp/pylib/gyp/__init__.py index ac6d918b84..668f38b60d 100755 --- a/gyp/pylib/gyp/__init__.py +++ b/gyp/pylib/gyp/__init__.py @@ -49,7 +49,7 @@ def FindBuildFiles(): def Load(build_files, format, default_variables={}, includes=[], depth='.', params=None, check=False, - circular_check=True): + circular_check=True, duplicate_basename_check=True): """ Loads one or more specified build files. default_variables and includes will be copied before use. @@ -126,6 +126,7 @@ def Load(build_files, format, default_variables={}, # Process the input specific to this generator. result = gyp.input.Load(build_files, default_variables, includes[:], depth, generator_input_info, check, circular_check, + duplicate_basename_check, params['parallel'], params['root_targets']) return [generator] + result @@ -324,6 +325,16 @@ def gyp_main(args): parser.add_option('--no-circular-check', dest='circular_check', action='store_false', default=True, regenerate=False, help="don't check for circular relationships between files") + # --no-duplicate-basename-check disables the check for duplicate basenames + # in a static_library/shared_library project. Visual C++ 2008 generator + # doesn't support this configuration. Libtool on Mac also generates warnings + # when duplicate basenames are passed into Make generator on Mac. + # TODO(yukawa): Remove this option when these legacy generators are + # deprecated. + parser.add_option('--no-duplicate-basename-check', + dest='duplicate_basename_check', action='store_false', + default=True, regenerate=False, + help="don't check for duplicate basenames") parser.add_option('--no-parallel', action='store_true', default=False, help='Disable multiprocessing') parser.add_option('-S', '--suffix', dest='suffix', default='', @@ -499,7 +510,8 @@ def gyp_main(args): # Start with the default variables from the command line. [generator, flat_list, targets, data] = Load( build_files, format, cmdline_default_variables, includes, options.depth, - params, options.check, options.circular_check) + params, options.check, options.circular_check, + options.duplicate_basename_check) # TODO(mark): Pass |data| for now because the generator needs a list of # build files that came in. In the future, maybe it should just accept diff --git a/gyp/pylib/gyp/common.py b/gyp/pylib/gyp/common.py index b6875e43ef..d482a20df3 100644 --- a/gyp/pylib/gyp/common.py +++ b/gyp/pylib/gyp/common.py @@ -131,13 +131,20 @@ def QualifiedTarget(build_file, target, toolset): @memoize -def RelativePath(path, relative_to): +def RelativePath(path, relative_to, follow_path_symlink=True): # Assuming both |path| and |relative_to| are relative to the current # directory, returns a relative path that identifies path relative to # relative_to. + # If |follow_symlink_path| is true (default) and |path| is a symlink, then + # this method returns a path to the real file represented by |path|. If it is + # false, this method returns a path to the symlink. If |path| is not a + # symlink, this option has no effect. # Convert to normalized (and therefore absolute paths). - path = os.path.realpath(path) + if follow_path_symlink: + path = os.path.realpath(path) + else: + path = os.path.abspath(path) relative_to = os.path.realpath(relative_to) # On Windows, we can't create a relative path to a different drive, so just diff --git a/gyp/pylib/gyp/generator/analyzer.py b/gyp/pylib/gyp/generator/analyzer.py index 15b80ef973..f403d4e266 100644 --- a/gyp/pylib/gyp/generator/analyzer.py +++ b/gyp/pylib/gyp/generator/analyzer.py @@ -183,7 +183,10 @@ class Target(object): added_to_compile_targets: used when determining if the target was added to the set of targets that needs to be built. in_roots: true if this target is a descendant of one of the root nodes. - is_executable: true if the type of target is executable.""" + is_executable: true if the type of target is executable. + is_static_library: true if the type of target is static_library. + is_or_has_linked_ancestor: true if the target does a link (eg executable), or + if there is a target in back_deps that does a link.""" def __init__(self, name): self.deps = set() self.match_status = MATCH_STATUS_TBD @@ -196,6 +199,8 @@ def __init__(self, name): self.added_to_compile_targets = False self.in_roots = False self.is_executable = False + self.is_static_library = False + self.is_or_has_linked_ancestor = False class Config(object): @@ -266,8 +271,8 @@ def _GetOrCreateTargetByName(targets, target_name): def _DoesTargetTypeRequireBuild(target_dict): """Returns true if the target type is such that it needs to be built.""" # If a 'none' target has rules or actions we assume it requires a build. - return target_dict['type'] != 'none' or \ - target_dict.get('actions') or target_dict.get('rules') + return bool(target_dict['type'] != 'none' or + target_dict.get('actions') or target_dict.get('rules')) def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files, @@ -309,7 +314,11 @@ def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files, target.visited = True target.requires_build = _DoesTargetTypeRequireBuild( target_dicts[target_name]) - target.is_executable = target_dicts[target_name]['type'] == 'executable' + target_type = target_dicts[target_name]['type'] + target.is_executable = target_type == 'executable' + target.is_static_library = target_type == 'static_library' + target.is_or_has_linked_ancestor = (target_type == 'executable' or + target_type == 'shared_library') build_file = gyp.common.ParseQualifiedTarget(target_name)[0] if not build_file in build_file_in_files: @@ -329,7 +338,7 @@ def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files, sources = _ExtractSources(target_name, target_dicts[target_name], toplevel_dir) for source in sources: - if source in files: + if _ToGypPath(os.path.normpath(source)) in files: print 'target', target_name, 'matches', source target.match_status = MATCH_STATUS_MATCHES matching_targets.append(target) @@ -378,6 +387,7 @@ def _DoesTargetDependOn(target): for dep in target.deps: if _DoesTargetDependOn(dep): target.match_status = MATCH_STATUS_MATCHES_BY_DEPENDENCY + print '\t', target.name, 'matches by dep', dep.name return True target.match_status = MATCH_STATUS_DOESNT_MATCH return False @@ -388,6 +398,7 @@ def _GetTargetsDependingOn(possible_targets): directly on indirectly) on the matched targets. possible_targets: targets to search from.""" found = [] + print 'Targets that matched by dependency:' for target in possible_targets: if _DoesTargetDependOn(target): found.append(target) @@ -411,14 +422,27 @@ def _AddBuildTargets(target, roots, add_if_no_ancestor, result): _AddBuildTargets(back_dep_target, roots, False, result) target.added_to_compile_targets |= back_dep_target.added_to_compile_targets target.in_roots |= back_dep_target.in_roots + target.is_or_has_linked_ancestor |= ( + back_dep_target.is_or_has_linked_ancestor) # Always add 'executable' targets. Even though they may be built by other # targets that depend upon them it makes detection of what is going to be # built easier. + # And always add static_libraries that have no dependencies on them from + # linkables. This is necessary as the other dependencies on them may be + # static libraries themselves, which are not compile time dependencies. if target.in_roots and \ (target.is_executable or (not target.added_to_compile_targets and - (add_if_no_ancestor or target.requires_build))): + (add_if_no_ancestor or target.requires_build)) or + (target.is_static_library and add_if_no_ancestor and + not target.is_or_has_linked_ancestor)): + print '\t\tadding to build targets', target.name, 'executable', \ + target.is_executable, 'added_to_compile_targets', \ + target.added_to_compile_targets, 'add_if_no_ancestor', \ + add_if_no_ancestor, 'requires_build', target.requires_build, \ + 'is_static_library', target.is_static_library, \ + 'is_or_has_linked_ancestor', target.is_or_has_linked_ancestor result.add(target) target.added_to_compile_targets = True @@ -429,6 +453,7 @@ def _GetBuildTargets(matching_targets, roots): roots: set of root targets in the build files to search from.""" result = set() for target in matching_targets: + print '\tfinding build targets for match', target.name _AddBuildTargets(target, roots, True, result) return result @@ -473,7 +498,7 @@ def _WasGypIncludeFileModified(params, files): files.""" if params['options'].includes: for include in params['options'].includes: - if _ToGypPath(include) in files: + if _ToGypPath(os.path.normpath(include)) in files: print 'Include file modified, assuming all changed', include return True return False @@ -536,6 +561,10 @@ def GenerateOutput(target_list, target_dicts, data, params): data, target_list, target_dicts, toplevel_dir, frozenset(config.files), params['build_files']) + print 'roots:' + for root in roots: + print '\t', root.name + unqualified_mapping = _GetUnqualifiedToTargetMapping(all_targets, config.targets) invalid_targets = None @@ -544,10 +573,20 @@ def GenerateOutput(target_list, target_dicts, data, params): if matching_targets: search_targets = _LookupTargets(config.targets, unqualified_mapping) + print 'supplied targets' + for target in config.targets: + print '\t', target + print 'expanded supplied targets' + for target in search_targets: + print '\t', target.name matched_search_targets = _GetTargetsDependingOn(search_targets) + print 'raw matched search targets:' + for target in matched_search_targets: + print '\t', target.name # Reset the visited status for _GetBuildTargets. for target in all_targets.itervalues(): target.visited = False + print 'Finding build targets' build_targets = _GetBuildTargets(matching_targets, roots) matched_search_targets = [gyp.common.ParseQualifiedTarget(target.name)[1] for target in matched_search_targets] diff --git a/gyp/pylib/gyp/generator/cmake.py b/gyp/pylib/gyp/generator/cmake.py index 8f5feddee1..eece6ea98d 100644 --- a/gyp/pylib/gyp/generator/cmake.py +++ b/gyp/pylib/gyp/generator/cmake.py @@ -150,20 +150,17 @@ def SetFileProperty(output, source_name, property_name, values, sep): output.write('")\n') -def SetFilesProperty(output, source_names, property_name, values, sep): +def SetFilesProperty(output, variable, property_name, values, sep): """Given a set of source files, sets the given property on them.""" - output.write('set_source_files_properties(\n') - for source_name in source_names: - output.write(' ') - output.write(source_name) - output.write('\n') - output.write(' PROPERTIES\n ') + output.write('set_source_files_properties(') + WriteVariable(output, variable) + output.write(' PROPERTIES ') output.write(property_name) output.write(' "') for value in values: output.write(CMakeStringEscape(value)) output.write(sep) - output.write('"\n)\n') + output.write('")\n') def SetTargetProperty(output, target_name, property_name, values, sep=''): @@ -236,11 +233,11 @@ def StringToCMakeTargetName(a): """Converts the given string 'a' to a valid CMake target name. All invalid characters are replaced by '_'. - Invalid for cmake: ' ', '/', '(', ')' + Invalid for cmake: ' ', '/', '(', ')', '"' Invalid for make: ':' Invalid for unknown reasons but cause failures: '.' """ - return a.translate(string.maketrans(' /():.', '______')) + return a.translate(string.maketrans(' /():."', '_______')) def WriteActions(target_name, actions, extra_sources, extra_deps, @@ -488,7 +485,7 @@ def __init__(self, ext, command): copy = file_copy if os.path.basename(src) else dir_copy - copy.cmake_inputs.append(NormjoinPath(path_to_gyp, src)) + copy.cmake_inputs.append(NormjoinPathForceCMakeSource(path_to_gyp, src)) copy.cmake_outputs.append(NormjoinPathForceCMakeSource(path_to_gyp, dst)) copy.gyp_inputs.append(src) copy.gyp_outputs.append(dst) @@ -640,6 +637,12 @@ def WriteTarget(namer, qualified_target, target_dicts, build_dir, config_to_use, target_type = spec.get('type', '') target_toolset = spec.get('toolset') + cmake_target_type = cmake_target_type_from_gyp_target_type.get(target_type) + if cmake_target_type is None: + print ('Target %s has unknown target type %s, skipping.' % + ( target_name, target_type ) ) + return + SetVariable(output, 'TARGET', target_name) SetVariable(output, 'TOOLSET', target_toolset) @@ -667,27 +670,89 @@ def WriteTarget(namer, qualified_target, target_dicts, build_dir, config_to_use, srcs = spec.get('sources', []) # Gyp separates the sheep from the goats based on file extensions. - def partition(l, p): - return reduce(lambda x, e: x[not p(e)].append(e) or x, l, ([], [])) - compilable_srcs, other_srcs = partition(srcs, Compilable) + # A full separation is done here because of flag handing (see below). + s_sources = [] + c_sources = [] + cxx_sources = [] + linkable_sources = [] + other_sources = [] + for src in srcs: + _, ext = os.path.splitext(src) + src_type = COMPILABLE_EXTENSIONS.get(ext, None) + src_norm_path = NormjoinPath(path_from_cmakelists_to_gyp, src); - # CMake gets upset when executable targets provide no sources. - if target_type == 'executable' and not compilable_srcs and not extra_sources: - print ('Executable %s has no complilable sources, treating as "none".' % - target_name ) - target_type = 'none' + if src_type == 's': + s_sources.append(src_norm_path) + elif src_type == 'cc': + c_sources.append(src_norm_path) + elif src_type == 'cxx': + cxx_sources.append(src_norm_path) + elif Linkable(ext): + linkable_sources.append(src_norm_path) + else: + other_sources.append(src_norm_path) - cmake_target_type = cmake_target_type_from_gyp_target_type.get(target_type) - if cmake_target_type is None: - print ('Target %s has unknown target type %s, skipping.' % - ( target_name, target_type ) ) - return + for extra_source in extra_sources: + src, real_source = extra_source + _, ext = os.path.splitext(real_source) + src_type = COMPILABLE_EXTENSIONS.get(ext, None) + + if src_type == 's': + s_sources.append(src) + elif src_type == 'cc': + c_sources.append(src) + elif src_type == 'cxx': + cxx_sources.append(src) + elif Linkable(ext): + linkable_sources.append(src) + else: + other_sources.append(src) + + s_sources_name = None + if s_sources: + s_sources_name = cmake_target_name + '__asm_srcs' + SetVariableList(output, s_sources_name, s_sources) + + c_sources_name = None + if c_sources: + c_sources_name = cmake_target_name + '__c_srcs' + SetVariableList(output, c_sources_name, c_sources) + + cxx_sources_name = None + if cxx_sources: + cxx_sources_name = cmake_target_name + '__cxx_srcs' + SetVariableList(output, cxx_sources_name, cxx_sources) + + linkable_sources_name = None + if linkable_sources: + linkable_sources_name = cmake_target_name + '__linkable_srcs' + SetVariableList(output, linkable_sources_name, linkable_sources) + + other_sources_name = None + if other_sources: + other_sources_name = cmake_target_name + '__other_srcs' + SetVariableList(output, other_sources_name, other_sources) + + # CMake gets upset when executable targets provide no sources. + # http://www.cmake.org/pipermail/cmake/2010-July/038461.html + dummy_sources_name = None + has_sources = (s_sources_name or + c_sources_name or + cxx_sources_name or + linkable_sources_name or + other_sources_name) + if target_type == 'executable' and not has_sources: + dummy_sources_name = cmake_target_name + '__dummy_srcs' + SetVariable(output, dummy_sources_name, + "${obj}.${TOOLSET}/${TARGET}/genc/dummy.c") + output.write('if(NOT EXISTS "') + WriteVariable(output, dummy_sources_name) + output.write('")\n') + output.write(' file(WRITE "') + WriteVariable(output, dummy_sources_name) + output.write('" "")\n') + output.write("endif()\n") - other_srcs_name = None - if other_srcs: - other_srcs_name = cmake_target_name + '__other_srcs' - SetVariableList(output, other_srcs_name, - [NormjoinPath(path_from_cmakelists_to_gyp, src) for src in other_srcs]) # CMake is opposed to setting linker directories and considers the practice # of setting linker directories dangerous. Instead, it favors the use of @@ -713,31 +778,48 @@ def partition(l, p): output.write(' ') output.write(cmake_target_type.modifier) - if other_srcs_name: - WriteVariable(output, other_srcs_name, ' ') - - output.write('\n') - - for src in compilable_srcs: - output.write(' ') - output.write(NormjoinPath(path_from_cmakelists_to_gyp, src)) - output.write('\n') - for extra_source in extra_sources: - output.write(' ') - src, _ = extra_source - output.write(NormjoinPath(path_from_cmakelists_to_gyp, src)) - output.write('\n') + if s_sources_name: + WriteVariable(output, s_sources_name, ' ') + if c_sources_name: + WriteVariable(output, c_sources_name, ' ') + if cxx_sources_name: + WriteVariable(output, cxx_sources_name, ' ') + if linkable_sources_name: + WriteVariable(output, linkable_sources_name, ' ') + if other_sources_name: + WriteVariable(output, other_sources_name, ' ') + if dummy_sources_name: + WriteVariable(output, dummy_sources_name, ' ') output.write(')\n') + # Let CMake know if the 'all' target should depend on this target. + exclude_from_all = ('TRUE' if qualified_target not in all_qualified_targets + else 'FALSE') + SetTargetProperty(output, cmake_target_name, + 'EXCLUDE_FROM_ALL', exclude_from_all) + for extra_target_name in extra_deps: + SetTargetProperty(output, extra_target_name, + 'EXCLUDE_FROM_ALL', exclude_from_all) + # Output name and location. if target_type != 'none': + # Link as 'C' if there are no other files + if not c_sources and not cxx_sources: + SetTargetProperty(output, cmake_target_name, 'LINKER_LANGUAGE', ['C']) + # Mark uncompiled sources as uncompiled. - if other_srcs_name: + if other_sources_name: output.write('set_source_files_properties(') - WriteVariable(output, other_srcs_name, '') + WriteVariable(output, other_sources_name, '') output.write(' PROPERTIES HEADER_FILE_ONLY "TRUE")\n') + # Mark object sources as linkable. + if linkable_sources_name: + output.write('set_source_files_properties(') + WriteVariable(output, other_sources_name, '') + output.write(' PROPERTIES EXTERNAL_OBJECT "TRUE")\n') + # Output directory target_output_directory = spec.get('product_dir') if target_output_directory is None: @@ -804,122 +886,84 @@ def partition(l, p): cmake_target_output_basename) SetFileProperty(output, cmake_target_output, 'GENERATED', ['TRUE'], '') - # Let CMake know if the 'all' target should depend on this target. - exclude_from_all = ('TRUE' if qualified_target not in all_qualified_targets - else 'FALSE') - SetTargetProperty(output, cmake_target_name, - 'EXCLUDE_FROM_ALL', exclude_from_all) - for extra_target_name in extra_deps: - SetTargetProperty(output, extra_target_name, - 'EXCLUDE_FROM_ALL', exclude_from_all) - - # Includes - includes = config.get('include_dirs') - if includes: - # This (target include directories) is what requires CMake 2.8.8 - includes_name = cmake_target_name + '__include_dirs' - SetVariableList(output, includes_name, - [NormjoinPathForceCMakeSource(path_from_cmakelists_to_gyp, include) - for include in includes]) - output.write('set_property(TARGET ') - output.write(cmake_target_name) - output.write(' APPEND PROPERTY INCLUDE_DIRECTORIES ') - WriteVariable(output, includes_name, '') - output.write(')\n') - - # Defines - defines = config.get('defines') - if defines is not None: - SetTargetProperty(output, - cmake_target_name, - 'COMPILE_DEFINITIONS', - defines, - ';') - - # Compile Flags - http://www.cmake.org/Bug/view.php?id=6493 - # CMake currently does not have target C and CXX flags. - # So, instead of doing... - - # cflags_c = config.get('cflags_c') - # if cflags_c is not None: - # SetTargetProperty(output, cmake_target_name, - # 'C_COMPILE_FLAGS', cflags_c, ' ') - - # cflags_cc = config.get('cflags_cc') - # if cflags_cc is not None: - # SetTargetProperty(output, cmake_target_name, - # 'CXX_COMPILE_FLAGS', cflags_cc, ' ') - - # Instead we must... - s_sources = [] - c_sources = [] - cxx_sources = [] - for src in srcs: - _, ext = os.path.splitext(src) - src_type = COMPILABLE_EXTENSIONS.get(ext, None) - - if src_type == 's': - s_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cc': - c_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cxx': - cxx_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - for extra_source in extra_sources: - src, real_source = extra_source - _, ext = os.path.splitext(real_source) - src_type = COMPILABLE_EXTENSIONS.get(ext, None) - - if src_type == 's': - s_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cc': - c_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cxx': - cxx_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - cflags = config.get('cflags', []) - cflags_c = config.get('cflags_c', []) - cflags_cxx = config.get('cflags_cc', []) - if c_sources and not (s_sources or cxx_sources): - flags = [] - flags.extend(cflags) - flags.extend(cflags_c) - SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - - elif cxx_sources and not (s_sources or c_sources): - flags = [] - flags.extend(cflags) - flags.extend(cflags_cxx) - SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - - else: - if s_sources and cflags: - SetFilesProperty(output, s_sources, 'COMPILE_FLAGS', cflags, ' ') + # Includes + includes = config.get('include_dirs') + if includes: + # This (target include directories) is what requires CMake 2.8.8 + includes_name = cmake_target_name + '__include_dirs' + SetVariableList(output, includes_name, + [NormjoinPathForceCMakeSource(path_from_cmakelists_to_gyp, include) + for include in includes]) + output.write('set_property(TARGET ') + output.write(cmake_target_name) + output.write(' APPEND PROPERTY INCLUDE_DIRECTORIES ') + WriteVariable(output, includes_name, '') + output.write(')\n') - if c_sources and (cflags or cflags_c): + # Defines + defines = config.get('defines') + if defines is not None: + SetTargetProperty(output, + cmake_target_name, + 'COMPILE_DEFINITIONS', + defines, + ';') + + # Compile Flags - http://www.cmake.org/Bug/view.php?id=6493 + # CMake currently does not have target C and CXX flags. + # So, instead of doing... + + # cflags_c = config.get('cflags_c') + # if cflags_c is not None: + # SetTargetProperty(output, cmake_target_name, + # 'C_COMPILE_FLAGS', cflags_c, ' ') + + # cflags_cc = config.get('cflags_cc') + # if cflags_cc is not None: + # SetTargetProperty(output, cmake_target_name, + # 'CXX_COMPILE_FLAGS', cflags_cc, ' ') + + # Instead we must... + cflags = config.get('cflags', []) + cflags_c = config.get('cflags_c', []) + cflags_cxx = config.get('cflags_cc', []) + if (not cflags_c or not c_sources) and (not cflags_cxx or not cxx_sources): + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', cflags, ' ') + + elif c_sources and not (s_sources or cxx_sources): flags = [] flags.extend(cflags) flags.extend(cflags_c) - SetFilesProperty(output, c_sources, 'COMPILE_FLAGS', flags, ' ') + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - if cxx_sources and (cflags or cflags_cxx): + elif cxx_sources and not (s_sources or c_sources): flags = [] flags.extend(cflags) flags.extend(cflags_cxx) - SetFilesProperty(output, cxx_sources, 'COMPILE_FLAGS', flags, ' ') + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - # Have assembly link as c if there are no other files - if not c_sources and not cxx_sources and s_sources: - SetTargetProperty(output, cmake_target_name, 'LINKER_LANGUAGE', ['C']) - - # Linker flags - ldflags = config.get('ldflags') - if ldflags is not None: - SetTargetProperty(output, cmake_target_name, 'LINK_FLAGS', ldflags, ' ') + else: + # TODO: This is broken, one cannot generally set properties on files, + # as other targets may require different properties on the same files. + if s_sources and cflags: + SetFilesProperty(output, s_sources_name, 'COMPILE_FLAGS', cflags, ' ') + + if c_sources and (cflags or cflags_c): + flags = [] + flags.extend(cflags) + flags.extend(cflags_c) + SetFilesProperty(output, c_sources_name, 'COMPILE_FLAGS', flags, ' ') + + if cxx_sources and (cflags or cflags_cxx): + flags = [] + flags.extend(cflags) + flags.extend(cflags_cxx) + SetFilesProperty(output, cxx_sources_name, 'COMPILE_FLAGS', flags, ' ') + + # Linker flags + ldflags = config.get('ldflags') + if ldflags is not None: + SetTargetProperty(output, cmake_target_name, 'LINK_FLAGS', ldflags, ' ') # Note on Dependencies and Libraries: # CMake wants to handle link order, resolving the link line up front. @@ -1040,19 +1084,48 @@ def GenerateOutputForConfig(target_list, target_dicts, data, output.write('cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)\n') output.write('cmake_policy(VERSION 2.8.8)\n') - _, project_target, _ = gyp.common.ParseQualifiedTarget(target_list[-1]) + gyp_file, project_target, _ = gyp.common.ParseQualifiedTarget(target_list[-1]) output.write('project(') output.write(project_target) output.write(')\n') SetVariable(output, 'configuration', config_to_use) + ar = None + cc = None + cxx = None + + make_global_settings = data[gyp_file].get('make_global_settings', []) + build_to_top = gyp.common.InvertRelativePath(build_dir, + options.toplevel_dir) + for key, value in make_global_settings: + if key == 'AR': + ar = os.path.join(build_to_top, value) + if key == 'CC': + cc = os.path.join(build_to_top, value) + if key == 'CXX': + cxx = os.path.join(build_to_top, value) + + ar = gyp.common.GetEnvironFallback(['AR_target', 'AR'], ar) + cc = gyp.common.GetEnvironFallback(['CC_target', 'CC'], cc) + cxx = gyp.common.GetEnvironFallback(['CXX_target', 'CXX'], cxx) + + if ar: + SetVariable(output, 'CMAKE_AR', ar) + if cc: + SetVariable(output, 'CMAKE_C_COMPILER', cc) + if cxx: + SetVariable(output, 'CMAKE_CXX_COMPILER', cxx) + # The following appears to be as-yet undocumented. # http://public.kitware.com/Bug/view.php?id=8392 output.write('enable_language(ASM)\n') # ASM-ATT does not support .S files. # output.write('enable_language(ASM-ATT)\n') + if cc: + SetVariable(output, 'CMAKE_ASM_COMPILER', cc) + SetVariable(output, 'builddir', '${CMAKE_BINARY_DIR}') SetVariable(output, 'obj', '${builddir}/obj') output.write('\n') @@ -1066,6 +1139,11 @@ def GenerateOutputForConfig(target_list, target_dicts, data, output.write('set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE 1)\n') output.write('\n') + # Force ninja to use rsp files. Otherwise link and ar lines can get too long, + # resulting in 'Argument list too long' errors. + output.write('set(CMAKE_NINJA_FORCE_RESPONSE_FILE 1)\n') + output.write('\n') + namer = CMakeNamer(target_list) # The list of targets upon which the 'all' target should depend. diff --git a/gyp/pylib/gyp/generator/dump_dependency_json.py b/gyp/pylib/gyp/generator/dump_dependency_json.py index 927ba6ebad..160eafe2ef 100644 --- a/gyp/pylib/gyp/generator/dump_dependency_json.py +++ b/gyp/pylib/gyp/generator/dump_dependency_json.py @@ -14,6 +14,9 @@ generator_wants_static_library_dependencies_adjusted = False +generator_filelist_paths = { +} + generator_default_variables = { } for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR', @@ -56,6 +59,17 @@ def CalculateGeneratorInputInfo(params): global generator_wants_static_library_dependencies_adjusted generator_wants_static_library_dependencies_adjusted = True + toplevel = params['options'].toplevel_dir + generator_dir = os.path.relpath(params['options'].generator_output or '.') + # output_dir: relative path from generator_dir to the build directory. + output_dir = generator_flags.get('output_dir', 'out') + qualified_out_dir = os.path.normpath(os.path.join( + toplevel, generator_dir, output_dir, 'gypfiles')) + global generator_filelist_paths + generator_filelist_paths = { + 'toplevel': toplevel, + 'qualified_out_dir': qualified_out_dir, + } def GenerateOutput(target_list, target_dicts, data, params): # Map of target -> list of targets it depends on. @@ -74,7 +88,11 @@ def GenerateOutput(target_list, target_dicts, data, params): edges[target].append(dep) targets_to_visit.append(dep) - filename = 'dump.json' + try: + filepath = params['generator_flags']['output_dir'] + except KeyError: + filepath = '.' + filename = os.path.join(filepath, 'dump.json') f = open(filename, 'w') json.dump(edges, f) f.close() diff --git a/gyp/pylib/gyp/generator/make.py b/gyp/pylib/gyp/generator/make.py index 06c7fdc2e8..786b4e0791 100644 --- a/gyp/pylib/gyp/generator/make.py +++ b/gyp/pylib/gyp/generator/make.py @@ -211,10 +211,10 @@ def CalculateGeneratorInputInfo(params): LINK_COMMANDS_AIX = """\ quiet_cmd_alink = AR($(TOOLSET)) $@ -cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^) +cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) -X32_64 crs $@ $(filter %.o,$^) quiet_cmd_alink_thin = AR($(TOOLSET)) $@ -cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^) +cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) -X32_64 crs $@ $(filter %.o,$^) quiet_cmd_link = LINK($(TOOLSET)) $@ cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS) @@ -273,9 +273,9 @@ def CalculateGeneratorInputInfo(params): %(make_global_settings)s CC.target ?= %(CC.target)s -CFLAGS.target ?= $(CFLAGS) +CFLAGS.target ?= $(CPPFLAGS) $(CFLAGS) CXX.target ?= %(CXX.target)s -CXXFLAGS.target ?= $(CXXFLAGS) +CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS) LINK.target ?= %(LINK.target)s LDFLAGS.target ?= $(LDFLAGS) AR.target ?= $(AR) @@ -286,9 +286,9 @@ def CalculateGeneratorInputInfo(params): # TODO(evan): move all cross-compilation logic to gyp-time so we don't need # to replicate this environment fallback in make as well. CC.host ?= %(CC.host)s -CFLAGS.host ?= +CFLAGS.host ?= $(CPPFLAGS_host) $(CFLAGS_host) CXX.host ?= %(CXX.host)s -CXXFLAGS.host ?= +CXXFLAGS.host ?= $(CPPFLAGS_host) $(CXXFLAGS_host) LINK.host ?= %(LINK.host)s LDFLAGS.host ?= AR.host ?= %(AR.host)s @@ -365,7 +365,7 @@ def CalculateGeneratorInputInfo(params): quiet_cmd_copy = COPY $@ # send stderr to /dev/null to ignore messages when linking directories. -cmd_copy = rm -rf "$@" && cp -af "$<" "$@" +cmd_copy = ln -f "$<" "$@" 2>/dev/null || (rm -rf "$@" && cp %(copy_archive_args)s "$<" "$@") %(link_commands)s """ @@ -1019,7 +1019,8 @@ def WriteRules(self, rules, extra_sources, extra_outputs, # accidentally writing duplicate dummy rules for those outputs. self.WriteLn('%s: obj := $(abs_obj)' % outputs[0]) self.WriteLn('%s: builddir := $(abs_builddir)' % outputs[0]) - self.WriteMakeRule(outputs, inputs + ['FORCE_DO_CMD'], actions) + self.WriteMakeRule(outputs, inputs, actions, + command="%s_%d" % (name, count)) # Spaces in rule filenames are not supported, but rule variables have # spaces in them (e.g. RULE_INPUT_PATH expands to '$(abspath $<)'). # The spaces within the variables are valid, so remove the variables @@ -1688,6 +1689,7 @@ def WriteDoCmd(self, outputs, inputs, command, part_of_all, comment=None, self.WriteMakeRule(outputs, inputs, actions = ['$(call do_cmd,%s%s)' % (command, suffix)], comment = comment, + command = command, force = True) # Add our outputs to the list of targets we read depfiles from. # all_deps is only used for deps file reading, and for deps files we replace @@ -1698,7 +1700,7 @@ def WriteDoCmd(self, outputs, inputs, command, part_of_all, comment=None, def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, - order_only=False, force=False, phony=False): + order_only=False, force=False, phony=False, command=None): """Write a Makefile rule, with some extra tricks. outputs: a list of outputs for the rule (note: this is not directly @@ -1711,6 +1713,7 @@ def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, force: if true, include FORCE_DO_CMD as an order-only dep phony: if true, the rule does not actually generate the named output, the output is just a name to run the rule + command: (optional) command name to generate unambiguous labels """ outputs = map(QuoteSpaces, outputs) inputs = map(QuoteSpaces, inputs) @@ -1719,44 +1722,38 @@ def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, self.WriteLn('# ' + comment) if phony: self.WriteLn('.PHONY: ' + ' '.join(outputs)) - # TODO(evanm): just make order_only a list of deps instead of these hacks. - if order_only: - order_insert = '| ' - pick_output = ' '.join(outputs) - else: - order_insert = '' - pick_output = outputs[0] - if force: - force_append = ' FORCE_DO_CMD' - else: - force_append = '' if actions: self.WriteLn("%s: TOOLSET := $(TOOLSET)" % outputs[0]) - self.WriteLn('%s: %s%s%s' % (pick_output, order_insert, ' '.join(inputs), - force_append)) + force_append = ' FORCE_DO_CMD' if force else '' + + if order_only: + # Order only rule: Just write a simple rule. + # TODO(evanm): just make order_only a list of deps instead of this hack. + self.WriteLn('%s: | %s%s' % + (' '.join(outputs), ' '.join(inputs), force_append)) + elif len(outputs) == 1: + # Regular rule, one output: Just write a simple rule. + self.WriteLn('%s: %s%s' % (outputs[0], ' '.join(inputs), force_append)) + else: + # Regular rule, more than one output: Multiple outputs are tricky in + # make. We will write three rules: + # - All outputs depend on an intermediate file. + # - Make .INTERMEDIATE depend on the intermediate. + # - The intermediate file depends on the inputs and executes the + # actual command. + # - The intermediate recipe will 'touch' the intermediate file. + # - The multi-output rule will have an do-nothing recipe. + intermediate = "%s.intermediate" % (command if command else self.target) + self.WriteLn('%s: %s' % (' '.join(outputs), intermediate)) + self.WriteLn('\t%s' % '@:'); + self.WriteLn('%s: %s' % ('.INTERMEDIATE', intermediate)) + self.WriteLn('%s: %s%s' % + (intermediate, ' '.join(inputs), force_append)) + actions.insert(0, '$(call do_cmd,touch)') + if actions: for action in actions: self.WriteLn('\t%s' % action) - if not order_only and len(outputs) > 1: - # If we have more than one output, a rule like - # foo bar: baz - # that for *each* output we must run the action, potentially - # in parallel. That is not what we're trying to write -- what - # we want is that we run the action once and it generates all - # the files. - # http://www.gnu.org/software/hello/manual/automake/Multiple-Outputs.html - # discusses this problem and has this solution: - # 1) Write the naive rule that would produce parallel runs of - # the action. - # 2) Make the outputs seralized on each other, so we won't start - # a parallel run until the first run finishes, at which point - # we'll have generated all the outputs and we're done. - self.WriteLn('%s: %s' % (' '.join(outputs[1:]), outputs[0])) - # Add a dummy command to the "extra outputs" rule, otherwise make seems to - # think these outputs haven't (couldn't have?) changed, and thus doesn't - # flag them as changed (i.e. include in '$?') when evaluating dependent - # rules, which in turn causes do_cmd() to skip running dependent commands. - self.WriteLn('%s: ;' % (' '.join(outputs[1:]))) self.WriteLn() @@ -1920,13 +1917,11 @@ def _InstallableTargetInstallPath(self): """Returns the location of the final output for an installable target.""" # Xcode puts shared_library results into PRODUCT_DIR, and some gyp files # rely on this. Emulate this behavior for mac. - - # XXX(TooTallNate): disabling this code since we don't want this behavior... - #if (self.type == 'shared_library' and - # (self.flavor != 'mac' or self.toolset != 'target')): - # # Install all shared libs into a common directory (per toolset) for - # # convenient access with LD_LIBRARY_PATH. - # return '$(builddir)/lib.%s/%s' % (self.toolset, self.alias) + if (self.type == 'shared_library' and + (self.flavor != 'mac' or self.toolset != 'target')): + # Install all shared libs into a common directory (per toolset) for + # convenient access with LD_LIBRARY_PATH. + return '$(builddir)/lib.%s/%s' % (self.toolset, self.alias) return '$(builddir)/' + self.alias @@ -2015,6 +2010,7 @@ def CalculateMakefilePath(build_file, base_name): srcdir_prefix = '$(srcdir)/' flock_command= 'flock' + copy_archive_arguments = '-af' header_params = { 'default_target': default_target, 'builddir': builddir_name, @@ -2024,6 +2020,7 @@ def CalculateMakefilePath(build_file, base_name): 'link_commands': LINK_COMMANDS_LINUX, 'extra_commands': '', 'srcdir': srcdir, + 'copy_archive_args': copy_archive_arguments, } if flavor == 'mac': flock_command = './gyp-mac-tool flock' @@ -2048,7 +2045,9 @@ def CalculateMakefilePath(build_file, base_name): 'flock': 'lockf', }) elif flavor == 'aix': + copy_archive_arguments = '-pPRf' header_params.update({ + 'copy_archive_args': copy_archive_arguments, 'link_commands': LINK_COMMANDS_AIX, 'flock': './gyp-flock-tool flock', 'flock_index': 2, diff --git a/gyp/pylib/gyp/generator/msvs.py b/gyp/pylib/gyp/generator/msvs.py index 8e6bd7ba0a..44cc1304a2 100644 --- a/gyp/pylib/gyp/generator/msvs.py +++ b/gyp/pylib/gyp/generator/msvs.py @@ -86,6 +86,9 @@ def _import_OrderedDict(): 'msvs_enable_winrt', 'msvs_requires_importlibrary', 'msvs_enable_winphone', + 'msvs_application_type_revision', + 'msvs_target_platform_version', + 'msvs_target_platform_minversion', ] @@ -2344,6 +2347,9 @@ def _GenerateMSBuildRuleTargetsFile(targets_path, msbuild_rules): rule_name, {'Condition': "'@(%s)' != '' and '%%(%s.ExcludedFromBuild)' != " "'true'" % (rule_name, rule_name), + 'EchoOff': 'true', + 'StandardOutputImportance': 'High', + 'StandardErrorImportance': 'High', 'CommandLineTemplate': '%%(%s.CommandLineTemplate)' % rule_name, 'AdditionalOptions': '%%(%s.AdditionalOptions)' % rule_name, 'Inputs': rule_inputs @@ -2634,8 +2640,23 @@ def _GetMSBuildGlobalProperties(spec, guid, gyp_file_name): if spec.get('msvs_enable_winrt'): properties[0].append(['DefaultLanguage', 'en-US']) properties[0].append(['AppContainerApplication', 'true']) - properties[0].append(['ApplicationTypeRevision', '8.1']) - + if spec.get('msvs_application_type_revision'): + app_type_revision = spec.get('msvs_application_type_revision') + properties[0].append(['ApplicationTypeRevision', app_type_revision]) + else: + properties[0].append(['ApplicationTypeRevision', '8.1']) + + if spec.get('msvs_target_platform_version'): + target_platform_version = spec.get('msvs_target_platform_version') + properties[0].append(['WindowsTargetPlatformVersion', + target_platform_version]) + if spec.get('msvs_target_platform_minversion'): + target_platform_minversion = spec.get('msvs_target_platform_minversion') + properties[0].append(['WindowsTargetPlatformMinVersion', + target_platform_minversion]) + else: + properties[0].append(['WindowsTargetPlatformMinVersion', + target_platform_version]) if spec.get('msvs_enable_winphone'): properties[0].append(['ApplicationType', 'Windows Phone']) else: @@ -2777,9 +2798,6 @@ def _GetMSBuildAttributes(spec, config, build_file): product_name = spec.get('product_name', '$(ProjectName)') target_name = prefix + product_name msbuild_attributes['TargetName'] = target_name - if 'TargetExt' not in msbuild_attributes and 'product_extension' in spec: - ext = spec.get('product_extension') - msbuild_attributes['TargetExt'] = '.' + ext if spec.get('msvs_external_builder'): external_out_dir = spec.get('msvs_external_builder_out_dir', '.') @@ -2833,9 +2851,6 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file): attributes['OutputDirectory']) _AddConditionalProperty(properties, condition, 'TargetName', attributes['TargetName']) - if 'TargetExt' in attributes: - _AddConditionalProperty(properties, condition, 'TargetExt', - attributes['TargetExt']) if attributes.get('TargetPath'): _AddConditionalProperty(properties, condition, 'TargetPath', diff --git a/gyp/pylib/gyp/generator/ninja.py b/gyp/pylib/gyp/generator/ninja.py index 624c99ae89..0e8ae97908 100644 --- a/gyp/pylib/gyp/generator/ninja.py +++ b/gyp/pylib/gyp/generator/ninja.py @@ -921,6 +921,11 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, os.environ.get('CFLAGS', '').split() + cflags_c) cflags_cc = (os.environ.get('CPPFLAGS', '').split() + os.environ.get('CXXFLAGS', '').split() + cflags_cc) + elif self.toolset == 'host': + cflags_c = (os.environ.get('CPPFLAGS_host', '').split() + + os.environ.get('CFLAGS_host', '').split() + cflags_c) + cflags_cc = (os.environ.get('CPPFLAGS_host', '').split() + + os.environ.get('CXXFLAGS_host', '').split() + cflags_cc) defines = config.get('defines', []) + extra_defines self.WriteVariableList(ninja_file, 'defines', @@ -1169,7 +1174,7 @@ def WriteLinkForArch(self, ninja_file, spec, config_name, config, ldflags.append(r'-Wl,-rpath=\$$ORIGIN/%s' % rpath) ldflags.append('-Wl,-rpath-link=%s' % rpath) self.WriteVariableList(ninja_file, 'ldflags', - gyp.common.uniquer(map(self.ExpandSpecial, ldflags))) + map(self.ExpandSpecial, ldflags)) library_dirs = config.get('library_dirs', []) if self.flavor == 'win': @@ -1672,7 +1677,7 @@ def CommandWithWrapper(cmd, wrappers, prog): def GetDefaultConcurrentLinks(): """Returns a best-guess for a number of concurrent links.""" - pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) + pool_size = int(os.environ.get('GYP_LINK_CONCURRENCY', 0)) if pool_size: return pool_size @@ -1696,8 +1701,10 @@ class MEMORYSTATUSEX(ctypes.Structure): stat.dwLength = ctypes.sizeof(stat) ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) - mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB - hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) + # VS 2015 uses 20% more working set than VS 2013 and can consume all RAM + # on a 64 GB machine. + mem_limit = max(1, stat.ullTotalPhys / (5 * (2 ** 30))) # total / 5GB + hard_cap = max(1, int(os.environ.get('GYP_LINK_CONCURRENCY_MAX', 2**32))) return min(mem_limit, hard_cap) elif sys.platform.startswith('linux'): if os.path.exists("/proc/meminfo"): @@ -2235,7 +2242,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, master_ninja.rule( 'copy', description='COPY $in $out', - command='rm -rf $out && cp -af $in $out') + command='ln -f $in $out 2>/dev/null || (rm -rf $out && cp -af $in $out)') master_ninja.newline() all_targets = set() @@ -2275,7 +2282,11 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, if flavor == 'mac': gyp.xcode_emulation.MergeGlobalXcodeSettingsToSpec(data[build_file], spec) - build_file = gyp.common.RelativePath(build_file, options.toplevel_dir) + # If build_file is a symlink, we must not follow it because there's a chance + # it could point to a path above toplevel_dir, and we cannot correctly deal + # with that case at the moment. + build_file = gyp.common.RelativePath(build_file, options.toplevel_dir, + False) qualified_target_for_hash = gyp.common.QualifiedTarget(build_file, name, toolset) diff --git a/gyp/pylib/gyp/input.py b/gyp/pylib/gyp/input.py index 34fbc54711..bc68c3765d 100644 --- a/gyp/pylib/gyp/input.py +++ b/gyp/pylib/gyp/input.py @@ -57,7 +57,7 @@ def IsPathSection(section): # If section ends in one of the '=+?!' characters, it's applied to a section # without the trailing characters. '/' is notably absent from this list, # because there's no way for a regular expression to be treated as a path. - while section[-1:] in '=+?!': + while section and section[-1:] in '=+?!': section = section[:-1] if section in path_sections: @@ -893,11 +893,15 @@ def ExpandVariables(input, phase, variables, build_file): else: # Fix up command with platform specific workarounds. contents = FixupPlatformCommand(contents) - p = subprocess.Popen(contents, shell=use_shell, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - cwd=build_file_dir) + try: + p = subprocess.Popen(contents, shell=use_shell, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + cwd=build_file_dir) + except Exception, e: + raise GypError("%s while executing command '%s' in %s" % + (e, contents, build_file)) p_stdout, p_stderr = p.communicate('') @@ -905,8 +909,8 @@ def ExpandVariables(input, phase, variables, build_file): sys.stderr.write(p_stderr) # Simulate check_call behavior, since check_call only exists # in python 2.5 and later. - raise GypError("Call to '%s' returned exit status %d." % - (contents, p.returncode)) + raise GypError("Call to '%s' returned exit status %d while in %s." % + (contents, p.returncode, build_file)) replacement = p_stdout.rstrip() cached_command_results[cache_key] = replacement @@ -1662,8 +1666,8 @@ def DeepDependencies(self, dependencies=None): if dependency.ref is None: continue if dependency.ref not in dependencies: - dependencies.add(dependency.ref) dependency.DeepDependencies(dependencies) + dependencies.add(dependency.ref) return dependencies @@ -2488,6 +2492,35 @@ def ValidateTargetType(target, target_dict): target_type)) +def ValidateSourcesInTarget(target, target_dict, build_file, + duplicate_basename_check): + if not duplicate_basename_check: + return + if target_dict.get('type', None) != 'static_library': + return + sources = target_dict.get('sources', []) + basenames = {} + for source in sources: + name, ext = os.path.splitext(source) + is_compiled_file = ext in [ + '.c', '.cc', '.cpp', '.cxx', '.m', '.mm', '.s', '.S'] + if not is_compiled_file: + continue + basename = os.path.basename(name) # Don't include extension. + basenames.setdefault(basename, []).append(source) + + error = '' + for basename, files in basenames.iteritems(): + if len(files) > 1: + error += ' %s: %s\n' % (basename, ' '.join(files)) + + if error: + print('static library %s has several files with the same basename:\n' % + target + error + 'libtool on Mac cannot handle that. Use ' + '--no-duplicate-basename-check to disable this validation.') + raise GypError('Duplicate basenames in sources section, see list above') + + def ValidateRulesInTarget(target, target_dict, extra_sources_for_rules): """Ensures that the rules sections in target_dict are valid and consistent, and determines which sources they apply to. @@ -2708,7 +2741,7 @@ def SetGeneratorGlobals(generator_input_info): def Load(build_files, variables, includes, depth, generator_input_info, check, - circular_check, parallel, root_targets): + circular_check, duplicate_basename_check, parallel, root_targets): SetGeneratorGlobals(generator_input_info) # A generator can have other lists (in addition to sources) be processed # for rules. @@ -2840,6 +2873,8 @@ def Load(build_files, variables, includes, depth, generator_input_info, check, target_dict = targets[target] build_file = gyp.common.BuildFile(target) ValidateTargetType(target, target_dict) + ValidateSourcesInTarget(target, target_dict, build_file, + duplicate_basename_check) ValidateRulesInTarget(target, target_dict, extra_sources_for_rules) ValidateRunAsInTarget(target, target_dict, build_file) ValidateActionsInTarget(target, target_dict, build_file) diff --git a/gyp/pylib/gyp/mac_tool.py b/gyp/pylib/gyp/mac_tool.py index 366439a062..eeeaceb0c7 100755 --- a/gyp/pylib/gyp/mac_tool.py +++ b/gyp/pylib/gyp/mac_tool.py @@ -603,8 +603,7 @@ def _ExpandVariables(self, data, substitutions): if isinstance(data, list): return [self._ExpandVariables(v, substitutions) for v in data] if isinstance(data, dict): - return dict((k, self._ExpandVariables(data[k], - substitutions)) for k in data) + return {k: self._ExpandVariables(data[k], substitutions) for k in data} return data if __name__ == '__main__': diff --git a/gyp/pylib/gyp/msvs_emulation.py b/gyp/pylib/gyp/msvs_emulation.py index ce5c46ea5b..ca67b122f0 100644 --- a/gyp/pylib/gyp/msvs_emulation.py +++ b/gyp/pylib/gyp/msvs_emulation.py @@ -442,6 +442,7 @@ def GetCflags(self, config): cl('FloatingPointModel', map={'0': 'precise', '1': 'strict', '2': 'fast'}, prefix='/fp:', default='0') + cl('CompileAsManaged', map={'false': '', 'true': '/clr'}) cl('WholeProgramOptimization', map={'true': '/GL'}) cl('WarningLevel', prefix='/W') cl('WarnAsError', map={'true': '/WX'}) @@ -593,6 +594,15 @@ def GetLdflags(self, config, gyp_to_build_path, expand_special, '2': 'WINDOWS%s' % minimum_required_version}, prefix='/SUBSYSTEM:') + stack_reserve_size = self._Setting( + ('VCLinkerTool', 'StackReserveSize'), config, default='') + if stack_reserve_size: + stack_commit_size = self._Setting( + ('VCLinkerTool', 'StackCommitSize'), config, default='') + if stack_commit_size: + stack_commit_size = ',' + stack_commit_size + ldflags.append('/STACK:%s%s' % (stack_reserve_size, stack_commit_size)) + ld('TerminalServerAware', map={'1': ':NO', '2': ''}, prefix='/TSAWARE') ld('LinkIncremental', map={'1': ':NO', '2': ''}, prefix='/INCREMENTAL') ld('BaseAddress', prefix='/BASE:') diff --git a/gyp/pylib/gyp/win_tool.py b/gyp/pylib/gyp/win_tool.py index 417e465f78..bb6f1ea436 100755 --- a/gyp/pylib/gyp/win_tool.py +++ b/gyp/pylib/gyp/win_tool.py @@ -123,7 +123,9 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): stderr=subprocess.STDOUT) out, _ = link.communicate() for line in out.splitlines(): - if not line.startswith(' Creating library '): + if (not line.startswith(' Creating library ') and + not line.startswith('Generating code') and + not line.startswith('Finished generating code')): print line return link.returncode diff --git a/gyp/pylib/gyp/xcode_emulation.py b/gyp/pylib/gyp/xcode_emulation.py index f1a839a2f5..407ead074b 100644 --- a/gyp/pylib/gyp/xcode_emulation.py +++ b/gyp/pylib/gyp/xcode_emulation.py @@ -525,6 +525,13 @@ def GetCflags(self, configname, arch=None): if self._Test('GCC_WARN_ABOUT_MISSING_NEWLINE', 'YES', default='NO'): cflags.append('-Wnewline-eof') + # In Xcode, this is only activated when GCC_COMPILER_VERSION is clang or + # llvm-gcc. It also requires a fairly recent libtool, and + # if the system clang isn't used, DYLD_LIBRARY_PATH needs to contain the + # path to the libLTO.dylib that matches the used clang. + if self._Test('LLVM_LTO', 'YES', default='NO'): + cflags.append('-flto') + self._AppendPlatformVersionMinFlags(cflags) # TODO: @@ -831,8 +838,9 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): # These flags reflect the compilation options used by xcode to compile # extensions. ldflags.append('-lpkstart') - ldflags.append(sdk_root + - '/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit') + if XcodeVersion() < '0900': + ldflags.append(sdk_root + + '/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit') ldflags.append('-fapplication-extension') ldflags.append('-Xlinker -rpath ' '-Xlinker @executable_path/../../Frameworks') @@ -1024,7 +1032,23 @@ def _AdjustLibrary(self, library, config_name=None): sdk_root = self._SdkPath(config_name) if not sdk_root: sdk_root = '' - return l.replace('$(SDKROOT)', sdk_root) + # Xcode 7 started shipping with ".tbd" (text based stubs) files instead of + # ".dylib" without providing a real support for them. What it does, for + # "/usr/lib" libraries, is do "-L/usr/lib -lname" which is dependent on the + # library order and cause collision when building Chrome. + # + # Instead substitude ".tbd" to ".dylib" in the generated project when the + # following conditions are both true: + # - library is referenced in the gyp file as "$(SDKROOT)/**/*.dylib", + # - the ".dylib" file does not exists but a ".tbd" file do. + library = l.replace('$(SDKROOT)', sdk_root) + if l.startswith('$(SDKROOT)'): + basename, ext = os.path.splitext(library) + if ext == '.dylib' and not os.path.exists(library): + tbd_library = basename + '.tbd' + if os.path.exists(tbd_library): + library = tbd_library + return library def AdjustLibraries(self, libraries, config_name=None): """Transforms entries like 'Cocoa.framework' in libraries into entries like