Skip to content
Browse files

Parallelize ninja generation when GYP_PARALLEL=1.

This is a really easy approach, parallelizing by configs (Debug/Release). 
It's probably not worth parallelizing this step much more because there's 
some overhead and the total time of this step isn't that large.

This gives a huge speedup on Windows, and it's about even on a fast
Mac/Linux box.

Review URL: http://codereview.chromium.org/11026061/



git-svn-id: http://gyp.googlecode.com/svn/trunk@1518 78cadc50-ecff-11dd-a971-7dbc132099af
  • Loading branch information...
1 parent 6c312e3 commit 5a10131a4bf9462ebf86224278d8db66487064b5 dmazzoni@chromium.org committed Oct 9, 2012
Showing with 20 additions and 8 deletions.
  1. +5 −5 pylib/gyp/__init__.py
  2. +15 −3 pylib/gyp/generator/ninja.py
View
10 pylib/gyp/__init__.py
@@ -47,7 +47,7 @@ def FindBuildFiles():
def Load(build_files, format, default_variables={},
includes=[], depth='.', params=None, check=False,
- circular_check=True, parallel=False):
+ circular_check=True):
"""
Loads one or more specified build files.
default_variables and includes will be copied before use.
@@ -126,7 +126,7 @@ def Load(build_files, format, default_variables={},
# Process the input specific to this generator.
result = gyp.input.Load(build_files, default_variables, includes[:],
depth, generator_input_info, check, circular_check,
- parallel)
+ params['parallel'])
return [generator] + result
def NameValueListToDict(name_value_list):
@@ -488,15 +488,15 @@ def gyp_main(args):
'cwd': os.getcwd(),
'build_files_arg': build_files_arg,
'gyp_binary': sys.argv[0],
- 'home_dot_gyp': home_dot_gyp}
+ 'home_dot_gyp': home_dot_gyp,
+ 'parallel': options.parallel}
# Start with the default variables from the command line.
[generator, flat_list, targets, data] = Load(build_files, format,
cmdline_default_variables,
includes, options.depth,
params, options.check,
- options.circular_check,
- options.parallel)
+ options.circular_check)
# TODO(mark): Pass |data| for now because the generator needs a list of
# build files that came in. In the future, maybe it should just accept
View
18 pylib/gyp/generator/ninja.py
@@ -4,6 +4,7 @@
import copy
import hashlib
+import multiprocessing
import os.path
import re
import subprocess
@@ -1747,13 +1748,24 @@ def PerformBuild(data, configurations, params):
subprocess.check_call(arguments)
+def CallGenerateOutputForConfig(arglist):
+ (target_list, target_dicts, data, params, config_name) = arglist
+ GenerateOutputForConfig(target_list, target_dicts, data, params, config_name)
+
def GenerateOutput(target_list, target_dicts, data, params):
user_config = params.get('generator_flags', {}).get('config', None)
if user_config:
GenerateOutputForConfig(target_list, target_dicts, data, params,
user_config)
else:
config_names = target_dicts[target_list[0]]['configurations'].keys()
- for config_name in config_names:
- GenerateOutputForConfig(target_list, target_dicts, data, params,
- config_name)
+ if params['parallel']:
+ pool = multiprocessing.Pool(len(config_names))
+ arglists = []
+ for config_name in config_names:
+ arglists.append((target_list, target_dicts, data, params, config_name))
+ pool.map(CallGenerateOutputForConfig, arglists)
+ else:
+ for config_name in config_names:
+ GenerateOutputForConfig(target_list, target_dicts, data, params,
+ config_name)

0 comments on commit 5a10131

Please sign in to comment.
Something went wrong with that request. Please try again.