Skip to content

Commit 34949c8

Browse files
authored
Tweaks to perf_compare.py script to reduce RAM usage (#19321)
Reduce default parallelism, since it could require many GBs of RAM. Add --multi-file flag which reduces RAM usage further, but performance might be slightly worse.
1 parent fabe37f commit 34949c8

File tree

1 file changed

+15
-4
lines changed

1 file changed

+15
-4
lines changed

misc/perf_compare.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@ def heading(s: str) -> None:
3535
print()
3636

3737

38-
def build_mypy(target_dir: str) -> None:
38+
def build_mypy(target_dir: str, multi_file: bool) -> None:
3939
env = os.environ.copy()
4040
env["CC"] = "clang"
4141
env["MYPYC_OPT_LEVEL"] = "2"
4242
env["PYTHONHASHSEED"] = "1"
43+
if multi_file:
44+
env["MYPYC_MULTI_FILE"] = "1"
4345
cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
4446
subprocess.run(cmd, env=env, check=True, cwd=target_dir)
4547

@@ -110,6 +112,12 @@ def main() -> None:
110112
action="store_true",
111113
help="measure incremental run (fully cached)",
112114
)
115+
parser.add_argument(
116+
"--multi-file",
117+
default=False,
118+
action="store_true",
119+
help="compile each mypy module to a separate C file (reduces RAM use)",
120+
)
113121
parser.add_argument(
114122
"--dont-setup",
115123
default=False,
@@ -127,9 +135,9 @@ def main() -> None:
127135
parser.add_argument(
128136
"-j",
129137
metavar="N",
130-
default=8,
138+
default=4,
131139
type=int,
132-
help="set maximum number of parallel builds (default=8)",
140+
help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!",
133141
)
134142
parser.add_argument(
135143
"-r",
@@ -155,6 +163,7 @@ def main() -> None:
155163
args = parser.parse_args()
156164
incremental: bool = args.incremental
157165
dont_setup: bool = args.dont_setup
166+
multi_file: bool = args.multi_file
158167
commits = args.commit
159168
num_runs: int = args.num_runs + 1
160169
max_workers: int = args.j
@@ -185,7 +194,9 @@ def main() -> None:
185194
print("(This will take a while...)")
186195

187196
with ThreadPoolExecutor(max_workers=max_workers) as executor:
188-
futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs]
197+
futures = [
198+
executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs
199+
]
189200
for future in as_completed(futures):
190201
future.result()
191202

0 commit comments

Comments
 (0)