forked from pex-tool/pex
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pex.py
executable file
·946 lines (840 loc) · 35.8 KB
/
pex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
"""
The pex.bin.pex utility builds PEX environments and .pex files specified by
sources, requirements and their dependencies.
"""
from __future__ import absolute_import, print_function
import itertools
import json
import os
import shlex
import sys
from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser
from textwrap import TextWrapper
from pex import pex_warnings
from pex.argparse import HandleBoolAction
from pex.commands.command import (
GlobalConfigurationError,
global_environment,
register_global_arguments,
)
from pex.common import die, safe_mkdtemp
from pex.enum import Enum
from pex.inherit_path import InheritPath
from pex.interpreter_constraints import InterpreterConstraints
from pex.layout import Layout, maybe_install
from pex.orderedset import OrderedSet
from pex.pex import PEX
from pex.pex_bootstrapper import ensure_venv
from pex.pex_builder import CopyMode, PEXBuilder
from pex.pex_info import PexInfo
from pex.resolve import requirement_options, resolver_options, target_configuration, target_options
from pex.resolve.config import finalize as finalize_resolve_config
from pex.resolve.configured_resolver import ConfiguredResolver
from pex.resolve.lock_resolver import resolve_from_lock
from pex.resolve.pex_repository_resolver import resolve_from_pex
from pex.resolve.requirement_configuration import RequirementConfiguration
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
)
from pex.resolve.resolvers import Unsatisfiable
from pex.resolver import resolve
from pex.result import catch, try_
from pex.targets import Targets
from pex.tracer import TRACER
from pex.typing import TYPE_CHECKING, cast
from pex.variables import ENV, Variables
from pex.venv.bin_path import BinPath
from pex.version import __version__
if TYPE_CHECKING:
from argparse import Namespace
from typing import Dict, List, Optional
from pex.resolve.resolver_options import ResolverConfiguration
CANNOT_SETUP_INTERPRETER = 102
INVALID_OPTIONS = 103
def log(msg, V=0):
if V != 0:
print(msg, file=sys.stderr)
class HandleVenvAction(Action):
def __init__(self, *args, **kwargs):
kwargs["nargs"] = "?"
kwargs["choices"] = (BinPath.PREPEND.value, BinPath.APPEND.value)
super(HandleVenvAction, self).__init__(*args, **kwargs)
def __call__(self, parser, namespace, value, option_str=None):
bin_path = BinPath.FALSE if value is None else BinPath.for_value(value)
setattr(namespace, self.dest, bin_path)
class PrintVariableHelpAction(Action):
def __call__(self, parser, namespace, values, option_str=None):
for variable_name, variable_type, variable_help in Variables.iter_help():
print("\n%s: %s\n" % (variable_name, variable_type))
for line in TextWrapper(initial_indent=" " * 4, subsequent_indent=" " * 4).wrap(
variable_help
):
print(line)
sys.exit(0)
def configure_clp_pex_resolution(parser):
# type: (ArgumentParser) -> None
group = parser.add_argument_group(
title="Resolver options",
description=(
"Tailor how to find, resolve and translate the packages that get put into the PEX "
"environment."
),
)
resolver_options.register(group, include_pex_repository=True, include_lock=True)
group.add_argument(
"--pex-path",
dest="pex_path",
type=str,
default=None,
help=(
"A {pathsep!r} separated list of other pex files to merge into the runtime "
"environment.".format(pathsep=os.pathsep)
),
)
def configure_clp_pex_options(parser):
# type: (ArgumentParser) -> None
group = parser.add_argument_group(
"PEX output options",
"Tailor the behavior of the emitted .pex file if -o is specified.",
)
group.add_argument(
"--include-tools",
dest="include_tools",
default=False,
action=HandleBoolAction,
help="Whether to include runtime tools in the pex file. If included, these can be run by "
"exporting PEX_TOOLS=1 and following the usage and --help information.",
)
group.add_argument(
"--zip-safe",
"--not-zip-safe",
dest="zip_safe",
metavar="DEPRECATED",
default=None,
action=HandleBoolAction,
help=(
"Deprecated: This option is no longer used since user code is now always unzipped "
"before execution."
),
)
group.add_argument(
"--layout",
dest="layout",
default=Layout.ZIPAPP,
choices=Layout.values(),
type=Layout.for_value,
help=(
"By default, a PEX is created as a single file zipapp when `-o` is specified, but "
"either a packed or loose directory tree based layout can be chosen instead. A packed "
"layout PEX is an executable directory structure designed to have cache-friendly "
"characteristics for syncing incremental updates to PEXed applications over a network. "
"At the top level of the packed directory tree there is an executable `__main__.py`"
"script. The directory can also be executed by passing its path to a Python "
"executable; e.g: `python packed-pex-dir/`. The Pex bootstrap code and all dependency "
"code are packed into individual zip files for efficient caching and syncing. A loose "
"layout PEX is similar to a packed PEX, except that neither the Pex bootstrap code nor "
"the dependency code are packed into zip files, but are instead present as collections "
"of loose files in the directory tree providing different caching and syncing "
"tradeoffs. Both zipapp and packed layouts install themselves in the PEX_ROOT as loose "
"apps by default before executing, but these layouts compose with `--venv` execution "
"mode as well and support `--seed`ing."
),
)
group.add_argument(
"--compress",
"--compressed",
"--no-compress",
"--not-compressed",
"--no-compression",
dest="compress",
default=True,
action=HandleBoolAction,
help=(
"Whether to compress zip entries when creating either a zipapp PEX file or a packed "
"PEX's bootstrap and dependency zip files. Does nothing for loose layout PEXes."
),
)
runtime_mode = group.add_mutually_exclusive_group()
runtime_mode.add_argument(
"--unzip",
"--no-unzip",
dest="unzip",
metavar="DEPRECATED",
default=None,
action=HandleBoolAction,
help=(
"Deprecated: This option is no longer used since unzipping PEX zip files before "
"execution is now the default."
),
)
runtime_mode.add_argument(
"--venv",
dest="venv",
metavar="{prepend,append}",
default=False,
action=HandleVenvAction,
help="Convert the pex file to a venv before executing it. If 'prepend' or 'append' is "
"specified, then all scripts and console scripts provided by distributions in the pex file "
"will be added to the PATH in the corresponding position. If the the pex file will be run "
"multiple times under a stable runtime PEX_ROOT, the venv creation will only be done once "
"and subsequent runs will enjoy lower startup latency.",
)
group.add_argument(
"--venv-copies",
"--no-venv-copies",
dest="venv_copies",
default=False,
action=HandleBoolAction,
help=(
"If --venv is specified, create the venv using copies of base interpreter files "
"instead of symlinks. This allows --venv mode PEXes to work across interpreter "
"upgrades without being forced to remove the PEX_ROOT to allow the venv to re-build "
"using the upgraded interpreter."
),
)
group.add_argument(
"--venv-site-packages-copies",
"--no-venv-site-packages-copies",
dest="venv_site_packages_copies",
default=False,
action=HandleBoolAction,
help=(
"If --venv is specified, populate the venv site packages using hard links or copies of "
"resolved PEX dependencies instead of symlinks. This can be used to work around "
"problems with tools or libraries that are confused by symlinked source files."
),
)
group.add_argument(
"--always-write-cache",
dest="always_write_cache",
default=None,
action="store_true",
help=(
"Deprecated: This option is no longer used; all internally cached distributions in a "
"PEX are always installed into the local Pex dependency cache."
),
)
group.add_argument(
"--ignore-errors",
dest="ignore_errors",
default=False,
action="store_true",
help="Ignore requirement resolution solver errors when building pexes and later invoking "
"them.",
)
group.add_argument(
"--inherit-path",
dest="inherit_path",
default=InheritPath.FALSE,
choices=InheritPath.values(),
type=InheritPath.for_value,
help="Inherit the contents of sys.path (including site-packages, user site-packages and "
"PYTHONPATH) running the pex. Possible values: {false} (does not inherit sys.path), "
"{fallback} (inherits sys.path after packaged dependencies), {prefer} (inherits sys.path "
"before packaged dependencies), No value (alias for prefer, for backwards "
"compatibility).".format(
false=InheritPath.FALSE, fallback=InheritPath.FALLBACK, prefer=InheritPath.PREFER
),
)
group.add_argument(
"--compile",
"--no-compile",
dest="compile",
default=False,
action=HandleBoolAction,
help="Compiling means that the built pex will include .pyc files, which will result in "
"slightly faster startup performance. However, compiling means that the generated pex "
"likely will not be reproducible, meaning that if you were to run `./pex -o` with the "
"same inputs then the new pex would not be byte-for-byte identical to the original.",
)
group.add_argument(
"--use-system-time",
"--no-use-system-time",
dest="use_system_time",
default=False,
action=HandleBoolAction,
help="Use the current system time to generate timestamps for the new pex. Otherwise, Pex "
"will use midnight on January 1, 1980. By using system time, the generated pex "
"will not be reproducible, meaning that if you were to run `./pex -o` with the "
"same inputs then the new pex would not be byte-for-byte identical to the original.",
)
group.add_argument(
"--runtime-pex-root",
dest="runtime_pex_root",
default=None,
help="Specify the pex root to be used in the generated .pex file (if unspecified, "
"uses ~/.pex).",
)
group.add_argument(
"--strip-pex-env",
"--no-strip-pex-env",
dest="strip_pex_env",
default=True,
action=HandleBoolAction,
help="Strip all `PEX_*` environment variables used to control the pex runtime before handing "
"off control to the pex entrypoint. You might want to set this to `False` if the new "
"pex executes other pexes (or the Pex CLI itself) and you want the executed pex to be "
"controllable via `PEX_*` environment variables.",
)
def configure_clp_pex_environment(parser):
# type: (ArgumentParser) -> None
group = parser.add_argument_group(
"PEX target environment options",
"Specify which target environments the PEX should run on. If more than one interpreter or "
"platform is specified, a multi-platform PEX will be created that can run on all specified "
"targets. N.B.: You may need to adjust the `--python-shebang` so that it works in all "
"the specified target environments.",
)
target_options.register(group)
group.add_argument(
"--python-shebang",
dest="python_shebang",
default=None,
help="The exact shebang (#!...) line to add at the top of the PEX file minus the "
"#!. This overrides the default behavior, which picks an environment Python "
"interpreter compatible with the one used to build the PEX file.",
)
group.add_argument(
"--sh-boot",
"--no-sh-boot",
dest="sh_boot",
default=False,
action=HandleBoolAction,
help=(
"Create a modified ZIPAPP that uses `/bin/sh` to boot. If you know the machines that "
"the PEX will be distributed to have POSIX compliant `/bin/sh` (almost all do, "
"see: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/sh.html); then this "
"is probably the way you want your PEX to boot. Instead of launching via a Python "
"shebang, the PEX will launch via a `#!/bin/sh` shebang that executes a small script "
"embedded in the head of the PEX ZIPAPP that performs initial interpreter selection "
"and re-execution of the underlying PEX in a way that is often more robust than a "
"Python shebang and always faster on 2nd and subsequent runs since the sh script has a "
"constant overhead of O(1ms) whereas the Python overhead to perform the same "
"interpreter selection and re-execution is O(100ms)."
),
)
def configure_clp_pex_entry_points(parser):
# type: (ArgumentParser) -> None
group = parser.add_argument_group(
"PEX entry point options",
"Specify what target/module the PEX should invoke if any.",
)
entry_point = group.add_mutually_exclusive_group()
entry_point.add_argument(
"-m",
"-e",
"--entry-point",
dest="entry_point",
metavar="MODULE[:SYMBOL]",
default=None,
help="Set the entry point to module or module:symbol. If just specifying module, pex "
"behaves like python -m, e.g. python -m SimpleHTTPServer. If specifying "
"module:symbol, pex assume symbol is a n0-arg callable and imports that symbol and invokes "
"it as if via `sys.exit(symbol())`.",
)
entry_point.add_argument(
"-c",
"--script",
"--console-script",
dest="script",
default=None,
metavar="SCRIPT_NAME",
help="Set the entry point as to the script or console_script as defined by a any of the "
'distributions in the pex. For example: "pex -c fab fabric" or "pex -c mturk boto".',
)
entry_point.add_argument(
"--exe",
"--executable",
"--python-script",
dest="executable",
default=None,
metavar="EXECUTABLE",
help=(
"Set the entry point to an existing local python script. For example: "
'"pex --exe bin/my-python-script".'
),
)
group.add_argument(
"--validate-entry-point",
dest="validate_ep",
default=False,
action="store_true",
help="Validate the entry point by importing it in separate process. Warning: this could have "
"side effects. For example, entry point `a.b.c:m` will translate to "
"`from a.b.c import m` during validation.",
)
class InjectEnvAction(Action):
def __call__(self, parser, namespace, value, option_str=None):
components = value.split("=", 1)
if len(components) != 2:
raise ArgumentError(
self,
"Environment variable values must be of the form `name=value`. "
"Given: {value}".format(value=value),
)
self.default.append(tuple(components))
group.add_argument(
"--inject-env",
dest="inject_env",
default=[],
action=InjectEnvAction,
help="Environment variables to freeze in to the application environment.",
)
class InjectArgAction(Action):
def __call__(self, parser, namespace, value, option_str=None):
self.default.extend(shlex.split(value))
group.add_argument(
"--inject-args",
dest="inject_args",
default=[],
action=InjectArgAction,
help="Command line arguments to the application to freeze in.",
)
class Seed(Enum["Seed.Value"]):
class Value(Enum.Value):
pass
NONE = Value("none")
ARGS = Value("args")
VERBOSE = Value("verbose")
class HandleSeedAction(Action):
def __init__(self, *args, **kwargs):
kwargs["nargs"] = "?"
kwargs["choices"] = [seed.value for seed in Seed.values()]
super(HandleSeedAction, self).__init__(*args, **kwargs)
def __call__(self, parser, namespace, value, option_str=None):
seed = Seed.ARGS if value is None else Seed.for_value(value)
setattr(namespace, self.dest, seed)
def configure_clp():
# type: () -> ArgumentParser
usage = (
"%(prog)s [-o OUTPUT.PEX] [options] [-- arg1 arg2 ...]\n\n"
"%(prog)s builds a PEX (Python Executable) file based on the given specifications: "
"sources, requirements, their dependencies and other options."
"\n"
"Command-line options can be provided in one or more files by prefixing the filenames "
"with an @ symbol. These files must contain one argument per line."
)
parser = ArgumentParser(
usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter,
fromfile_prefix_chars="@",
)
parser.add_argument("-V", "--version", action="version", version=__version__)
configure_clp_pex_resolution(parser)
configure_clp_pex_options(parser)
configure_clp_pex_environment(parser)
configure_clp_pex_entry_points(parser)
parser.add_argument(
"-o",
"--output-file",
dest="pex_name",
default=None,
help="The name of the generated .pex file: Omitting this will run PEX "
"immediately and not save it to a file.",
)
parser.add_argument(
"-p",
"--preamble-file",
dest="preamble_file",
metavar="FILE",
default=None,
type=str,
help="The name of a file to be included as the preamble for the generated .pex file",
)
parser.add_argument(
"-D",
"--sources-directory",
dest="sources_directory",
metavar="DIR",
default=[],
type=str,
action="append",
help=(
"Add a directory containing sources and/or resources to be packaged into the generated "
".pex file. This option can be used multiple times."
),
)
parser.add_argument(
"-R",
"--resources-directory",
dest="resources_directory",
metavar="DIR",
default=[],
type=str,
action="append",
help=(
"Add resources directory to be packaged into the generated .pex file."
" This option can be used multiple times. DEPRECATED: Use -D/--sources-directory "
"instead."
),
)
requirement_options.register(parser)
parser.add_argument(
"--requirements-pex",
dest="requirements_pexes",
metavar="FILE",
default=[],
type=str,
action="append",
help="Add requirements from the given .pex file. This option can be used multiple times.",
)
register_global_arguments(parser, include_verbosity=True)
parser.add_argument(
"--seed",
dest="seed",
action=HandleSeedAction,
default=Seed.NONE,
help=(
"Seed local Pex caches for the generated PEX and print out the command line to run "
"directly from the seed with ({args}) or else a json object including the 'pex_root' "
"path, the 'python' binary path and the seeded 'pex' path ({seed}).".format(
args=Seed.ARGS, seed=Seed.VERBOSE
)
),
)
parser.add_argument(
"--no-seed",
dest="seed",
action="store_const",
const=Seed.NONE,
metavar="DEPRECATED",
help="Deprecated: Use --seed=none instead.",
)
parser.add_argument(
"--help-variables",
action=PrintVariableHelpAction,
nargs=0,
help="Print out help about the various environment variables used to change the behavior of "
"a running PEX file.",
)
return parser
def build_pex(
requirement_configuration, # type: RequirementConfiguration
resolver_configuration, # type: ResolverConfiguration
interpreter_constraints, # type: InterpreterConstraints
targets, # type: Targets
options, # type: Namespace
):
# type: (...) -> PEXBuilder
preamble = None # type: Optional[str]
if options.preamble_file:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=targets.interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead."
)
if options.zip_safe is not None:
pex_warnings.warn(
"The `--zip-safe/--not-zip-safe` option is deprecated. This option is no longer used "
"since user code is now always unzipped before execution."
)
if options.unzip is not None:
pex_warnings.warn(
"The `--unzip/--no-unzip` option is deprecated. This option is no longer used since "
"unzipping PEX zip files before execution is now the default."
)
if options.always_write_cache is not None:
pex_warnings.warn(
"The `--always-write-cache` option is deprecated. This option is no longer used; all "
"internally cached distributions in a PEX are always installed into the local Pex "
"dependency cache."
)
directories = OrderedSet(
options.sources_directory + options.resources_directory
) # type: OrderedSet[str]
for directory in directories:
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.inject_env = dict(options.inject_env)
pex_info.inject_args = options.inject_args
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv or BinPath.FALSE
pex_info.venv_copies = options.venv_copies
pex_info.venv_site_packages_copies = options.venv_site_packages_copies
pex_info.includes_tools = options.include_tools or options.venv
pex_info.pex_path = options.pex_path.split(os.pathsep) if options.pex_path else ()
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = options.inherit_path
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
pex_info.interpreter_constraints = interpreter_constraints
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(
" ".join(
itertools.chain.from_iterable(
(
requirement_configuration.requirements or (),
requirement_configuration.requirement_files or (),
)
)
)
)
):
try:
if isinstance(resolver_configuration, LockRepositoryConfiguration):
lock = try_(resolver_configuration.parse_lock())
with TRACER.timed(
"Resolving requirements from lock file {lock_file}".format(
lock_file=lock.source
)
):
pip_configuration = resolver_configuration.pip_configuration
result = try_(
resolve_from_lock(
targets=targets,
lock=lock,
resolver=ConfiguredResolver(pip_configuration=pip_configuration),
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
transitive=pip_configuration.transitive,
indexes=pip_configuration.repos_configuration.indexes,
find_links=pip_configuration.repos_configuration.find_links,
resolver_version=pip_configuration.resolver_version,
network_configuration=pip_configuration.network_configuration,
password_entries=pip_configuration.repos_configuration.password_entries,
build=pip_configuration.allow_builds,
use_wheel=pip_configuration.allow_wheels,
prefer_older_binary=pip_configuration.prefer_older_binary,
use_pep517=pip_configuration.use_pep517,
build_isolation=pip_configuration.build_isolation,
compile=options.compile,
max_parallel_jobs=pip_configuration.max_jobs,
pip_version=lock.pip_version,
)
)
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
with TRACER.timed(
"Resolving requirements from PEX {pex_repository}.".format(
pex_repository=resolver_configuration.pex_repository
)
):
result = resolve_from_pex(
targets=targets,
pex=resolver_configuration.pex_repository,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
network_configuration=resolver_configuration.network_configuration,
transitive=resolver_configuration.transitive,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
result = resolve(
targets=targets,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
allow_prereleases=resolver_configuration.allow_prereleases,
transitive=resolver_configuration.transitive,
indexes=resolver_configuration.repos_configuration.indexes,
find_links=resolver_configuration.repos_configuration.find_links,
resolver_version=resolver_configuration.resolver_version,
network_configuration=resolver_configuration.network_configuration,
password_entries=resolver_configuration.repos_configuration.password_entries,
build=resolver_configuration.allow_builds,
use_wheel=resolver_configuration.allow_wheels,
prefer_older_binary=resolver_configuration.prefer_older_binary,
use_pep517=resolver_configuration.use_pep517,
build_isolation=resolver_configuration.build_isolation,
compile=options.compile,
max_parallel_jobs=resolver_configuration.max_jobs,
ignore_errors=options.ignore_errors,
preserve_log=resolver_configuration.preserve_log,
pip_version=resolver_configuration.version,
resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
)
for installed_dist in result.installed_distributions:
pex_builder.add_distribution(
installed_dist.distribution, fingerprint=installed_dist.fingerprint
)
for direct_req in installed_dist.direct_requirements:
pex_builder.add_requirement(direct_req)
except Unsatisfiable as e:
die(str(e))
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
elif options.executable:
pex_builder.set_executable(
filename=options.executable, env_filename="__pex_executable__.py"
)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder
def transform_legacy_arg(arg):
# type: (str) -> str
# inherit-path used to be a boolean arg (so either was absent, or --inherit-path)
# Now it takes a string argument, so --inherit-path is invalid.
# Fix up the args we're about to parse to preserve backwards compatibility.
if arg == "--inherit-path":
return "--inherit-path={}".format(InheritPath.PREFER.value)
return arg
def _compatible_with_current_platform(interpreter, platforms):
if not platforms:
return True
current_platforms = set(interpreter.supported_platforms)
current_platforms.add(None)
return current_platforms.intersection(platforms)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1 :]
except ValueError:
args, cmdline = args, []
options = parser.parse_args(args=args)
try:
with global_environment(options) as env:
requirement_configuration = requirement_options.configure(options)
try:
resolver_configuration = resolver_options.configure(options)
except resolver_options.InvalidConfigurationError as e:
die(str(e))
target_config = target_options.configure(options)
try:
targets = target_config.resolve_targets()
except target_configuration.InterpreterNotFound as e:
die(str(e))
except target_configuration.InterpreterConstraintsNotSatisfied as e:
die(str(e), exit_code=CANNOT_SETUP_INTERPRETER)
resolver_configuration = finalize_resolve_config(
resolver_configuration, targets, context="PEX building"
)
sys.exit(
catch(
do_main,
options=options,
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
interpreter_constraints=target_config.interpreter_constraints,
targets=targets,
cmdline=cmdline,
env=env,
)
)
except GlobalConfigurationError as e:
die(str(e))
def do_main(
options, # type: Namespace
requirement_configuration, # type: RequirementConfiguration
resolver_configuration, # type: ResolverConfiguration
interpreter_constraints, # type: InterpreterConstraints
targets, # type: Targets
cmdline, # type: List[str]
env, # type: Dict[str, str]
):
with TRACER.timed("Building pex"):
pex_builder = build_pex(
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
interpreter_constraints=interpreter_constraints,
targets=targets,
options=options,
)
pex_builder.freeze(bytecode_compile=options.compile)
interpreter = pex_builder.interpreter
pex = PEX(
pex_builder.path(),
interpreter=interpreter,
verify_entry_point=options.validate_ep,
)
pex_file = options.pex_name
if pex_file is not None:
log("Saving PEX file to {pex_file}".format(pex_file=pex_file), V=options.verbosity)
if options.sh_boot:
with TRACER.timed("Creating /bin/sh boot script"):
pex_builder.set_sh_boot_script(
pex_name=pex_file,
targets=targets,
python_shebang=options.python_shebang,
)
pex_builder.build(
pex_file,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time,
layout=options.layout,
compress=options.compress,
)
if options.seed != Seed.NONE:
seed_info = seed_cache(
options,
PEX(pex_file, interpreter=interpreter),
verbose=options.seed == Seed.VERBOSE,
)
print(seed_info)
else:
if not _compatible_with_current_platform(interpreter, targets.platforms):
log("WARNING: attempting to run PEX with incompatible platforms!", V=1)
log(
"Running on platform {} but built for {}".format(
interpreter.platform, ", ".join(map(str, targets.platforms))
),
V=1,
)
log(
"Running PEX file at %s with args %s" % (pex_builder.path(), cmdline),
V=options.verbosity,
)
sys.exit(pex.run(args=list(cmdline), env=env))
def seed_cache(
options, # type: Namespace
pex, # type: PEX
verbose=False, # type : bool
):
# type: (...) -> str
pex_path = cast(str, options.pex_name)
with TRACER.timed("Seeding local caches for {}".format(pex_path)):
pex_info = pex.pex_info()
pex_root = pex_info.pex_root
def create_verbose_info(final_pex_path):
# type: (str) -> Dict[str, str]
return dict(pex_root=pex_root, python=pex.interpreter.binary, pex=final_pex_path)
if options.venv:
with TRACER.timed("Creating venv from {}".format(pex_path)):
with ENV.patch(PEX=os.path.realpath(os.path.expanduser(pex_path))):
venv_pex = ensure_venv(pex)
if verbose:
return json.dumps(create_verbose_info(final_pex_path=venv_pex.pex))
else:
return venv_pex.pex
pex_hash = pex_info.pex_hash
if pex_hash is None:
raise AssertionError(
"There was no pex_hash stored in {} for {}.".format(PexInfo.PATH, pex_path)
)
with TRACER.timed("Seeding caches for {}".format(pex_path)):
final_pex_path = os.path.join(
maybe_install(pex=pex_path, pex_root=pex_root, pex_hash=pex_hash)
or os.path.abspath(pex_path),
"__main__.py",
)
if verbose:
return json.dumps(create_verbose_info(final_pex_path=final_pex_path))
else:
return final_pex_path
if __name__ == "__main__":
main()