diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py index eaa16835666f..dfd304053330 100644 --- a/src/python/pants/bin/goal_runner.py +++ b/src/python/pants/bin/goal_runner.py @@ -10,8 +10,6 @@ from pants.base.cmd_line_spec_parser import CmdLineSpecParser from pants.base.workunit import WorkUnit, WorkUnitLabel -from pants.bin.repro import Reproducer -from pants.binaries.binary_util import BinaryUtil from pants.build_graph.build_file_parser import BuildFileParser from pants.engine.native import Native from pants.engine.round_engine import RoundEngine @@ -20,13 +18,9 @@ from pants.goal.run_tracker import RunTracker from pants.help.help_printer import HelpPrinter from pants.init.engine_initializer import EngineInitializer -from pants.init.subprocess import Subprocess from pants.init.target_roots_calculator import TargetRootsCalculator from pants.java.nailgun_executor import NailgunProcessGroup from pants.option.ranked_value import RankedValue -from pants.reporting.reporting import Reporting -from pants.scm.subsystems.changed import Changed -from pants.source.source_root import SourceRootConfig from pants.task.task import QuietTaskMixin from pants.util.filtering import create_filters, wrap_filters @@ -199,19 +193,6 @@ def __init__(self, context, goals, run_tracker, kill_nailguns, exiter=sys.exit): self._kill_nailguns = kill_nailguns self._exiter = exiter - @classmethod - def subsystems(cls): - """Subsystems used outside of any task.""" - return { - SourceRootConfig, - Reporting, - Reproducer, - RunTracker, - Changed, - BinaryUtil.Factory, - Subprocess.Factory - } - def _execute_engine(self): workdir = self._context.options.for_global_scope().pants_workdir if not workdir.endswith('.pants.d'): diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index b833bf35b47e..80b8a7c926d0 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -7,9 +7,10 @@ from pants.base.build_environment import get_buildroot from pants.bin.goal_runner import GoalRunner -from pants.bin.repro import Reproducer from pants.goal.run_tracker import RunTracker -from pants.init.options_initializer import OptionsInitializer +from pants.init.logging import setup_logging_from_options +from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer +from pants.init.repro import Reproducer from pants.option.options_bootstrapper import OptionsBootstrapper from pants.reporting.reporting import Reporting from pants.util.contextutil import hard_exit_handler, maybe_profiled @@ -48,7 +49,10 @@ def _run(self): # Bootstrap options and logging. options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env, args=self._args) - options, build_config = OptionsInitializer(options_bootstrapper, exiter=self._exiter).setup() + bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() + setup_logging_from_options(bootstrap_options) + build_config = BuildConfigInitializer.get(options_bootstrapper) + options = OptionsInitializer.create(options_bootstrapper, build_config) global_options = options.for_global_scope() # Apply exiter options. diff --git a/src/python/pants/build_graph/build_configuration.py b/src/python/pants/build_graph/build_configuration.py index 0c1d1d88b971..a4a5c006c24f 100644 --- a/src/python/pants/build_graph/build_configuration.py +++ b/src/python/pants/build_graph/build_configuration.py @@ -50,7 +50,8 @@ def registered_aliases(self): return BuildFileAliases( targets=target_factories_by_alias, objects=self._exposed_object_by_alias.copy(), - context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy()) + context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy() + ) def register_aliases(self, aliases): """Registers the given aliases to be exposed in parsed BUILD files. diff --git a/src/python/pants/engine/legacy/BUILD b/src/python/pants/engine/legacy/BUILD index 9212f9594a4e..dcd66f1c19d7 100644 --- a/src/python/pants/engine/legacy/BUILD +++ b/src/python/pants/engine/legacy/BUILD @@ -31,6 +31,17 @@ python_library( ], ) +python_library( + name='options_parsing', + sources=['options_parsing.py'], + dependencies=[ + 'src/python/pants/build_graph', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:objects', + 'src/python/pants/option', + ], +) + python_library( name='structs', sources=['structs.py'], diff --git a/src/python/pants/engine/legacy/options_parsing.py b/src/python/pants/engine/legacy/options_parsing.py new file mode 100644 index 000000000000..6aa6b889670b --- /dev/null +++ b/src/python/pants/engine/legacy/options_parsing.py @@ -0,0 +1,61 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.build_graph.build_configuration import BuildConfiguration +from pants.engine.fs import FileContent, PathGlobs +from pants.engine.rules import RootRule, rule +from pants.engine.selectors import Get, Select +from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer +from pants.option.options_bootstrapper import OptionsBootstrapper +from pants.util.objects import datatype + + +class OptionsParseRequest(datatype(['args', 'env'])): + """Represents a request for Options computation.""" + + @classmethod + def create(cls, args, env): + assert isinstance(args, (list, tuple)) + return cls( + tuple(args), + tuple(env.items() if isinstance(env, dict) else env) + ) + + +class Options(datatype(['options', 'build_config'])): + """Represents the result of an Options computation.""" + + +@rule(OptionsBootstrapper, [Select(OptionsParseRequest)]) +def reify_options_bootstrapper(parse_request): + options_bootstrapper = OptionsBootstrapper( + env=dict(parse_request.env), + args=parse_request.args + ) + # TODO: Once we have the ability to get FileContent for arbitrary + # paths outside of the buildroot, we can invert this to use + # OptionsBootstrapper.produce_and_set_bootstrap_options() which + # will yield lists of file paths for use as subject values and permit + # us to avoid the direct file I/O that this rule currently requires. + options_bootstrapper.construct_and_set_bootstrap_options() + yield options_bootstrapper + + +# TODO: Accommodate file_option, dir_option, etc. +@rule(Options, [Select(OptionsBootstrapper), Select(BuildConfiguration)]) +def parse_options(options_bootstrapper, build_config): + options = OptionsInitializer.create(options_bootstrapper, build_config) + options.freeze() + return Options(options, build_config) + + +def create_options_parsing_rules(): + return [ + reify_options_bootstrapper, + parse_options, + RootRule(OptionsParseRequest), + ] diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 32efbfd53fa0..c98fe70e3ff1 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -124,6 +124,10 @@ def __str__(self): class SingletonRule(datatype(['output_constraint', 'value']), Rule): """A default rule for a product, which is thus a singleton for that product.""" + @classmethod + def from_instance(cls, obj): + return cls(type(obj), obj) + def __new__(cls, output_type, value): # Validate result type. if isinstance(output_type, Exactly): diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index 69b88340f20a..19dfcc1361c2 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -16,6 +16,7 @@ python_library( 'src/python/pants/core_tasks', 'src/python/pants/engine/legacy:address_mapper', 'src/python/pants/engine/legacy:graph', + 'src/python/pants/engine/legacy:options_parsing', 'src/python/pants/engine/legacy:parser', 'src/python/pants/engine/legacy:source_mapper', 'src/python/pants/engine/legacy:structs', @@ -26,7 +27,6 @@ python_library( 'src/python/pants/engine:scheduler', 'src/python/pants/goal', 'src/python/pants/goal:run_tracker', - 'src/python/pants/logging', 'src/python/pants/option', 'src/python/pants/process', 'src/python/pants/python', diff --git a/src/python/pants/logging/__init__.py b/src/python/pants/init/build_configuration.py similarity index 100% rename from src/python/pants/logging/__init__.py rename to src/python/pants/init/build_configuration.py diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index 43fa22dfb535..ac96c446404e 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -16,6 +16,7 @@ from pants.engine.legacy.address_mapper import LegacyAddressMapper from pants.engine.legacy.graph import (LegacyBuildGraph, TransitiveHydratedTargets, create_legacy_graph_tasks) +from pants.engine.legacy.options_parsing import create_options_parsing_rules from pants.engine.legacy.parser import LegacyPythonCallbacksParser from pants.engine.legacy.structs import (AppAdaptor, GoTargetAdaptor, JavaLibraryAdaptor, JunitTestsAdaptor, PythonLibraryAdaptor, @@ -26,7 +27,9 @@ from pants.engine.parser import SymbolTable from pants.engine.rules import SingletonRule from pants.engine.scheduler import Scheduler +from pants.init.options_initializer import BuildConfigInitializer from pants.option.global_options import GlobMatchErrorBehavior +from pants.option.options_bootstrapper import OptionsBootstrapper from pants.scm.change_calculator import EngineChangeCalculator from pants.util.objects import datatype @@ -120,16 +123,16 @@ class EngineInitializer(object): """Constructs the components necessary to run the v2 engine with v1 BuildGraph compatibility.""" @staticmethod - def setup_legacy_graph(native, bootstrap_options, build_config): + def setup_legacy_graph(native, bootstrap_options, build_configuration): """Construct and return the components necessary for LegacyBuildGraph construction.""" return EngineInitializer.setup_legacy_graph_extended( bootstrap_options.pants_ignore, bootstrap_options.pants_workdir, bootstrap_options.build_file_imports, - build_config.registered_aliases(), + build_configuration, native=native, glob_match_error_behavior=bootstrap_options.glob_expansion_failure, - rules=build_config.rules(), + rules=build_configuration.rules(), build_ignore_patterns=bootstrap_options.build_ignore, exclude_target_regexps=bootstrap_options.exclude_target_regexp, subproject_roots=bootstrap_options.subproject_roots, @@ -143,7 +146,7 @@ def setup_legacy_graph_extended( pants_ignore_patterns, workdir, build_file_imports_behavior, - build_file_aliases, + build_configuration, build_root=None, native=None, glob_match_error_behavior=None, @@ -165,8 +168,8 @@ def setup_legacy_graph_extended( :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. - :param build_file_aliases: BuildFileAliases to register. - :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` + :param build_configuration: The `BuildConfiguration` object to get build file aliases from. + :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` @@ -181,9 +184,9 @@ def setup_legacy_graph_extended( """ build_root = build_root or get_buildroot() - - if not rules: - rules = [] + build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper()) + build_file_aliases = build_configuration.registered_aliases() + rules = rules or build_configuration.rules() or [] symbol_table = LegacySymbolTable(build_file_aliases) @@ -206,12 +209,15 @@ def setup_legacy_graph_extended( # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( + [ + SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)), + SingletonRule.from_instance(build_configuration), + ] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_process_rules() + create_graph_rules(address_mapper, symbol_table) + - [SingletonRule(GlobMatchErrorBehavior, - GlobMatchErrorBehavior.create(glob_match_error_behavior))] + + create_options_parsing_rules() + rules ) diff --git a/src/python/pants/init/global_subsystems.py b/src/python/pants/init/global_subsystems.py new file mode 100644 index 000000000000..7c456070236c --- /dev/null +++ b/src/python/pants/init/global_subsystems.py @@ -0,0 +1,29 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.binaries.binary_util import BinaryUtil +from pants.goal.run_tracker import RunTracker +from pants.init.repro import Reproducer +from pants.init.subprocess import Subprocess +from pants.reporting.reporting import Reporting +from pants.scm.subsystems.changed import Changed +from pants.source.source_root import SourceRootConfig + + +class GlobalSubsystems(object): + @classmethod + def get(cls): + """Subsystems used outside of any task.""" + return { + SourceRootConfig, + Reporting, + Reproducer, + RunTracker, + Changed, + BinaryUtil.Factory, + Subprocess.Factory + } diff --git a/src/python/pants/logging/setup.py b/src/python/pants/init/logging.py similarity index 90% rename from src/python/pants/logging/setup.py rename to src/python/pants/init/logging.py index 906c91be8c15..9376dde15d9e 100644 --- a/src/python/pants/logging/setup.py +++ b/src/python/pants/init/logging.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import (absolute_import, division, generators, nested_scopes, print_function, @@ -7,6 +7,7 @@ import logging import os +import sys import time from collections import namedtuple from logging import FileHandler, Formatter, StreamHandler @@ -18,8 +19,12 @@ class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_ """A structured result for logging setup.""" -# TODO: Once pantsd had a separate launcher entry point, and so no longer needs to call this -# function, move this into the pants.init package, and remove the pants.logging package. +def setup_logging_from_options(bootstrap_options): + # N.B. quiet help says 'Squelches all console output apart from errors'. + level = 'ERROR' if bootstrap_options.quiet else bootstrap_options.level.upper() + return setup_logging(level, console_stream=sys.stderr, log_dir=bootstrap_options.logdir) + + def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name=None): """Configures logging for a given scope, by default the global scope. diff --git a/src/python/pants/init/options_initializer.py b/src/python/pants/init/options_initializer.py index 24267e0d3837..605308e679dc 100644 --- a/src/python/pants/init/options_initializer.py +++ b/src/python/pants/init/options_initializer.py @@ -14,8 +14,8 @@ from pants.base.exceptions import BuildConfigurationError from pants.goal.goal import Goal from pants.init.extension_loader import load_backends_and_plugins +from pants.init.global_subsystems import GlobalSubsystems from pants.init.plugin_resolver import PluginResolver -from pants.logging.setup import setup_logging from pants.option.global_options import GlobalOptionsRegistrar from pants.subsystem.subsystem import Subsystem @@ -23,55 +23,32 @@ logger = logging.getLogger(__name__) -class OptionsInitializer(object): - """Initializes backends/plugins, global options and logging. +class BuildConfigInitializer(object): + """Initializes a BuildConfiguration object. This class uses a class-level cache for the internally generated `BuildConfiguration` object, which permits multiple invocations in the same runtime context without re-incurring backend & plugin loading, which can be expensive and cause issues (double task registration, etc). """ - # Class-level cache for the `BuildConfiguration` object. - _build_configuration = None - - def __init__(self, options_bootstrapper, working_set=None, exiter=sys.exit): - """ - :param OptionsBootStrapper options_bootstrapper: An options bootstrapper instance. - :param pkg_resources.WorkingSet working_set: The working set of the current run as returned by - PluginResolver.resolve(). - :param func exiter: A function that accepts an exit code value and exits (for tests). - """ - self._options_bootstrapper = options_bootstrapper - self._working_set = working_set or PluginResolver(self._options_bootstrapper).resolve() - self._exiter = exiter - - @classmethod - def _has_build_configuration(cls): - return cls._build_configuration is not None - - @classmethod - def _get_build_configuration(cls): - return cls._build_configuration + _cached_build_config = None @classmethod - def _set_build_configuration(cls, build_configuration): - cls._build_configuration = build_configuration + def get(cls, options_bootstrapper, working_set=None): + if cls._cached_build_config is None: + cls._cached_build_config = cls(options_bootstrapper, working_set).setup() + return cls._cached_build_config @classmethod def reset(cls): - cls._set_build_configuration(None) + cls._cached_build_config = None - def _setup_logging(self, quiet, level, log_dir): - """Initializes logging.""" - # N.B. quiet help says 'Squelches all console output apart from errors'. - level = 'ERROR' if quiet else level.upper() - setup_logging(level, console_stream=sys.stderr, log_dir=log_dir) + def __init__(self, options_bootstrapper, working_set=None): + self._options_bootstrapper = options_bootstrapper + self._bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() + self._working_set = working_set or PluginResolver(self._options_bootstrapper).resolve() def _load_plugins(self, working_set, python_paths, plugins, backend_packages): - """Load backends and plugins. - - :returns: A `BuildConfiguration` object constructed during backend/plugin loading. - """ # Add any extra paths to python path (e.g., for loading extra source backends). for path in python_paths: if path not in sys.path: @@ -81,23 +58,38 @@ def _load_plugins(self, working_set, python_paths, plugins, backend_packages): # Load plugins and backends. return load_backends_and_plugins(plugins, working_set, backend_packages) - def _install_options(self, options_bootstrapper, build_configuration): + def setup(self): + """Load backends and plugins. + + :returns: A `BuildConfiguration` object constructed during backend/plugin loading. + """ + return self._load_plugins( + self._working_set, + self._bootstrap_options.pythonpath, + self._bootstrap_options.plugins, + self._bootstrap_options.backend_packages + ) + + +class OptionsInitializer(object): + """Initializes options.""" + + @staticmethod + def _install_options(options_bootstrapper, build_configuration): """Parse and register options. :returns: An Options object representing the full set of runtime options. """ - # TODO: This inline import is currently necessary to resolve a ~legitimate cycle between - # `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`. - from pants.bin.goal_runner import GoalRunner - # Now that plugins and backends are loaded, we can gather the known scopes. # Gather the optionables that are not scoped to any other. All known scopes are reachable # via these optionables' known_scope_infos() methods. - top_level_optionables = ({GlobalOptionsRegistrar} | - GoalRunner.subsystems() | - build_configuration.subsystems() | - set(Goal.get_optionables())) + top_level_optionables = ( + {GlobalOptionsRegistrar} | + GlobalSubsystems.get() | + build_configuration.subsystems() | + set(Goal.get_optionables()) + ) known_scope_infos = sorted({ si for optionable in top_level_optionables for si in optionable.known_scope_infos() @@ -111,18 +103,11 @@ def _install_options(self, options_bootstrapper, build_configuration): for optionable_cls in distinct_optionable_classes: optionable_cls.register_options_on_scope(options) - # Make the options values available to all subsystems. - Subsystem.set_options(options) - return options - def setup(self, init_logging=True): - """Initializes logging, loads backends/plugins and parses options. - - :param bool init_logging: Whether or not to initialize logging as part of setup. - :returns: A tuple of (options, build_configuration). - """ - global_bootstrap_options = self._options_bootstrapper.get_bootstrap_options().for_global_scope() + @classmethod + def create(cls, options_bootstrapper, build_configuration, init_subsystems=True): + global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if global_bootstrap_options.pants_version != pants_version(): raise BuildConfigurationError( @@ -130,23 +115,10 @@ def setup(self, init_logging=True): .format(global_bootstrap_options.pants_version, pants_version()) ) - # Get logging setup prior to loading backends so that they can log as needed. - if init_logging: - self._setup_logging(global_bootstrap_options.quiet, - global_bootstrap_options.level, - global_bootstrap_options.logdir) - - # Conditionally load backends/plugins and materialize a `BuildConfiguration` object. - if not self._has_build_configuration(): - build_configuration = self._load_plugins(self._working_set, - global_bootstrap_options.pythonpath, - global_bootstrap_options.plugins, - global_bootstrap_options.backend_packages) - self._set_build_configuration(build_configuration) - else: - build_configuration = self._get_build_configuration() - # Parse and register options. - options = self._install_options(self._options_bootstrapper, build_configuration) + options = cls._install_options(options_bootstrapper, build_configuration) - return options, build_configuration + if init_subsystems: + Subsystem.set_options(options) + + return options diff --git a/src/python/pants/bin/repro.py b/src/python/pants/init/repro.py similarity index 100% rename from src/python/pants/bin/repro.py rename to src/python/pants/init/repro.py diff --git a/src/python/pants/init/util.py b/src/python/pants/init/util.py index 039347d80346..8ba3663b268b 100644 --- a/src/python/pants/init/util.py +++ b/src/python/pants/init/util.py @@ -6,7 +6,7 @@ unicode_literals, with_statement) from pants.goal.goal import Goal -from pants.init.options_initializer import OptionsInitializer +from pants.init.options_initializer import BuildConfigInitializer from pants.subsystem.subsystem import Subsystem @@ -22,5 +22,5 @@ def clean_global_runtime_state(reset_subsystem=False): # Reset Goals and Tasks. Goal.clear() - # Reset backend/plugins state. - OptionsInitializer.reset() + # Reset global plugin state. + BuildConfigInitializer.reset() diff --git a/src/python/pants/logging/BUILD b/src/python/pants/logging/BUILD deleted file mode 100644 index 31f53083f944..000000000000 --- a/src/python/pants/logging/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -python_library( - dependencies=[ - 'src/python/pants/util:dirutil', - ], -) diff --git a/src/python/pants/option/config.py b/src/python/pants/option/config.py index 554e38aa4f3f..4df2a1a34f2d 100644 --- a/src/python/pants/option/config.py +++ b/src/python/pants/option/config.py @@ -6,8 +6,10 @@ unicode_literals, with_statement) import getpass +import io import itertools import os +from contextlib import contextmanager import six from six.moves import configparser @@ -33,27 +35,61 @@ class ConfigValidationError(ConfigError): pass @classmethod - def load(cls, configpaths, seed_values=None): + def load_file_contents(cls, file_contents, seed_values=None): + """Loads config from the given string payloads. + + A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT + section, and be available for use in substitutions. The caller may override some of these + seed values. + + :param list[FileContents] file_contents: Load from these FileContents. Later instances take + precedence over earlier ones. If empty, returns an + empty config. + :param seed_values: A dict with optional override seed values for buildroot, pants_workdir, + pants_supportdir and pants_distdir. + """ + + @contextmanager + def opener(file_content): + with io.StringIO(file_content.content.decode('utf-8')) as fh: + yield fh + + return cls._meta_load(opener, file_contents, seed_values) + + @classmethod + def load(cls, config_paths, seed_values=None): """Loads config from the given paths. A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT section, and be available for use in substitutions. The caller may override some of these seed values. - :param list configpaths: Load from these paths. Later instances take precedence over earlier - ones. If empty, returns an empty config. + :param list config_paths: Load from these paths. Later instances take precedence over earlier + ones. If empty, returns an empty config. :param seed_values: A dict with optional override seed values for buildroot, pants_workdir, pants_supportdir and pants_distdir. """ - if not configpaths: + + @contextmanager + def opener(f): + with open(f, 'r') as fh: + yield fh + + return cls._meta_load(opener, config_paths, seed_values) + + @classmethod + def _meta_load(cls, open_ctx, config_items, seed_values=None): + if not config_items: return _EmptyConfig() single_file_configs = [] - for configpath in configpaths: + for config_item in config_items: parser = cls._create_parser(seed_values) - with open(configpath, 'r') as ini: + with open_ctx(config_item) as ini: parser.readfp(ini) - single_file_configs.append(_SingleFileConfig(configpath, parser)) + config_path = config_item.path if hasattr(config_item, 'path') else config_item + single_file_configs.append(_SingleFileConfig(config_path, parser)) + return _ChainedConfig(single_file_configs) @classmethod @@ -80,6 +116,7 @@ def _create_parser(cls, seed_values=None): def update_dir_from_seed_values(key, default): all_seed_values[key] = seed_values.get(key, os.path.join(buildroot, default)) + update_dir_from_seed_values('pants_workdir', '.pants.d') update_dir_from_seed_values('pants_supportdir', 'build-support') update_dir_from_seed_values('pants_distdir', 'dist') diff --git a/src/python/pants/option/options.py b/src/python/pants/option/options.py index a7e9ec481a24..81e85ab16f72 100644 --- a/src/python/pants/option/options.py +++ b/src/python/pants/option/options.py @@ -65,6 +65,9 @@ class Options(object): class OptionTrackerRequiredError(Exception): """Options requires an OptionTracker instance.""" + class FrozenOptionsError(Exception): + """Options are frozen and can't be mutated.""" + @classmethod def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. @@ -150,6 +153,13 @@ def __init__(self, goals, scope_to_flags, target_specs, passthru, passthru_owner self._bootstrap_option_values = bootstrap_option_values self._known_scope_to_info = known_scope_to_info self._option_tracker = option_tracker + self._frozen = False + + # TODO: Eliminate this in favor of a builder/factory. + @property + def frozen(self): + """Whether or not this Options object is frozen from writes.""" + return self._frozen @property def tracker(self): @@ -186,6 +196,10 @@ def known_scope_to_info(self): def scope_to_flags(self): return self._scope_to_flags + def freeze(self): + """Freezes this Options instance.""" + self._frozen = True + def drop_flag_values(self): """Returns a copy of these options that ignores values specified via flags. @@ -236,8 +250,13 @@ def passthru_args_for_scope(self, scope): else: return [] + def _assert_not_frozen(self): + if self._frozen: + raise self.FrozenOptionsError('cannot mutate frozen Options instance {!r}.'.format(self)) + def register(self, scope, *args, **kwargs): """Register an option in the given scope.""" + self._assert_not_frozen() self.get_parser(scope).register(*args, **kwargs) deprecated_scope = self.known_scope_to_info[scope].deprecated_scope if deprecated_scope: @@ -245,6 +264,7 @@ def register(self, scope, *args, **kwargs): def registration_function_for_optionable(self, optionable_class): """Returns a function for registering options on the given scope.""" + self._assert_not_frozen() # TODO(benjy): Make this an instance of a class that implements __call__, so we can # docstring it, and so it's less weird than attatching properties to a function. def register(*args, **kwargs): @@ -258,11 +278,14 @@ def register(*args, **kwargs): def get_parser(self, scope): """Returns the parser for the given scope, so code can register on it directly.""" + self._assert_not_frozen() return self._parser_hierarchy.get_parser_by_scope(scope) def walk_parsers(self, callback): + self._assert_not_frozen() self._parser_hierarchy.walk(callback) + # TODO: Eagerly precompute backing data for this? def for_scope(self, scope, inherit_from_enclosing_scope=True): """Return the option values for the given scope. diff --git a/src/python/pants/option/options_bootstrapper.py b/src/python/pants/option/options_bootstrapper.py index 0241fa384f13..d33418f69982 100644 --- a/src/python/pants/option/options_bootstrapper.py +++ b/src/python/pants/option/options_bootstrapper.py @@ -11,6 +11,7 @@ import sys from pants.base.build_environment import get_default_pants_config_file +from pants.engine.fs import FileContent from pants.option.arg_splitter import GLOBAL_SCOPE, GLOBAL_SCOPE_CONFIG_SECTION from pants.option.config import Config from pants.option.custom_types import ListValueComponent @@ -66,68 +67,100 @@ def __init__(self, env=None, args=None): self._full_options = {} # We memoize the full options here. self._option_tracker = OptionTracker() + def produce_and_set_bootstrap_options(self): + """Cooperatively populates the internal bootstrap_options cache with + a producer of `FileContent`.""" + flags = set() + short_flags = set() + + def capture_the_flags(*args, **kwargs): + for arg in args: + flags.add(arg) + if len(arg) == 2: + short_flags.add(arg) + elif kwargs.get('type') == bool: + flags.add('--no-{}'.format(arg[2:])) + + GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) + + def is_bootstrap_option(arg): + components = arg.split('=', 1) + if components[0] in flags: + return True + for flag in short_flags: + if arg.startswith(flag): + return True + return False + + # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. + # Stop before '--' since args after that are pass-through and may have duplicate names to our + # bootstrap options. + bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) + + config_file_paths = self.get_config_file_paths(env=self._env, args=self._args) + config_files_products = yield config_file_paths + pre_bootstrap_config = Config.load_file_contents(config_files_products) + + def bootstrap_options_from_config(config): + bootstrap_options = Options.create( + env=self._env, + config=config, + known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], + args=bargs, + option_tracker=self._option_tracker + ) + + def register_global(*args, **kwargs): + ## Only use of Options.register? + bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) + + GlobalOptionsRegistrar.register_bootstrap_options(register_global) + return bootstrap_options + + initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) + bootstrap_option_values = initial_bootstrap_options.for_global_scope() + + # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped + # from (typically pants.ini), then config override, then rcfiles. + full_configpaths = pre_bootstrap_config.sources() + if bootstrap_option_values.pantsrc: + rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] + existing_rcfiles = filter(os.path.exists, rcfiles) + full_configpaths.extend(existing_rcfiles) + + full_config_files_products = yield full_configpaths + self._post_bootstrap_config = Config.load_file_contents( + full_config_files_products, + seed_values=bootstrap_option_values + ) + + # Now recompute the bootstrap options with the full config. This allows us to pick up + # bootstrap values (such as backends) from a config override file, for example. + self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config) + + def construct_and_set_bootstrap_options(self): + """Populates the internal bootstrap_options cache.""" + def filecontent_for(path): + with open(path, 'rb') as fh: + return FileContent(path, fh.read()) + + # N.B. This adaptor is meant to simulate how we would co-operatively invoke options bootstrap + # via an `@rule` after we have a solution in place for producing `FileContent` of abspaths. + producer = self.produce_and_set_bootstrap_options() + next_item = None + while 1: + try: + files = next_item or next(producer) + next_item = producer.send([filecontent_for(f) for f in files]) + except StopIteration as e: + break + def get_bootstrap_options(self): """:returns: an Options instance that only knows about the bootstrap options. :rtype: :class:`Options` """ if not self._bootstrap_options: - flags = set() - short_flags = set() - - def capture_the_flags(*args, **kwargs): - for arg in args: - flags.add(arg) - if len(arg) == 2: - short_flags.add(arg) - elif kwargs.get('type') == bool: - flags.add('--no-{}'.format(arg[2:])) - - GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) - - def is_bootstrap_option(arg): - components = arg.split('=', 1) - if components[0] in flags: - return True - for flag in short_flags: - if arg.startswith(flag): - return True - return False - - # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. - # Stop before '--' since args after that are pass-through and may have duplicate names to our - # bootstrap options. - bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) - - configpaths = self.get_config_file_paths(env=self._env, args=self._args) - pre_bootstrap_config = Config.load(configpaths) - - def bootstrap_options_from_config(config): - bootstrap_options = Options.create(env=self._env, config=config, - known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs, - option_tracker=self._option_tracker) - - def register_global(*args, **kwargs): - bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) - GlobalOptionsRegistrar.register_bootstrap_options(register_global) - return bootstrap_options - - initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) - bootstrap_option_values = initial_bootstrap_options.for_global_scope() - - # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped - # from (typically pants.ini), then config override, then rcfiles. - full_configpaths = pre_bootstrap_config.sources() - if bootstrap_option_values.pantsrc: - rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] - existing_rcfiles = filter(os.path.exists, rcfiles) - full_configpaths.extend(existing_rcfiles) - - self._post_bootstrap_config = Config.load(full_configpaths, - seed_values=bootstrap_option_values) - - # Now recompute the bootstrap options with the full config. This allows us to pick up - # bootstrap values (such as backends) from a config override file, for example. - self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config) + self.construct_and_set_bootstrap_options() return self._bootstrap_options def get_full_options(self, known_scope_infos): diff --git a/src/python/pants/pantsd/BUILD b/src/python/pants/pantsd/BUILD index 7b80465cf917..be55f5d3e6e4 100644 --- a/src/python/pants/pantsd/BUILD +++ b/src/python/pants/pantsd/BUILD @@ -64,7 +64,6 @@ python_library( 'src/python/pants/engine:native', 'src/python/pants/goal:run_tracker', 'src/python/pants/init', - 'src/python/pants/logging', 'src/python/pants/pantsd/service:fs_event_service', 'src/python/pants/pantsd/service:pailgun_service', 'src/python/pants/pantsd/service:scheduler_service', diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py index 5fd22ba7d3e6..4f148269c817 100644 --- a/src/python/pants/pantsd/pants_daemon.py +++ b/src/python/pants/pantsd/pants_daemon.py @@ -18,9 +18,9 @@ from pants.bin.daemon_pants_runner import DaemonExiter, DaemonPantsRunner from pants.engine.native import Native from pants.init.engine_initializer import EngineInitializer -from pants.init.options_initializer import OptionsInitializer +from pants.init.logging import setup_logging +from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer from pants.init.target_roots_calculator import TargetRootsCalculator -from pants.logging.setup import setup_logging from pants.option.arg_splitter import GLOBAL_SCOPE from pants.option.options_bootstrapper import OptionsBootstrapper from pants.option.options_fingerprinter import OptionsFingerprinter @@ -116,7 +116,8 @@ def create(cls, bootstrap_options=None, full_init=True): if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) - _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) + options_bootstrapper = OptionsBootstrapper() + build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) diff --git a/src/python/pants/pantsd/service/pailgun_service.py b/src/python/pants/pantsd/service/pailgun_service.py index de5334a46fe0..ec81fd4e492c 100644 --- a/src/python/pants/pantsd/service/pailgun_service.py +++ b/src/python/pants/pantsd/service/pailgun_service.py @@ -11,7 +11,7 @@ import traceback from contextlib import contextmanager -from pants.init.options_initializer import OptionsInitializer +from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer from pants.option.options_bootstrapper import OptionsBootstrapper from pants.pantsd.pailgun_server import PailgunServer from pants.pantsd.service.pants_service import PantsService @@ -59,7 +59,9 @@ def runner_factory(sock, arguments, environment): deferred_exc = None self._logger.debug('execution commandline: %s', arguments) - options, _ = OptionsInitializer(OptionsBootstrapper(args=arguments)).setup(init_logging=False) + options_bootstrapper = OptionsBootstrapper(args=arguments) + build_config = BuildConfigInitializer.get(options_bootstrapper) + options = OptionsInitializer.create(options_bootstrapper, build_config) graph_helper, target_roots = None, None try: diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD index a25870604318..ca281f4ed09b 100644 --- a/tests/python/pants_test/engine/legacy/BUILD +++ b/tests/python/pants_test/engine/legacy/BUILD @@ -141,6 +141,19 @@ python_tests( tags = {'integration'}, ) +python_tests( + name = 'options_parsing', + sources = ['test_options_parsing.py'], + dependencies = [ + '3rdparty/python:mock', + 'src/python/pants/bin', + 'src/python/pants/build_graph', + 'src/python/pants/init', + 'tests/python/pants_test:test_base', + 'tests/python/pants_test/engine:util', + ] +) + python_tests( name = 'pants_engine_integration', sources = ['test_pants_engine_integration.py'], diff --git a/tests/python/pants_test/engine/legacy/test_graph.py b/tests/python/pants_test/engine/legacy/test_graph.py index 35437c71d962..b8afe7ebdacf 100644 --- a/tests/python/pants_test/engine/legacy/test_graph.py +++ b/tests/python/pants_test/engine/legacy/test_graph.py @@ -17,7 +17,7 @@ from pants.build_graph.build_file_aliases import BuildFileAliases, TargetMacro from pants.build_graph.target import Target from pants.init.engine_initializer import EngineInitializer -from pants.init.options_initializer import OptionsInitializer +from pants.init.options_initializer import BuildConfigInitializer from pants.init.target_roots_calculator import TargetRootsCalculator from pants.option.options_bootstrapper import OptionsBootstrapper from pants.subsystem.subsystem import Subsystem @@ -41,36 +41,38 @@ def _make_setup_args(self, specs): options.target_specs = specs return options - def _default_build_file_aliases(self): + def _default_build_config(self, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 - _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) - return build_config.registered_aliases() + build_config = BuildConfigInitializer.get(OptionsBootstrapper()) + if build_file_aliases: + build_config.register_aliases(build_file_aliases) + return build_config @contextmanager def graph_helper(self, - build_file_aliases=None, + build_configuration=None, build_file_imports_behavior='allow', include_trace_on_error=True, path_ignore_patterns=None): with temporary_dir() as work_dir: path_ignore_patterns = path_ignore_patterns or [] - build_file_aliases = build_file_aliases or self._default_build_file_aliases() + build_config = build_configuration or self._default_build_config() # TODO: This test should be swapped to using TestBase. graph_helper = EngineInitializer.setup_legacy_graph_extended( path_ignore_patterns, work_dir, build_file_imports_behavior, - build_file_aliases=build_file_aliases, + build_configuration=build_configuration, native=self._native, include_trace_on_error=include_trace_on_error ) yield graph_helper @contextmanager - def open_scheduler(self, specs, build_file_aliases=None): - with self.graph_helper(build_file_aliases=build_file_aliases) as graph_helper: + def open_scheduler(self, specs, build_configuration=None): + with self.graph_helper(build_configuration=build_configuration) as graph_helper: graph, target_roots = self.create_graph_from_specs(graph_helper, specs) addresses = tuple(graph.inject_roots_closure(target_roots)) yield graph, addresses, graph_helper.scheduler.new_session() @@ -186,10 +188,10 @@ def test_target_macro_override(self): tag_macro = functools.partial(macro, target_cls, tag) target_symbols = {'target': TargetMacro.Factory.wrap(tag_macro, target_cls)} - build_file_aliases = self._default_build_file_aliases().merge(BuildFileAliases(targets=target_symbols)) + build_config = self._default_build_config(BuildFileAliases(targets=target_symbols)) # Confirm that python_tests in a small directory are marked. - with self.open_scheduler([spec], build_file_aliases=build_file_aliases) as (graph, addresses, _): + with self.open_scheduler([spec], build_configuration=build_config) as (graph, addresses, _): self.assertTrue(len(addresses) > 0, 'No targets matched by {}'.format(addresses)) for address in addresses: self.assertIn(tag, graph.get_target(address).tags) diff --git a/tests/python/pants_test/engine/legacy/test_options_parsing.py b/tests/python/pants_test/engine/legacy/test_options_parsing.py new file mode 100644 index 000000000000..ab1393d242b5 --- /dev/null +++ b/tests/python/pants_test/engine/legacy/test_options_parsing.py @@ -0,0 +1,37 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os +import unittest +from contextlib import contextmanager + +from pants.base.build_environment import get_buildroot +from pants.engine.legacy.options_parsing import Options, OptionsParseRequest +from pants.init.engine_initializer import EngineInitializer +from pants_test.engine.util import init_native +from pants_test.test_base import TestBase + + +class TestEngineOptionsParsing(TestBase): + + # TODO: pants_test.engine.util.run_rule ? + def test_options_parsing_request(self): + products = self.scheduler.product_request( + Options, + [ + OptionsParseRequest.create( + ['./pants', '-ldebug', 'binary', 'src/python::'], + dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]') + ) + ] + ) + options = products[0].options + self.assertIn('binary', options.goals) + global_options = options.for_global_scope() + self.assertEquals(global_options.level, 'debug') + self.assertEquals(global_options.enable_pantsd, True) + self.assertEquals(global_options.binaries_baseurls, ['https://bins.com']) diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD index e1a3607e6610..745f59521f4c 100644 --- a/tests/python/pants_test/init/BUILD +++ b/tests/python/pants_test/init/BUILD @@ -3,6 +3,7 @@ python_tests( + sources=rglobs('*.py'), dependencies = [ '3rdparty/python:mock', '3rdparty/python:parameterized', diff --git a/tests/python/pants_test/bin/repro_mixin.py b/tests/python/pants_test/init/repro_mixin.py similarity index 100% rename from tests/python/pants_test/bin/repro_mixin.py rename to tests/python/pants_test/init/repro_mixin.py diff --git a/tests/python/pants_test/logging/test_setup.py b/tests/python/pants_test/init/test_logging.py similarity index 95% rename from tests/python/pants_test/logging/test_setup.py rename to tests/python/pants_test/init/test_logging.py index d5cd41f8019b..dfc7e18f5150 100644 --- a/tests/python/pants_test/logging/test_setup.py +++ b/tests/python/pants_test/init/test_logging.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import (absolute_import, division, generators, nested_scopes, print_function, @@ -12,7 +12,7 @@ import six -from pants.logging.setup import setup_logging +from pants.init.logging import setup_logging from pants.util.contextutil import temporary_dir diff --git a/tests/python/pants_test/init/test_options_initializer.py b/tests/python/pants_test/init/test_options_initializer.py index 4db87b90d190..de89d7099209 100644 --- a/tests/python/pants_test/init/test_options_initializer.py +++ b/tests/python/pants_test/init/test_options_initializer.py @@ -10,14 +10,14 @@ from pkg_resources import WorkingSet from pants.base.exceptions import BuildConfigurationError -from pants.init.options_initializer import OptionsInitializer +from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer from pants.option.options_bootstrapper import OptionsBootstrapper class OptionsInitializerTest(unittest.TestCase): def test_invalid_version(self): options_bootstrapper = OptionsBootstrapper(args=['--pants-version=99.99.9999']) - initializer = OptionsInitializer(options_bootstrapper, WorkingSet()) + build_config = BuildConfigInitializer(options_bootstrapper) with self.assertRaises(BuildConfigurationError): - initializer.setup() + OptionsInitializer.create(options_bootstrapper, build_config) diff --git a/tests/python/pants_test/bin/test_repro.py b/tests/python/pants_test/init/test_repro.py similarity index 94% rename from tests/python/pants_test/bin/test_repro.py rename to tests/python/pants_test/init/test_repro.py index 150b7c3dfe18..353fea494407 100644 --- a/tests/python/pants_test/bin/test_repro.py +++ b/tests/python/pants_test/init/test_repro.py @@ -9,10 +9,10 @@ import unittest from functools import partial -from pants.bin.repro import Repro from pants.fs.archive import TGZ +from pants.init.repro import Repro from pants.util.contextutil import temporary_dir -from pants_test.bin.repro_mixin import ReproMixin +from pants_test.init.repro_mixin import ReproMixin class ReproTest(unittest.TestCase, ReproMixin): diff --git a/tests/python/pants_test/bin/test_repro_ignore.py b/tests/python/pants_test/init/test_repro_ignore.py similarity index 95% rename from tests/python/pants_test/bin/test_repro_ignore.py rename to tests/python/pants_test/init/test_repro_ignore.py index 326bb862bb21..1d3afb92f1e1 100644 --- a/tests/python/pants_test/bin/test_repro_ignore.py +++ b/tests/python/pants_test/init/test_repro_ignore.py @@ -10,10 +10,10 @@ from functools import partial from pants.base.build_root import BuildRoot -from pants.bin.repro import Reproducer from pants.fs.archive import TGZ +from pants.init.repro import Reproducer from pants.util.contextutil import pushd, temporary_dir -from pants_test.bin.repro_mixin import ReproMixin +from pants_test.init.repro_mixin import ReproMixin from pants_test.subsystem.subsystem_util import global_subsystem_instance diff --git a/tests/python/pants_test/logging/BUILD b/tests/python/pants_test/logging/BUILD index 850813e8cc3f..2e9c3b7d91af 100644 --- a/tests/python/pants_test/logging/BUILD +++ b/tests/python/pants_test/logging/BUILD @@ -5,7 +5,6 @@ python_tests( sources=globs('*.py', exclude=['test_native_engine_logging.py', 'test_workunit_label.py']), dependencies=[ '3rdparty/python:six', - 'src/python/pants/logging', 'src/python/pants/util:contextutil', ], ) diff --git a/tests/python/pants_test/option/test_config.py b/tests/python/pants_test/option/test_config.py index 5050806c1381..be9f60d1ba7b 100644 --- a/tests/python/pants_test/option/test_config.py +++ b/tests/python/pants_test/option/test_config.py @@ -15,47 +15,53 @@ class ConfigTest(unittest.TestCase): def setUp(self): + self.ini1_content = textwrap.dedent( + """ + [DEFAULT] + name: foo + answer: 42 + scale: 1.2 + path: /a/b/%(answer)s + embed: %(path)s::foo + disclaimer: + Let it be known + that. + blank_section: + + [a] + list: [1, 2, 3, %(answer)s] + listappend: +[7, 8, 9] + + [b] + preempt: True + dict: { + 'a': 1, + 'b': %(answer)s, + 'c': ['%(answer)s', %(answer)s] + } + """ + ) + + self.ini2_content = textwrap.dedent( + """ + [a] + fast: True + + [b] + preempt: False + + [defined_section] + """ + ) + with temporary_file() as ini1: - ini1.write(textwrap.dedent( - """ - [DEFAULT] - name: foo - answer: 42 - scale: 1.2 - path: /a/b/%(answer)s - embed: %(path)s::foo - disclaimer: - Let it be known - that. - blank_section: - - [a] - list: [1, 2, 3, %(answer)s] - listappend: +[7, 8, 9] - - [b] - preempt: True - dict: { - 'a': 1, - 'b': %(answer)s, - 'c': ['%(answer)s', %(answer)s] - } - """)) + ini1.write(self.ini1_content) ini1.close() with temporary_file() as ini2: - ini2.write(textwrap.dedent( - """ - [a] - fast: True - - [b] - preempt: False - - [defined_section] - """)) + ini2.write(self.ini2_content) ini2.close() - self.config = Config.load(configpaths=[ini1.name, ini2.name]) + self.config = Config.load(config_paths=[ini1.name, ini2.name]) self.assertEqual([ini1.name, ini2.name], self.config.sources()) def test_getstring(self): diff --git a/tests/python/pants_test/option/test_options.py b/tests/python/pants_test/option/test_options.py index dfb7489fe813..9e6c8f9450cf 100644 --- a/tests/python/pants_test/option/test_options.py +++ b/tests/python/pants_test/option/test_options.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import io import os import shlex import tempfile @@ -14,6 +15,7 @@ from textwrap import dedent from pants.base.deprecated import CodeRemovedError +from pants.engine.fs import FileContent from pants.option.arg_splitter import GLOBAL_SCOPE from pants.option.config import Config from pants.option.custom_types import UnsetBool, file_option, target_option @@ -158,13 +160,17 @@ def register_global(*args, **kwargs): options.register('fingerprinting', '--fingerprinted', fingerprint=True) options.register('fingerprinting', '--definitely-not-fingerprinted', fingerprint=False) + @contextmanager + def _write_config_to_file(self, fp, config): + for section, options in config.items(): + fp.write('[{}]\n'.format(section)) + for key, value in options.items(): + fp.write('{}: {}\n'.format(key, value)) + def _create_config(self, config): with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp: - for section, options in config.items(): - fp.write('[{}]\n'.format(section)) - for key, value in options.items(): - fp.write('{}: {}\n'.format(key, value)) - return Config.load(configpaths=[fp.name]) + self._write_config_to_file(fp, config) + return Config.load(config_paths=[fp.name]) def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None): args = shlex.split(str(args_str)) @@ -1337,3 +1343,14 @@ class DummyOptionable1(Optionable): # Check that we got no warnings and that the actual scope took precedence. self.assertEquals(0, len(w)) self.assertEquals('xx', vals1.foo) + + +class OptionsTestStringPayloads(OptionsTest): + """Runs the same tests as OptionsTest, but backed with `Config.loads` vs `Config.load`.""" + + def _create_config_from_strings(self, config): + with io.StringIO('') as fp: + self._write_config_to_file(fp, config) + fp.seek(0) + payload = fp.read() + return Config.load_file_contents(config_payloads=[FileContent('blah', payload)]) diff --git a/tests/python/pants_test/pants_run_integration_test.py b/tests/python/pants_test/pants_run_integration_test.py index d3f1c2f00dca..ced39556d9e8 100644 --- a/tests/python/pants_test/pants_run_integration_test.py +++ b/tests/python/pants_test/pants_run_integration_test.py @@ -406,10 +406,8 @@ def assert_result(self, pants_run, value, expected=True, msg=None): def indent(content): return '\n\t'.join(content.splitlines()) - if pants_run.stdout_data: - details.append('stdout:\n\t{stdout}'.format(stdout=indent(pants_run.stdout_data))) - if pants_run.stderr_data: - details.append('stderr:\n\t{stderr}'.format(stderr=indent(pants_run.stderr_data))) + details.append('stdout:\n\t{stdout}'.format(stdout=indent(pants_run.stdout_data))) + details.append('stderr:\n\t{stderr}'.format(stderr=indent(pants_run.stderr_data))) error_msg = '\n'.join(details) assertion(value, pants_run.returncode, error_msg) diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py index ced93b566f00..fbe819b85a61 100644 --- a/tests/python/pants_test/pantsd/test_pantsd_integration.py +++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py @@ -343,7 +343,8 @@ def test_pantsd_aligned_output(self): checker.assert_started() for cmd, run in zip(cmds, daemon_runs): - self.assertEqual(run.stderr_data.strip(), '', 'Non-empty stderr for {}'.format(cmd)) + stderr_output = run.stderr_data.strip() + self.assertEqual(stderr_output, '', 'Non-empty stderr for {}: {}'.format(cmd, stderr_output)) self.assertNotEqual(run.stdout_data, '', 'Empty stdout for {}'.format(cmd)) for run_pairs in zip(non_daemon_runs, daemon_runs): diff --git a/tests/python/pants_test/test_base.py b/tests/python/pants_test/test_base.py index c2497eb02b90..888c2b5209bb 100644 --- a/tests/python/pants_test/test_base.py +++ b/tests/python/pants_test/test_base.py @@ -242,6 +242,13 @@ def alias_groups(cls): """ return BuildFileAliases(targets={'target': Target}) + + @classmethod + def build_config(cls): + build_config = BuildConfiguration() + build_config.register_aliases(cls.alias_groups()) + return build_config + def setUp(self): """ :API: public @@ -275,8 +282,7 @@ def setUp(self): 'write_to': [], } - self._build_configuration = BuildConfiguration() - self._build_configuration.register_aliases(self.alias_groups()) + self._build_configuration = self.build_config() self._build_file_parser = BuildFileParser(self._build_configuration, self.build_root) def buildroot_files(self, relpath=None): @@ -332,7 +338,7 @@ def _init_engine(cls): workdir=cls._pants_workdir(), build_file_imports_behavior='allow', native=init_native(), - build_file_aliases=cls.alias_groups(), + build_configuration=cls.build_config(), build_ignore_patterns=None, ).new_session() cls._scheduler = graph_session.scheduler_session