diff --git a/multiconf/__init__.py b/multiconf/__init__.py index c707e27..e0ad84b 100644 --- a/multiconf/__init__.py +++ b/multiconf/__init__.py @@ -4,7 +4,7 @@ # pylint: disable=unused-import from . import py_version_check -from .multiconf import McConfigRoot, AbstractConfigItem, ConfigItem, RepeatableConfigItem, ConfigBuilder +from .multiconf import McConfigRoot, AbstractConfigItem, ConfigItem, RepeatableConfigItem, DefaultItems, ConfigBuilder from .decorators import mc_config from .config_errors import ConfigException, ConfigDefinitionException, ConfigApiException, InvalidUsageException from .config_errors import ConfigAttributeError, ConfigExcludedAttributeError, ConfigExcludedKeyError @@ -13,7 +13,7 @@ __all__ = [ - 'mc_config', 'McConfigRoot', 'AbstractConfigItem', 'ConfigItem', 'RepeatableConfigItem', 'ConfigBuilder', + 'mc_config', 'McConfigRoot', 'AbstractConfigItem', 'ConfigItem', 'RepeatableConfigItem', 'DefaultItems', 'ConfigBuilder', 'ConfigException', 'ConfigDefinitionException', 'ConfigApiException', 'InvalidUsageException', 'ConfigAttributeError', 'ConfigExcludedAttributeError', 'MC_REQUIRED', 'MC_TODO', 'McInvalidValue', 'McTodoHandling', diff --git a/multiconf/decorators.py b/multiconf/decorators.py index e761aca..8960ed3 100644 --- a/multiconf/decorators.py +++ b/multiconf/decorators.py @@ -7,14 +7,14 @@ from .config_errors import ConfigException, ConfigDefinitionException, _line_msg, _error_msg, _warning_msg from .repeatable import RepeatableDict from .multiconf import McConfigRoot -from . import ConfigBuilder, RepeatableConfigItem +from . import ConfigBuilder, RepeatableConfigItem, DefaultItems from .check_identifiers import check_valid_identifier, check_valid_identifiers def _not_allowed_on_class(cls, decorator_name, not_cls): if issubclass(cls, not_cls): print(_line_msg(up_level=2), file=sys.stderr) - msg = "Decorator '@" + decorator_name + "' is not allowed on instance of " + ConfigBuilder.__name__ + "." + msg = "Decorator '@" + decorator_name + "' is not allowed on instance of " + not_cls.__name__ + "." print(_error_msg(msg), file=sys.stderr) raise ConfigDefinitionException(msg) @@ -46,6 +46,7 @@ def named_as(insert_as_name): """Determine the name used to insert item in parent""" def deco(cls): _not_allowed_on_class(cls, named_as.__name__, ConfigBuilder) + _not_allowed_on_class(cls, named_as.__name__, DefaultItems) check_valid_identifier(insert_as_name) cls._mc_deco_named_as = insert_as_name return cls @@ -57,6 +58,7 @@ def nested_repeatables(*attr_names): """Specify which nested (child) items will be repeatable.""" def deco(cls): _not_allowed_on_class(cls, nested_repeatables.__name__, ConfigBuilder) + _not_allowed_on_class(cls, nested_repeatables.__name__, DefaultItems) cls._mc_deco_nested_repeatables = _add_super_list_deco_values(cls, attr_names, 'nested_repeatables') # Make descriptor work, an instance of the descriptor class mut be assigened at the class level @@ -71,6 +73,7 @@ def deco(cls): def required(*attr_names): """Specify nested (child) items that must be defined.""" def deco(cls): + _not_allowed_on_class(cls, required.__name__, DefaultItems) cls._mc_deco_required = _add_super_list_deco_values(cls, attr_names, 'required') return cls diff --git a/multiconf/json_output.py b/multiconf/json_output.py index 40e17af..c7ead51 100644 --- a/multiconf/json_output.py +++ b/multiconf/json_output.py @@ -20,6 +20,8 @@ _property_method_value_hidden = '@property method value - call disabled' _mc_filter_out_keys = ('env', 'env_factory', 'contained_in', 'root_conf', 'attributes', 'mc_config_result', 'num_invalid_property_usage', 'named_as') +_mc_hidden_if_not_true = ('mc_is_default_value_item',) +_mc_show_if_names_only = ('mc_is_default_value_item',) def _class_tuple(obj, obj_info=""): @@ -125,7 +127,12 @@ def _ref_item_str(self, objval): if isinstance(objval, self.with_item_types): return excl + objval.ref_type_info_for_json() + ", id: " + str(self.ref_repr(objval)) - return excl + objval.ref_type_info_for_json() + " " + _mc_identification_msg_str(objval) + try: + ref_type_info = objval.ref_type_info_for_json() + except AttributeError: + ref_type_info = '' + + return excl + ref_type_info + " " + _mc_identification_msg_str(objval) def _ref_earlier_str(self, objval): return "#ref" + self._ref_item_str(objval) @@ -139,7 +146,7 @@ def _ref_self_str(self, objval): def _ref_outside_str(self, objval): # A reference to an item which is outside of the currently dumped hierarchy. # Showing self.ref_repr(obj) does not help here as the object is not dumped, instead try to show some attributes which may identify the object - return "#outside-ref: " + _mc_identification_msg_str(objval) + return "#ref outside: " + _mc_identification_msg_str(objval) def _ref_mc_item_str(self, objval): if ref_id(objval) in self.seen: @@ -247,9 +254,12 @@ def _handle_one_dir_entry_one_env(self, obj, key, _val, env, attributes_overridi continue if isinstance(real_attr, (property, self.multiconf_property_wrapper_type)): + if key in _mc_hidden_if_not_true and not getattr(obj, key): + return key, () + calc_or_static = _calculated_value - if names_only: + if names_only and key not in _mc_show_if_names_only: val = _property_method_value_hidden break @@ -303,7 +313,7 @@ def _handle_one_dir_entry_one_env(self, obj, key, _val, env, attributes_overridi if overridden_property: return key, [(overridden_property + calc_or_static + ' value was', val)] + property_inf if self.compact: - return key, [('', str(val) + calc_or_static)] + property_inf + return key, [('', (str(val).lower() if isinstance(val, bool) else str(val)) + calc_or_static)] + property_inf return key, [('', val), (calc_or_static, True)] + property_inf if isinstance(val, (list, tuple)): diff --git a/multiconf/multiconf.py b/multiconf/multiconf.py index 7e5a0a8..7640b04 100644 --- a/multiconf/multiconf.py +++ b/multiconf/multiconf.py @@ -95,7 +95,14 @@ def _mc_print_value_error_msg(self, attr_name, value, mc_caller_file_name, mc_ca self._mc_print_error(msg, file_name=mc_caller_file_name, line_num=mc_caller_line_num) - def _mc_print_no_proper_value_error_msg(self, mc_error_info_up_level): + def _mc_validate_attributes(self, mc_error_info_up_level): + """Verify that all attributes got a value""" + if not self._mc_attributes_to_check: + return + + if thread_local.is_under_default_item: + return + msg = "The following attribues defined earlier never received a proper value for {env}:".format(env=self.env) if mc_error_info_up_level is not None: mc_caller_file_name, mc_caller_line_num = find_user_file_line(up_level_start=mc_error_info_up_level + 1) @@ -107,6 +114,34 @@ def _mc_print_no_proper_value_error_msg(self, mc_error_info_up_level): value = getattr(self, attr_name) self._mc_print_value_error_msg(attr_name, value, value_file_name, value_line_num) + def _mc_validate_required(self, mc_error_info_up_level): + """Verify that all @required child items are present""" + if not self._mc_deco_required: + return + + missing_req = {} + for req in self._mc_deco_required: + if req not in self.__dict__: + missing_req[req] = None + + # Any required item not present on self, but found on a default value item with same name will be assigned to self + contained_in = self + while contained_in and missing_req: + contained_in, shared_req = contained_in._mc_find_closest_default_item(req) + if not shared_req: + break + + if isinstance(shared_req, (_ConfigBase, RepeatableDict)): + proxy_insert(self, req, shared_req) + del missing_req[req] + continue + + if missing_req: + if mc_error_info_up_level is not None: + mc_caller_file_name, mc_caller_line_num = find_user_file_line(up_level_start=mc_error_info_up_level) + print(_line_msg(file_name=mc_caller_file_name, line_num=mc_caller_line_num), file=sys.stderr) + print(self._mc_error_msg(f"Missing '@required' items: {list(missing_req)}"), file=sys.stderr) + @classmethod def named_as(cls): """Return the named_as property set by the @named_as decorator""" @@ -115,7 +150,7 @@ def named_as(cls): def ref_id_for_json(self): return id(self) - def json(self, compact=False, sort_attributes=False, property_methods=True, builders=False, skipkeys=True, warn_nesting=None, show_all_envs=False, + def json(self, compact=False, sort_attributes=False, property_methods=True, builders=False, default_items=False, skipkeys=True, warn_nesting=None, show_all_envs=False, depth=None, persistent_ids=False): """Create json representation of configuration. @@ -128,6 +163,7 @@ def json(self, compact=False, sort_attributes=False, property_methods=True, buil If `property_methods` is None the @property method is not called but the name is still output, with the value replace by a fixed message. False completely disables information about @property methods. builders (bool): Include ConfigBuilder items in json. + default_items (bool): Include DefaultItems in json. skipkeys (bool): Passed to json.dumps. show_all_envs (bool): Display attribute values for all envs in a single dump. Without this only the values for the current env is displayed. depth (int): The number of levels of child objects to dump. None means all. @@ -139,10 +175,18 @@ def json(self, compact=False, sort_attributes=False, property_methods=True, buil filter_callable = cr._mc_json_filter fallback_callable = cr._mc_json_fallback + + with_item_types = ( + (_ConfigBase, RepeatableConfigItem, RepeatableDict) if (builders and default_items) else + (ConfigBuilder, ConfigItem, RepeatableConfigItem, RepeatableDict) if builders else + (DefaultItems, ConfigItem, RepeatableConfigItem, RepeatableDict) if default_items else + (ConfigItem, RepeatableConfigItem, RepeatableDict) + ) + encoder = ConfigItemEncoder( filter_callable=filter_callable, fallback_callable=fallback_callable, compact=compact, sort_attributes=sort_attributes, property_methods=property_methods, - with_item_types=(RepeatableDict, _ConfigBase if builders else ConfigItem, RepeatableConfigItem), + with_item_types=with_item_types, warn_nesting=warn_nesting, multiconf_base_type=_ConfigBase, multiconf_property_wrapper_type=_McPropertyWrapper, @@ -239,12 +283,50 @@ def _mc_attributes_to_check_add(self, attr_name, mc_error_info_up_level): mc_caller_file_name, mc_caller_line_num = caller_file_line(up_level=mc_error_info_up_level + 1) if self._mc_attributes_to_check is None: self._mc_attributes_to_check = {} + self._mc_attributes_to_check[attr_name] = (self._mc_where, mc_caller_file_name, mc_caller_line_num) def _mc_attributes_to_check_del(self, attr_name): if self._mc_attributes_to_check: self._mc_attributes_to_check.pop(attr_name, None) + def _mc_find_closest_default_item(self, item_name): + contained_in = self._mc_contained_in + while contained_in: + default_items = contained_in.__dict__.get(DefaultItems.name) + if default_items: + default_item = getattr(default_items, item_name, MC_NO_VALUE) + if default_item != MC_NO_VALUE: + if isinstance(default_item, RepeatableDict): + # There should be only one, get it + for default_item in default_item._all_items.values(): # pragma: no branch + break + + return contained_in, default_item + + contained_in = contained_in._mc_contained_in + + return None, None + + def _mc_resolve_shared_attribute(self, default_item, attr_name, mc_error_info_up_level): + """Try to resolve attribute values from default_item. + + Return: True if value is found, False otherwise. + """ + + env_attr = self._mc_attributes[attr_name] + shared_value = getattr(default_item, attr_name, MC_NO_VALUE) + if shared_value in (MC_NO_VALUE, MC_REQUIRED): + return False + + # This will pop attributes from self._mc_attributes_to_check + shared_attr = default_item._mc_attributes[attr_name] + self._mc_setattr_env_value( + thread_local.env, attr_name, env_attr, shared_value, old_value=getattr(self, attr_name), from_eg=shared_attr.from_eg, + mc_force=False, mc_error_info_up_level=mc_error_info_up_level + 1) + + return True + def _mc_freeze(self, mc_error_info_up_level): if not self: self._mc_where = Where.FROZEN @@ -262,8 +344,9 @@ def _mc_freeze(self, mc_error_info_up_level): # Call user 'mc_init' callback self.mc_init() - if self._mc_attributes_to_check and self.mc_validate.__code__ is _ConfigBase.mc_validate.__code__: - self._mc_print_no_proper_value_error_msg(mc_error_info_up_level) + if self.mc_validate.__code__ is _ConfigBase.mc_validate.__code__: + # mc_validate has not been overridden, so we can validate that all attributes have been set now + self._mc_validate_attributes(mc_error_info_up_level) if isinstance(self, ConfigBuilder): self._mc_builder_freeze() @@ -274,15 +357,30 @@ def _mc_freeze(self, mc_error_info_up_level): if must_pop: self.__class__._mc_hierarchy.pop() - missing_req = [] - for req in self._mc_deco_required: - if req not in self.__dict__: - missing_req.append(req) - if missing_req: - if mc_error_info_up_level is not None: - mc_caller_file_name, mc_caller_line_num = find_user_file_line(up_level_start=mc_error_info_up_level) - print(_line_msg(file_name=mc_caller_file_name, line_num=mc_caller_line_num), file=sys.stderr) - print(self._mc_error_msg("Missing '@required' items: {}".format(missing_req)), file=sys.stderr) + if not self._mc_is_default_value_item: + # Any child item not present on self, but found on a corresponding default value item matching self will be assigned to self + contained_in = self + while contained_in: + contained_in, default_item = contained_in._mc_find_closest_default_item(self.named_as()) + if not default_item: + break + + for shared_child_name, shared_child in default_item.items(with_excluded=True): + if isinstance(shared_child, RepeatableDict): + if shared_child_name not in self._mc_deco_nested_repeatables: + # Don't merge repeatables on shared items if they are not declared on self + continue + + repeatable = object.__getattribute__(self, shared_child_name) + for rep_key, rep in shared_child._all_items.items(): + repeatable._all_items.setdefault(rep_key, _mc_item_parent_proxy_factory(self, rep)) + continue + + if shared_child_name not in [named_as for named_as, it in self.items(with_excluded=True)]: + proxy_insert(self, shared_child_name, shared_child) + + if not thread_local.is_under_default_item: + self._mc_validate_required(mc_error_info_up_level + 1 if mc_error_info_up_level is not None else None) if self._mc_num_errors: self._mc_raise_errors() @@ -303,8 +401,7 @@ def _mc_call_mc_validate(self, env): self._mc_where = Where.NOWHERE self.mc_validate() - if self._mc_attributes_to_check: - self._mc_print_no_proper_value_error_msg(None) + self._mc_validate_attributes(None) if self._mc_num_errors: self._mc_raise_errors() self._mc_where = Where.FROZEN @@ -431,6 +528,16 @@ def _mc_setattr_env_value(self, current_env, attr_name, env_attr, value, old_val elif old_value not in (MC_NO_VALUE, MC_TODO, MC_REQUIRED): return + # Try to resolve value from shared items + # Find first occurrence of DefaultItems.name, by searching backwards towards root_conf + if not self._mc_is_default_value_item: + contained_in = self + while contained_in: + contained_in, default_item = contained_in._mc_find_closest_default_item(self.named_as()) + if default_item: + if self._mc_resolve_shared_attribute(default_item, attr_name, mc_error_info_up_level): + return + if self._mc_where == Where.IN_INIT: if value == MC_NO_VALUE: env_attr.set(current_env, value, self._mc_where, from_eg) @@ -445,6 +552,9 @@ def _mc_setattr_env_value(self, current_env, attr_name, env_attr, value, old_val self._mc_attributes_to_check_add(attr_name, mc_error_info_up_level) return + if thread_local.is_under_default_item: + return + # We will report the error now, so remove from check list self._mc_attributes_to_check_del(attr_name) @@ -882,6 +992,10 @@ def _mc_get_repeatable(self, repeatable_class_key, repeatable_cls_or_dict): repeatable_cls_key=repeatable_class_key, repeatable_cls=repeatable_cls_or_dict, ci_named_as=self.named_as(), ci_cls=type(self)) raise ConfigException(msg) + @property + def mc_is_default_value_item(self): + return self._mc_is_default_value_item + class AbstractConfigItem(_ConfigBase): # TODO metaclass=abc.ABCMeta """This may be used as the base of classes which will be basis for both Repeatable and non-repeatable ConfigItem. @@ -994,7 +1108,6 @@ def ref_type_info_for_json(self): return '' - class _ConfigBuilderMixin(): """Method definitions for ConfigBuilder classes @@ -1018,6 +1131,26 @@ def ref_type_info_for_json(self): return ' builder' +class _DefaultItemsMixin(): + """Method definitions for DefaultItems class + + This must have the same methods as the _RealConfigItemMixin class. + We never recurse or do any validation of DefaultItems. + """ + + def _mc_call_mc_validate_recursively(self, env): + pass + + def _mc_call_mc_post_validate_recursively(self): + pass + + def _mc_validate_properties_recursively(self, env): + pass + + def ref_type_info_for_json(self): + return ' default' + + class ConfigItem(AbstractConfigItem, _RealConfigItemMixin): """Base class for config items.""" @@ -1068,6 +1201,7 @@ def __new__(cls, *init_args, **init_kwargs): self._mc_root = contained_in._mc_root self._mc_built_by = built_by self._mc_handled_env_bits = thread_local.env.mask + self._mc_is_default_value_item = contained_in._mc_is_default_value_item for key in cls._mc_deco_nested_repeatables: od = RepeatableDict() @@ -1107,6 +1241,10 @@ def __new__(cls, mc_key=None, *init_args, **init_kwargs): contained_in = contained_in._mc_contained_in repeatable = contained_in._mc_get_repeatable(cls.named_as(), cls) + + if repeatable and isinstance(contained_in, DefaultItems): + raise ConfigException(f"'{cls.__name__}' cannot be repeated under '{DefaultItems.__name__}'. The first (and only) occurance of a '{RepeatableConfigItem.__name__}' instance is used to provide the default attribute values.") + mc_key = init_kwargs.get(cls._mc_key_name) or cls._mc_key_value or mc_key try: @@ -1140,6 +1278,7 @@ def __new__(cls, mc_key=None, *init_args, **init_kwargs): self._mc_root = contained_in._mc_root self._mc_built_by = built_by self._mc_handled_env_bits = thread_local.env.mask + self._mc_is_default_value_item = contained_in._mc_is_default_value_item for key in cls._mc_deco_nested_repeatables: od = RepeatableDict() @@ -1155,7 +1294,19 @@ def named_as(cls): return cls._mc_deco_named_as or (cls.__name__ + 's') -class ConfigBuilder(AbstractConfigItem, _ConfigBuilderMixin, metaclass=abc.ABCMeta): +class _UndeclaredRepeatableMixin(): + def _mc_get_repeatable(self, repeatable_class_key, repeatable_cls_or_dict): + """Create any nested repeatable without previous declaration.""" + repeatable = getattr(self, repeatable_class_key, None) + if repeatable is not None: + return repeatable + + repeatable = RepeatableDict() + object.__setattr__(self, repeatable_class_key, repeatable) + return repeatable + + +class ConfigBuilder(_ConfigBuilderMixin, _UndeclaredRepeatableMixin, AbstractConfigItem, metaclass=abc.ABCMeta): """Base class for 'builder' items which can create (a collection of) other items.""" def __new__(cls, mc_key='default-builder', *init_args, **init_kwargs): @@ -1197,6 +1348,7 @@ def __new__(cls, mc_key='default-builder', *init_args, **init_kwargs): self._mc_root = contained_in._mc_root self._mc_built_by = built_by self._mc_handled_env_bits = thread_local.env.mask + self._mc_is_default_value_item = contained_in._mc_is_default_value_item object.__setattr__(contained_in, private_key, self) @@ -1211,16 +1363,6 @@ def named_as(cls): """Try to generate a unique name""" return 'mc_ConfigBuilder_' + cls.__name__ - def _mc_get_repeatable(self, repeatable_class_key, repeatable_cls_or_dict): - """ConfigBuilder allows any nested repeatable without previous declaration.""" - repeatable = getattr(self, repeatable_class_key, None) - if repeatable is not None: - return repeatable - - repeatable = RepeatableDict() - object.__setattr__(self, repeatable_class_key, repeatable) - return repeatable - def _mc_builder_freeze(self): self._mc_where = Where.IN_MC_BUILD _ConfigBase._mc_last_item = None @@ -1231,32 +1373,18 @@ def _mc_builder_freeze(self): except _McExcludedException: _ConfigBase._mc_in_build = was_in_build - def insert(from_build, from_with_key, from_with): - """Insert items from with statement (single or repeatable) in a single (non repeatable) item from mc_build.""" - if from_build._mc_built_by is not self: - return - - if isinstance(from_with, RepeatableDict): - repeatable = from_build._mc_get_repeatable(from_with_key, from_with) - for wi_key, wi in from_with._all_items.items(): - pp = _mc_item_parent_proxy_factory(from_build, wi) - repeatable._all_items[wi_key] = pp - return - - pp = _mc_item_parent_proxy_factory(from_build, from_with) - object.__setattr__(from_build, from_with_key, pp) - # Now set all items created in the 'with' block of the builder on the items created in the 'mc_build' method - for item_from_with_key, item_from_with in self.items(): + for item_from_with_key, item_from_with in self.items(with_types=(ConfigItem, DefaultItems, RepeatableDict)): for item_from_build_key, item_from_build in self._mc_contained_in.items(): if isinstance(item_from_build, RepeatableDict): for bi_key, bi in item_from_build._all_items.items(): - insert(bi, item_from_with_key, item_from_with) - continue + if bi._mc_built_by is self: + proxy_insert(bi, item_from_with_key, item_from_with) continue - insert(item_from_build, item_from_with_key, item_from_with) + if item_from_build._mc_built_by is self: + proxy_insert(item_from_build, item_from_with_key, item_from_with) self._mc_freeze_previous(mc_error_info_up_level=1) self._mc_where = Where.NOWHERE @@ -1266,6 +1394,123 @@ def mc_build(self): """Override this in derived classes. This is where child ConfigItems are declared""" +class DefaultItems(_UndeclaredRepeatableMixin, _DefaultItemsMixin, AbstractConfigItem): + """Class for grouping config items to supply default attribute values across different ConfigItem instances. + + Items can be placed under a 'DefaultItems' item to provide default values (contained items and attributes) for other non-shared items of the same name. + It is not required to provide valid values for all attributes, neither is it required to satisfy the `required` decorator requirements for nested objects. + Any attribute left as MC_REQUIRED are expected to get a value in each non-shared ConfigItem, and missing `required` child objects must likewise be + satisfied in each non-shared ConfigItem. + RepeatableConfigItems will be merged in from default items to matching regular items iff they have a corresponding `_mc_deco_nested_repeatables`. + + E.g. attributes will be resolved:: + + class myitem(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with myitem() as ii: + ii.efgh = 7 + + with myitem() as ii: + ii.abcd = 6 + ii.ijkl = 8 + + it = config(prod).myitem + + # it.abcd == 6 + # t.efgh == 7 + # t.ijkl == 8 + + + E.g. nested items found under DefaultItems will be made available as if declared under a regular item:: + + class child(ConfigItem): + def __init__(self, xx): + super().__init__() + self.xx = xx + + class myitem(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with myitem(): + child(1) + + myitem() + + sout, serr = capsys.readouterr() + + it = config(pp).myitem + # it.child.xx == 1 + # it.child.contained_in == it + + + DefaultItems are not supported in 'mc_build' or under a ConfigBuilder. + User validate callback methods 'mc_validate' and 'mc_post_validate' will not be called for any default item. + The 'mc_init' method will be called. + The items under the DefaultItems cannot be accessed, they are used solely for resolving values for regular items. + """ + + name = 'mc_DefaultItems' + + def __new__(cls): + # cls._mc_debug_hierarchy('ConfigItem.__new__') + contained_in = cls._mc_hierarchy[-1] + + try: + self = contained_in.__dict__[DefaultItems.name] + if self._mc_handled_env_bits & thread_local.env.mask: + raise ConfigException(f"Repeated: {cls}.") + self._mc_handled_env_bits |= thread_local.env.mask + + self._mc_where = Where.IN_INIT + self._mc_num_errors = 0 + return self + except KeyError: + if contained_in._mc_where == Where.IN_MC_BUILD: + raise ConfigException(f"'{cls.__name__}' are not allowed in 'mc_build'.") + + if contained_in._mc_where == Where.IN_MC_INIT: + raise ConfigException(f"'{cls.__name__}' are not allowed in 'mc_init'.") + + if contained_in._mc_is_default_value_item: + raise ConfigException(f"'{cls.__name__}' cannot be nested.") + + self = super().__new__(cls) + self._mc_where = Where.IN_INIT + self._mc_num_errors = 0 + + self._mc_attributes = {} + self._mc_attributes_to_check = None + self._mc_contained_in = contained_in + self._mc_root = contained_in._mc_root + self._mc_built_by = None + self._mc_handled_env_bits = thread_local.env.mask + self._mc_is_default_value_item = True + + # Insert self in parent + object.__setattr__(contained_in, DefaultItems.name, self) + + return self + + def __enter__(self): + thread_local.is_under_default_item = True + return super().__enter__() + + def __exit__(self, exc_type, value, traceback): + res = super().__exit__(exc_type, value, traceback) + thread_local.is_under_default_item = False + return res + class _ItemParentProxy(): """The purpose of this is to set the current '_mc_contained_in' when accessing an item created by a builder and assigned under multiple parent items""" __slots__ = ('_mc_contained_in', '_mc_proxied_item') @@ -1320,6 +1565,21 @@ def _mc_item_parent_proxy_factory(ci, item): return ItemParentProxy(ci, item) +def proxy_insert(parent, item_key, item): + """Insert proxied item(s) (single or repeatable) into parent. + + This must be used to insert a proxied item under different parents. + """ + + if isinstance(item, RepeatableDict): + repeatable = parent._mc_get_repeatable(item_key, item) + for wi_key, wi in item._all_items.items(): + repeatable._all_items[wi_key] = _mc_item_parent_proxy_factory(parent, wi) + return + + object.__setattr__(parent, item_key, _mc_item_parent_proxy_factory(parent, item)) + + class McConfigRoot(_ConfigBase, _RealConfigItemMixin): """Class of root object allocated by the 'mc_config' decorator. @@ -1343,6 +1603,7 @@ def __init__(self, mc_json_filter=None, mc_json_fallback=None, env_factory=None, self._mc_contained_in = None self._mc_root = self self._mc_handled_env_bits = 0 + self._mc_is_default_value_item = False self._mc_config_result = {} self._mc_config_loaded = False self._mc_in_post_validate = False diff --git a/multiconf/thread_state.py b/multiconf/thread_state.py index e776024..c4a889d 100644 --- a/multiconf/thread_state.py +++ b/multiconf/thread_state.py @@ -9,6 +9,7 @@ class ThreadState(threading.local): def __init__(self): super().__init__() self.env = MC_NO_ENV + self.is_under_default_item = False thread_local = ThreadState() diff --git a/test/default_items_test.py b/test/default_items_test.py new file mode 100644 index 0000000..10a73a8 --- /dev/null +++ b/test/default_items_test.py @@ -0,0 +1,1014 @@ +# Copyright (c) 2012 Lars Hupfeldt Nielsen, Hupfeldt IT +# All rights reserved. This work is under a BSD license, see LICENSE.TXT. + +import sys + +from pytest import raises, xfail + +from multiconf import mc_config, DefaultItems, ConfigItem, RepeatableConfigItem, ConfigBuilder, MC_REQUIRED, ConfigException +from multiconf.decorators import required, named_as, nested_repeatables +from multiconf.envs import EnvFactory + +from .utils.utils import next_line_num, lines_in, start_file_line, total_msg +from .utils.messages import config_error_mc_required_expected, config_error_never_received_value_expected +from .utils.tstclasses import ItemWithAA, RepeatableItemWithAA + + +minor_version = sys.version_info[1] + +ef = EnvFactory() +pp = ef.Env('pp') +prod = ef.Env('prod') + + +def test_multiple_required_attributes_some_shared_resolved_for_configitem(capsys): + """ + Under DefaultItems it is OK not to provide any value for an MC_REQUIRED attribute, elsewhere it is not. + The item which is not shared resolves the remaining attributes without values from the corresponding shared item. + """ + + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with item() as ii: + ii.efgh = 7 + + with item() as ii: + ii.abcd = 6 + ii.ijkl = 8 + + sout, serr = capsys.readouterr() + + cr = config(prod) + it = cr.item + assert it.abcd == 6 + assert it.efgh == 7 + assert it.ijkl == 8 + + assert not sout + assert not serr + + +def test_multiple_required_attributes_some_shared_resolved_for_repeatable_configitem(capsys): + """ + Under DefaultItems it is OK not to provide any value for an MC_REQUIRED attribute, elsewhere it is not. + The items which are not shared resolves the remaining attributes without values from the corresponding shared item. + """ + + @named_as('three_attr_items') + class Item(RepeatableConfigItem): + def __init__(self, mc_key): + super().__init__(mc_key) + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @nested_repeatables('three_attr_items') + class root(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + with Item('default') as ii: + ii.abcd = 1 + ii.efgh = 7 + + with Item('aa') as iaa: + iaa.abcd = 6 + iaa.ijkl = 8 + + with Item('bb') as ibb: + ibb.efgh = 16 + ibb.ijkl = 17 + + sout, serr = capsys.readouterr() + + cr = config(prod).root + + iaa = cr.three_attr_items['aa'] + assert iaa.abcd == 6 + assert iaa.efgh == 7 + assert iaa.ijkl == 8 + + ibb = cr.three_attr_items['bb'] + assert ibb.abcd == 1 + assert ibb.efgh == 16 + assert ibb.ijkl == 17 + + assert not sout + assert not serr + + +def test_repeated_repeatable_configitem_error(capsys): + """Under DefaultItems a RepeatableConfigItem cannot be repeated!""" + + @named_as('three_attr_items') + class Item(RepeatableConfigItem): + pass + + @nested_repeatables('three_attr_items') + class root(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + Item('default') + Item('oops') + + print(str(exinfo.value)) + assert str(exinfo.value) == "'Item' cannot be repeated under 'DefaultItems'. The first (and only) occurance of a 'RepeatableConfigItem' instance is used to provide the default attribute values." + + +def test_nested_default_value_item_error(): + """DefaultItems cannot be nested""" + + @nested_repeatables('three_attr_items') + class root(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + DefaultItems() + + print(str(exinfo.value)) + assert str(exinfo.value) == "'DefaultItems' cannot be nested." + + +def test_repeated_default_value_item_error(): + """DefaultItems cannot be repeated (not declader as nested_repeatables, so this is normal behaviour)""" + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + DefaultItems() + DefaultItems() + + print(str(exinfo.value)) + assert str(exinfo.value) == "Repeated: ." + + +def test_default_items_cannot_be_repeated_even_if_declared_repeatable(capsys): + @nested_repeatables('DefaultItems') + class root(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + DefaultItems() + DefaultItems() + + assert str(exinfo.value) == "Repeated: ." + xfail("TODO: Error when using @nested_repeatables('DefaultItems')") + + +def test_required_attributes_shared_partial_env_assignment_all_resolved_for_configitem(capsys): + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with item() as ii: + ii.setattr('abcd', default=MC_REQUIRED, prod=106) + ii.efgh = 18 + + with item() as it: + it.setattr('abcd', pp=17) + it.ijkl = 19 + + sout, serr = capsys.readouterr() + assert not sout + assert not serr + + it = config(pp).item + assert it.abcd == 17 + assert it.efgh == 18 + assert it.ijkl == 19 + + it = config(prod).item + assert it.abcd == 106 + assert it.efgh == 18 + assert it.ijkl == 19 + + +def test_required_attributes_shared_multi_level_all_resolved_for_configitem(capsys): + class root(ConfigItem): + pass + + class l1(ConfigItem): + pass + + class l2(ConfigItem): + pass + + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + self.mnop = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + with item() as ii: + ii.abcd = 17 + ii.efgh = 1 + ii.ijkl = 2 + ii.mnop = 3 + + with l1(): + with DefaultItems(): + with item() as ii: + ii.efgh = 18 + + with l2(): + with DefaultItems(): + with item() as ii: + ii.ijkl = 19 + + with item() as it: + it.mnop = 20 + + cr = config(pp).root + it = cr.l1.l2.item + assert it.abcd == 17 + assert it.efgh == 18 + assert it.ijkl == 19 + assert it.mnop == 20 + + +def test_multiple_required_attributes_shared_not_assigned_for_configitem(capsys): + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config1(_): + with DefaultItems(): + with item() as ii: + ii.efgh = 7 + item() + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config2(_): + with DefaultItems(): + with item() as ii: + ii.efgh = 7 + + item() + + print(str(exinfo.value)) + assert total_msg(2) in str(exinfo.value) + + +def test_multiple_required_attributes_shared_not_assigned_some_envs_for_configitem(capsys): + errorline = [None, None, None] + + class item(ConfigItem): + def __init__(self): + super().__init__() + errorline[1] = next_line_num() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + errorline[2] = next_line_num() + self.ijkl = MC_REQUIRED + + with raises(ConfigException) as exinfo: + errorline[0] = next_line_num() + (1 if minor_version > 7 else 0) + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with item() as ii: + ii.setattr('abcd', default=MC_REQUIRED, prod=6) + ii.efgh = 7 + + item() + + _sout, serr = capsys.readouterr() + print(str(exinfo.value)) + assert total_msg(2) in str(exinfo.value) + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + config_error_never_received_value_expected.format(env=pp), + start_file_line(__file__, errorline[1]), + config_error_mc_required_expected.format(attr='abcd', env=pp), + start_file_line(__file__, errorline[2]), + config_error_mc_required_expected.format(attr='ijkl', env=pp), + ) + + +def test_shared_items_does_not_provide_matching_item_for_configitem_missing_attributes(capsys): + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + class item2(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with item2() as ii: + ii.abcd = 13 + + item() + + print(str(exinfo.value)) + assert total_msg(3) in str(exinfo.value) + + +def test_required_nested_items_shared_not_provided_for_configitem(capsys): + """Under DefaultItems it is OK not to provide any value for an 'required' nested item, elsewhere it is not.""" + + errorline = [None] + + @required('child') + class item(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config1(_): + with DefaultItems(): + item() + + with raises(ConfigException) as exinfo: + errorline[0] = next_line_num() + (1 if minor_version > 7 else 0) + @mc_config(ef, load_now=True) + def config2(_): + with DefaultItems(): + item() + + item() + + print(str(exinfo.value)) + assert total_msg(1) in str(exinfo.value) + + _sout, serr = capsys.readouterr() + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + "Missing '@required' items: ['child']", + ) + + +def test_required_nested_items_resolved_by_matching_default_item(capsys): + """If a @required item is missing from regular config, but found under a corresponding default value item, then that satisfies the requirement.""" + + class child(ConfigItem): + def __init__(self, xx): + super().__init__() + self.xx = xx + + @required('child') + class item(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + child(111) # This is ignored + with item(): + child(1) + + item() + + sout, serr = capsys.readouterr() + assert not sout + assert not serr + + it = config(pp).item + assert it.child + assert it.child.xx == 1 + assert it.child.contained_in == it + + +def test_required_nested_items_resolved_by_provided_required_default_item(capsys): + """If a @required item is missing from regular config, but found under a DefaultItem, then that satisfies the requirement.""" + + class child(ConfigItem): + def __init__(self, xx): + super().__init__() + self.xx = xx + + @required('child') + class item(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + child(111) + + item() + + sout, serr = capsys.readouterr() + assert not sout + assert not serr + + it = config(pp).item + assert it.child + assert it.child.xx == 111 + assert it.child.contained_in == it + + +def test_required_nested_items_of_repeatable_item_resolved_by_default_item(capsys): + """If a @required item of a repeatable is missing from regular config, but found under a corresponding default value item, then that satisfies the requirement.""" + + @nested_repeatables('Items') + class root(ConfigItem): + pass + + class child(ConfigItem): + def __init__(self, xx): + super().__init__() + self.xx = xx + + @required('child') + class Item(RepeatableConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + child(111) # This is ignored + with Item(): + child(1) + + Item('aa') + with Item('bb'): + child(40) + + Item('cc') + + sout, serr = capsys.readouterr() + assert not sout + assert not serr + + its = config(pp).root.Items + + aa = its['aa'] + assert aa.child.xx == 1 + assert aa.child.contained_in == aa + + bb = its['bb'] + assert bb.child.xx == 40 + assert bb.child.contained_in == bb + + cc = its['cc'] + assert cc.child.xx == 1 + assert cc.child.contained_in == cc + + +def test_nested_items_shared_by_default_item(capsys): + """If a default item is found which has nested children not existing on item, then those children will be available from item.""" + + class child(ConfigItem): + def __init__(self, xx): + super().__init__() + self.xx = xx + + class item(ConfigItem): + pass + + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + child(111) # This is ignored + with item(): + child(1) + + item() + + sout, serr = capsys.readouterr() + + it = config(pp).item + assert it.child + assert it.child.xx == 1 + assert it.child.contained_in == it + + assert not sout + assert not serr + + +def test_repeatable_nested_items_shared_by_default_item(capsys): + """If a default item is found which has nested repeatable children not existing on item, then those children will be available from item, iff item declared it as 'nested_repeatables'.""" + + @nested_repeatables('RepeatableItems') + class root(ItemWithAA): + pass + + @named_as('children') + class child(RepeatableItemWithAA): + pass + + @nested_repeatables('children') + class Item(RepeatableItemWithAA): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(aa='root'): + with DefaultItems(): + child('a', 111) # This is ignored + with Item(None, aa='default_item'): + child('b1', 1) + child('b2', 2) + + Item('aa', 'aa') + + with Item('bb', 'bb'): + child('c', 40) + + with Item('cc', 'cc'): + child('b1', 17) + + sout, serr = capsys.readouterr() + + its = config(pp).root.RepeatableItems + + aa = its['aa'] + ach = aa.children + assert ach['b1'].aa == 1 + assert ach['b1'].contained_in == aa + assert ach['b2'].aa == 2 + assert ach['b2'].contained_in == aa + + bb = its['bb'] + bch = bb.children + assert bch['c'].aa == 40 + assert bch['c'].contained_in == bb + assert bch['b1'].aa == 1 + assert bch['b1'].contained_in == bb + assert bch['b2'].aa == 2 + assert bch['b2'].contained_in == bb + assert list(it.aa for it in bch.values()) == [40, 1, 2] + + cc = its['cc'] + cch = cc.children + assert cch['b1'].aa == 17 + assert cch['b1'].contained_in == cc + assert cch['b2'].aa == 2 + assert cch['b2'].contained_in == cc + + assert not sout + assert not serr + + +def test_repeatable_nested_items_not_declare_as_nested_repeatables_shared_by_default_item(capsys): + """If a default item is found which has nested repeatable children not existing on item, then those children will be available from item, iff item declared it as 'nested_repeatables'.""" + + @nested_repeatables('RepeatableItems') + class root(ItemWithAA): + pass + + @named_as('children') + class child(RepeatableItemWithAA): + pass + + @nested_repeatables('children') + class ItemX(RepeatableItemWithAA): + pass + + class Item(RepeatableItemWithAA): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(aa='root'): + with DefaultItems(): + with ItemX(None, aa='default_item'): + child('b1', 1) + + Item('aa', 'aa') + + sout, serr = capsys.readouterr() + + its = config(pp).root.RepeatableItems + + aa = its['aa'] + assert not hasattr(aa, 'children') + + assert not sout + assert not serr + + +def test_repeatable_nested_items_but_non_repeatable_shared_by_default_item(capsys): + """Repeatable non-Repeatable mixup.""" + + @nested_repeatables('RepeatableItems') + class root(ItemWithAA): + pass + + @named_as('children') + class child(RepeatableItemWithAA): + pass + + @named_as('children') + class NonRepChild(ItemWithAA): + pass + + @nested_repeatables('children') + class Item(RepeatableItemWithAA): + pass + + class Item2(RepeatableItemWithAA): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(aa='root'): + with DefaultItems(): + child('a', 111) # This is ignored + with Item2(None, aa='default_item'): + NonRepChild(1) + + Item('aa', 'aa') + + with Item('bb', 'bb'): + child('c', 40) + + with Item('cc', 'cc'): + child('b1', 17) + + sout, serr = capsys.readouterr() + + its = config(pp).root.RepeatableItems + + aa = its['aa'] + ach = aa.children + assert not ach + + bb = its['bb'] + bch = bb.children + assert bch['c'].aa == 40 + assert bch['c'].contained_in == bb + assert list(it.aa for it in bch.values()) == [40] + + cc = its['cc'] + cch = cc.children + assert cch['b1'].aa == 17 + assert cch['b1'].contained_in == cc + + assert not sout + assert not serr + + +def test_non_repeatable_nested_items_but_repeatable_shared_by_default_item(capsys): + """Repeatable non-Repeatable mixup inverse.""" + + @nested_repeatables('RepeatableItems') + class root(ItemWithAA): + pass + + @named_as('children') + class child(RepeatableItemWithAA): + pass + + @named_as('children') + class NonRepChild(ItemWithAA): + pass + + @nested_repeatables('children') + class Item(RepeatableItemWithAA): + pass + + class Item2(RepeatableItemWithAA): + pass + + @required('children') + class Item3(RepeatableItemWithAA): + pass + + @mc_config(ef, load_now=True) + def config(_): + with root(aa='root'): + with DefaultItems(): + child('a', 111) # This is ignored + with Item(None, aa='default_item'): + child('b1', 1) + child('b2', 2) + + Item2('aa') + + with Item2('bb'): + NonRepChild(40) + + with Item2('cc'): + NonRepChild(17) + + with Item3('33'): + NonRepChild(47) + + sout, serr = capsys.readouterr() + + its = config(pp).root.RepeatableItems + + aa = its['aa'] + assert not hasattr(aa, 'children') + + bb = its['bb'] + bch = bb.children + assert bch.aa == 40 + assert bch.contained_in == bb + + cc = its['cc'] + cch = cc.children + assert cch.aa == 17 + assert cch.contained_in == cc + + x33 = its['33'] + x33ch = x33.children + assert x33ch.aa == 47 + assert x33ch.contained_in == x33 + + assert not sout + assert not serr + + +def test_required_will_not_resolve_to_default_repeatable(capsys): + """Repeatable non-Repeatable mixup inverse.""" + + errorline = [None] + + @nested_repeatables('RepeatableItems') + class root(ItemWithAA): + pass + + @named_as('children') + class child(RepeatableItemWithAA): + pass + + @nested_repeatables('children') + class Item(RepeatableItemWithAA): + pass + + @required('children') + class Item3(RepeatableItemWithAA): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + with root(aa='root'): + with DefaultItems(): + with Item(None, aa='default_item'): + child('b1', 1) + + errorline[0] = next_line_num() + Item3('33') + + print(str(exinfo.value)) + assert total_msg(1) in str(exinfo.value) + + _sout, serr = capsys.readouterr() + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + "Missing '@required' items: ['children']", + ) + + +def test_required_shared_items_attribute_inherited_env_values_missing(capsys): + """Under DefaultItems setattr must stil provide a value for all defined envs""" + + errorline = [None, None, None, None] + + class root(ConfigItem): + def __init__(self): + super().__init__() + errorline[1] = next_line_num() + self.anattr = MC_REQUIRED + self.anotherattr = MC_REQUIRED + + class root2(root): + def __init__(self): + super().__init__() + errorline[2] = next_line_num() + self.someattr2 = MC_REQUIRED + errorline[3] = next_line_num() + self.someotherattr2 = MC_REQUIRED + + @mc_config(ef) + def config(_): + with DefaultItems(): + with root2() as cr: + cr.setattr('anattr', prod=1) + cr.setattr('someattr2', prod=3) + cr.setattr('someotherattr2', pp=4) + + root2() + + with raises(ConfigException): + errorline[0] = next_line_num() + config.load(error_next_env=True) + + _sout, serr = capsys.readouterr() + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + config_error_never_received_value_expected.format(env=pp), + start_file_line(__file__, errorline[1]), + config_error_mc_required_expected.format(attr='anattr', env=pp), + start_file_line(__file__, errorline[2]), + config_error_mc_required_expected.format(attr='someattr2', env=pp), + start_file_line(__file__, errorline[3]), + config_error_mc_required_expected.format(attr='someotherattr2', env=prod), + # 'anotherattr' will not be verified because it might get a value in mc_init + # which is not called when there are errors in 'with' block + ) + + +def test_shared_under_builder(capsys): + class x(ConfigItem): + pass + + class root(ConfigBuilder): + def mc_build(self): + x() + + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + with item() as its: + its.efgh = 7 + + with item() as it: + it.abcd = 1 + it.ijkl = 2 + + it = config(pp).x.item + assert it.abcd == 1 + assert it.efgh == 7 + assert it.ijkl == 2 + + +def test_shared_not_allowed_in_mc_init(capsys): + class root(ConfigItem): + def mc_init(self): + DefaultItems() + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + root() + + print(str(exinfo.value)) + assert "'DefaultItems' are not allowed in 'mc_init'." in str(exinfo.value) + + +def test_shared_not_allowed_in_mc_build(capsys): + class root(ConfigBuilder): + def mc_build(self): + with DefaultItems(): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + root() + + print(str(exinfo.value)) + assert "'DefaultItems' are not allowed in 'mc_build'." in str(exinfo.value) + + +def test_shared_item_is_not_subtype(capsys): + """DefaultItems lookup is based solely on the named_as property.""" + errorline = [None, None] + + class item(ConfigItem): + def __init__(self): + super().__init__() + errorline[1] = next_line_num() + self.abcd = MC_REQUIRED + + @named_as('item') + class item2(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + errorline[0] = next_line_num() + (1 if minor_version > 7 else 0) + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + item2() + + item() + + _sout, serr = capsys.readouterr() + + print(str(exinfo.value)) + assert total_msg(1) in str(exinfo.value) + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + config_error_never_received_value_expected.format(env=pp), + start_file_line(__file__, errorline[1]), + config_error_mc_required_expected.format(attr='abcd', env=pp), + ) + + +def test_inherit_from_default_item_with_attribute_does_not_resolve_required(capsys): + """TODO: Not sure about the use for inheriting from DefaultItems, but currently needed to get coverage.""" + errorline = [None] + + class DefaultItemsX(DefaultItems): + abcd = 1 + + @required('abcd') + class item(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + errorline[0] = next_line_num() + (1 if minor_version > 7 else 0) + @mc_config(ef, load_now=True) + def config(_): + DefaultItemsX() + item() + + _sout, serr = capsys.readouterr() + print(serr) + print(str(exinfo.value)) + assert total_msg(1) in str(exinfo.value) + assert lines_in( + serr, + start_file_line(__file__, errorline[0]), + "Missing '@required' items: ['abcd']", + ) + + +def test_nested_items_shared_by_default_item_not_valid(capsys): + """If a default item is found which has nested children not existing on item, then those children must be valid.""" + errorline = [None] + + class child(ConfigItem): + def __init__(self, xx=MC_REQUIRED): + super().__init__() + errorline[0] = next_line_num() + self.xx = xx + + class item(ConfigItem): + pass + + with raises(ConfigException) as exinfo: + @mc_config(ef, load_now=True) + def config(_): + with DefaultItems(): + with item(): + with child() as ch: + ch.setattr('xx', pp=1) + + item() + + sout, serr = capsys.readouterr() + assert not sout + assert lines_in( + serr, + config_error_never_received_value_expected.format(env=prod), + start_file_line(__file__, errorline[0]), + config_error_mc_required_expected.format(attr='xx', env=prod), + ) + + xfail('TODO: ref item, child and message abount default ref') diff --git a/test/json_output_complex_item_references_test.py b/test/json_output_complex_item_references_test.py index 5264566..982f6a4 100644 --- a/test/json_output_complex_item_references_test.py +++ b/test/json_output_complex_item_references_test.py @@ -1,6 +1,8 @@ # Copyright (c) 2012 Lars Hupfeldt Nielsen, Hupfeldt IT # All rights reserved. This work is under a BSD license, see LICENSE.TXT. +import pytest + from multiconf import mc_config, ConfigItem from multiconf.envs import EnvFactory @@ -313,7 +315,7 @@ def config(root): "__class__": "Env", "name": "prod" }, - "aa": "#outside-ref: Excluded: , name: Tootsi" + "aa": "#ref outside: Excluded: , name: Tootsi" }""" def test_json_dump_ref_outside_exluded_item1(): @@ -339,7 +341,7 @@ def config(_): "__class__": "Env", "name": "prod" }, - "aa": "#outside-ref: Excluded: " + "aa": "#ref outside: Excluded: " }""" def test_json_dump_ref_outside_exluded_item_partially_set_name_attribute_mc_required(): @@ -366,7 +368,7 @@ def config(_): "__class__": "Env", "name": "prod" }, - "aa": "#outside-ref: Excluded: " + "aa": "#ref outside: Excluded: " }""" def test_json_dump_ref_outside_exluded_item_partially_set_name_attribute_non_existing(): @@ -384,3 +386,75 @@ def config(_): cr = config(prod) ref1 = cr.Ref1 assert compare_json(ref1, _json_dump_ref_outside_exluded_item_partially_set_name_attribute_non_existing_expected_json) + + +_json_dump_pprd_ref_same_env_expected_json = """{ + "__class__": "ItemWithAA", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "pprd" + }, + "aa": [ + { + "__class__": "Env", + "name": "pprd" + } + ], + "ItemWithAA": { + "__class__": "ItemWithAA", + "__id__": 0000, + "aa": [ + { + "__class__": "Env", + "name": "pprd" + } + ] + } +}""" + + +def test_json_dump_ref_same_env(): + @mc_config(ef, load_now=True) + def config(_): + with ItemWithAA(aa=[pprd]): + ItemWithAA(aa=[pprd]) + + cr = config(pprd) + ref1 = cr.ItemWithAA + assert compare_json(ref1, _json_dump_pprd_ref_same_env_expected_json) + pytest.xfail("expected #ref") + + +_json_dump_prod_ref_other_env_expected_json = """{ + "__class__": "ItemWithAA", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "prod" + }, + "aa": [ + { + "__class__": "Env", + "name": "pprd" + } + ], + "ItemWithAA": { + "__class__": "ItemWithAA", + "__id__": 0000, + "aa": [ + "#ref , name: 'pprd'" + ] + } +}""" + +def test_json_dump_ref_other_env(): + @mc_config(ef, load_now=True) + def config(_): + with ItemWithAA(aa=[pprd]): + ItemWithAA(aa=[pprd]) + + cr = config(prod) + ref1 = cr.ItemWithAA + assert compare_json(ref1, _json_dump_prod_ref_other_env_expected_json) + diff --git a/test/json_output_default_value_items_test.py b/test/json_output_default_value_items_test.py new file mode 100644 index 0000000..42faa82 --- /dev/null +++ b/test/json_output_default_value_items_test.py @@ -0,0 +1,192 @@ +# Copyright (c) 2012 Lars Hupfeldt Nielsen, Hupfeldt IT +# All rights reserved. This work is under a BSD license, see LICENSE.TXT. + +from multiconf import mc_config, MC_REQUIRED, ConfigItem, RepeatableConfigItem, ConfigBuilder, DefaultItems +from multiconf.envs import EnvFactory + +from .utils.compare_json import compare_json + + +ef = EnvFactory() +pp = ef.Env('pp') +prod = ef.Env('prod') + +ef2_prod = EnvFactory() +prod2 = ef2_prod.Env('prod') + + +_json_dump_with_shared_and_builder_expected_json_full = """{ + "__class__": "McConfigRoot", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "pp" + }, + "mc_ConfigBuilder_root default-builder": { + "__class__": "root", + "__id__": 0000, + "mc_DefaultItems": { + "__class__": "DefaultItems", + "__id__": 0000, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": "MC_REQUIRED", + "efgh": 7, + "ijkl": "MC_REQUIRED", + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true + }, + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true, + "name": "mc_DefaultItems", + "name #static": true + }, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": 1, + "efgh": 7, + "ijkl": 2 + } + }, + "x": { + "__class__": "x", + "__id__": 0000, + "mc_DefaultItems": "#ref default, id: 0000", + "item": "#ref, id: 0000" + } +}""" + + +_json_dump_with_shared_and_builder_expected_json_full = """{ + "__class__": "McConfigRoot", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "pp" + }, + "mc_ConfigBuilder_root default-builder": { + "__class__": "root", + "__id__": 0000, + "mc_DefaultItems": { + "__class__": "DefaultItems", + "__id__": 0000, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": "MC_REQUIRED", + "efgh": 7, + "ijkl": "MC_REQUIRED", + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true + }, + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true, + "name": "mc_DefaultItems", + "name #static": true + }, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": 1, + "efgh": 7, + "ijkl": 2 + } + }, + "x": { + "__class__": "x", + "__id__": 0000, + "mc_DefaultItems": "#ref default, id: 0000", + "item": "#ref, id: 0000" + } +}""" + +_json_dump_with_shared_expected_json_full = """{ + "__class__": "McConfigRoot", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "pp" + }, + "x": { + "__class__": "x", + "__id__": 0000, + "mc_DefaultItems": { + "__class__": "DefaultItems", + "__id__": 0000, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": "MC_REQUIRED", + "efgh": 7, + "ijkl": "MC_REQUIRED", + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true + }, + "mc_is_default_value_item": true, + "mc_is_default_value_item #calculated": true, + "name": "mc_DefaultItems", + "name #static": true + }, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": 1, + "efgh": 7, + "ijkl": 2 + } + } +}""" + +_json_dump_expected_json_full = """{ + "__class__": "McConfigRoot", + "__id__": 0000, + "env": { + "__class__": "Env", + "name": "pp" + }, + "x": { + "__class__": "x", + "__id__": 0000, + "item": { + "__class__": "item", + "__id__": 0000, + "abcd": 1, + "efgh": 7, + "ijkl": 2 + } + } +}""" + +def test_shared_json_dump(capsys): + class x(ConfigItem): + pass + + class root(ConfigBuilder): + def mc_build(self): + x() + + class item(ConfigItem): + def __init__(self): + super().__init__() + self.abcd = MC_REQUIRED + self.efgh = MC_REQUIRED + self.ijkl = MC_REQUIRED + + @mc_config(ef, load_now=True) + def config(_): + with root(): + with DefaultItems(): + with item() as its: + its.efgh = 7 + + with item() as it: + it.abcd = 1 + it.ijkl = 2 + + cr = config(pp) + + assert compare_json(cr, _json_dump_with_shared_and_builder_expected_json_full) + assert compare_json(cr, _json_dump_with_shared_expected_json_full, dump_builders=False) + assert compare_json(cr, _json_dump_expected_json_full, dump_builders=False, dump_default_items=False) diff --git a/test/json_output_test.py b/test/json_output_test.py index 19307bb..7699d4d 100644 --- a/test/json_output_test.py +++ b/test/json_output_test.py @@ -1002,7 +1002,7 @@ def config(rt): "__id__": 0000, "d": 3, "id": "n3", - "uplevel_ref": "#outside-ref: , id: 'n1', name: 'Number 1'", + "uplevel_ref": "#ref outside: , id: 'n1', name: 'Number 1'", "someitems": {} } } diff --git a/test/utils/compare_json.py b/test/utils/compare_json.py index 3ec9880..05c05ea 100644 --- a/test/utils/compare_json.py +++ b/test/utils/compare_json.py @@ -16,12 +16,12 @@ def decode(_string): def _compare_json( item, expected_json, replace_builders, dump_builders, sort_attributes, test_decode, test_containment, test_excluded, test_compact, property_methods, - expect_num_errors, warn_nesting, show_all_envs, depth, replace_ids, replace_address): + expect_num_errors, warn_nesting, show_all_envs, depth, replace_ids, replace_address, dump_default_items): try: compact_json = item.json( - compact=True, property_methods=property_methods, builders=dump_builders, sort_attributes=sort_attributes, warn_nesting=warn_nesting, + compact=True, property_methods=property_methods, builders=dump_builders, default_items=dump_default_items, sort_attributes=sort_attributes, warn_nesting=warn_nesting, show_all_envs=show_all_envs, depth=depth, persistent_ids=not replace_ids) - full_json = item.json(property_methods=property_methods, builders=dump_builders, sort_attributes=sort_attributes, warn_nesting=warn_nesting, + full_json = item.json(property_methods=property_methods, builders=dump_builders, default_items=dump_default_items, sort_attributes=sort_attributes, warn_nesting=warn_nesting, show_all_envs=show_all_envs, depth=depth, persistent_ids=not replace_ids) if replace_ids or replace_address: @@ -100,14 +100,15 @@ def show(msg, replaced_ids, json): def compare_json(item, expected_json, replace_builders=False, dump_builders=True, sort_attributes=True, test_decode=False, test_containment=True, test_excluded=False, test_compact=True, property_methods=True, expect_num_errors=0, warn_nesting=False, expected_all_envs_json=None, expect_all_envs_num_errors=None, depth=None, - replace_ids=True, replace_address=False): + replace_ids=True, replace_address=False, *, dump_default_items=True): + assert expected_json or expected_all_envs_json res = True if expected_json: res = _compare_json( item, expected_json, replace_builders, dump_builders, sort_attributes, test_decode, test_containment, test_excluded, test_compact, property_methods, expect_num_errors, warn_nesting, show_all_envs=False, depth=depth, - replace_ids=replace_ids, replace_address=replace_address) + replace_ids=replace_ids, replace_address=replace_address, dump_default_items=dump_default_items) res2 = True if expected_all_envs_json: @@ -116,6 +117,6 @@ def compare_json(item, expected_json, replace_builders=False, dump_builders=True item, expected_all_envs_json, replace_builders, dump_builders, sort_attributes, test_decode, test_containment=False, test_excluded=test_excluded, test_compact=False, property_methods=property_methods, expect_num_errors=expect_num_errors, warn_nesting=warn_nesting, show_all_envs=True, depth=depth, - replace_ids=replace_ids, replace_address=replace_address) + replace_ids=replace_ids, replace_address=replace_address, dump_default_items=dump_default_items) return res and res2