diff --git a/.pylint.ini b/.pylint.ini index 1bbf12f..5451dda 100644 --- a/.pylint.ini +++ b/.pylint.ini @@ -65,7 +65,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,import-star-module-level,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,long-suffix,old-ne-operator,old-octal-literal,suppressed-message,useless-suppression,locally-disabled,fixme,too-few-public-methods,too-many-public-methods,invalid-name,global-statement,too-many-ancestors,missing-docstring,no-else-return,too-many-instance-attributes,too-many-branches,arguments-differ +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,import-star-module-level,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,long-suffix,old-ne-operator,old-octal-literal,suppressed-message,useless-suppression,locally-disabled,fixme,too-few-public-methods,too-many-public-methods,invalid-name,global-statement,too-many-ancestors,missing-docstring,no-else-return,too-many-instance-attributes,too-many-branches,arguments-differ,unnecessary-lambda,no-member [REPORTS] diff --git a/setup.py b/setup.py index 7551b84..3d4fb82 100644 --- a/setup.py +++ b/setup.py @@ -79,5 +79,6 @@ def build_description(): install_requires=[ 'PyYAML ~= 3.11', 'simplejson ~= 3.8', + 'parse ~= 1.8.0', ], ) diff --git a/yorm/common.py b/yorm/common.py index d1521a9..c6b9cd0 100644 --- a/yorm/common.py +++ b/yorm/common.py @@ -23,6 +23,7 @@ verbosity = 0 # global verbosity setting for controlling string formatting attrs = collections.defaultdict(collections.OrderedDict) +path_formats = {} # LOGGING ###################################################################### diff --git a/yorm/decorators.py b/yorm/decorators.py index 475433a..b5adb3d 100644 --- a/yorm/decorators.py +++ b/yorm/decorators.py @@ -80,6 +80,7 @@ def sync_instances(path_format, format_spec=None, attrs=None, **kwargs): def decorator(cls): """Class decorator to map instances to files.""" + common.path_formats[cls] = path_format init = cls.__init__ def modified_init(self, *_args, **_kwargs): diff --git a/yorm/tests/test_utilities.py b/yorm/tests/test_utilities.py index 381e2ba..4f25082 100644 --- a/yorm/tests/test_utilities.py +++ b/yorm/tests/test_utilities.py @@ -20,9 +20,12 @@ def model_class(tmpdir): @yorm.sync("data/{self.kind}/{self.key}.yml", auto_create=False) class Model: - def __init__(self, kind, key): + def __init__(self, kind, key, **kwargs): self.kind = kind self.key = key + assert 0 <= len(kwargs) < 2 + if kwargs: + assert kwargs == {'test': 'test'} def __eq__(self, other): return (self.kind, self.key) == (other.kind, other.key) @@ -35,6 +38,20 @@ def instance(model_class): return model_class('foo', 'bar') +@pytest.fixture +def instance_pile(model_class): + instances = [ + model_class(kind, key) + for kind in ('spam', 'egg') + for key in ('foo', 'bar') + ] + # mostly because this is used primarily by match tests + for inst in instances: + inst.__mapper__.create() + + return instances + + def describe_create(): def it_creates_files(model_class): @@ -96,9 +113,100 @@ def it_requires_a_mapped_class_or_instance(): def describe_match(): - def it_is_not_yet_implemented(): - with expect.raises(NotImplementedError): - utilities.match(Mock) + def class_factory(model_class, instance_pile): + matches = list( + utilities.match( + model_class, + (lambda key, kind: model_class(kind, key, test="test")), + kind='spam', + key='foo', + ) + ) + assert len(matches) == 1 + instance = matches[0] + assert instance.kind == 'spam' + assert instance.key == 'foo' + assert instance in instance_pile + + def class_no_factory(model_class, instance_pile): + matches = list( + utilities.match( + model_class, + kind='spam', + key='foo', + ) + ) + assert len(matches) == 1 + instance = matches[0] + assert instance.kind == 'spam' + assert instance.key == 'foo' + assert instance in instance_pile + + def string_self_factory(model_class, instance_pile): + matches = list( + utilities.match( + "data/{self.kind}/{self.key}.yml", + (lambda key, kind: model_class(kind, key)), + kind='spam', + key='bar', + ) + ) + assert len(matches) == 1 + instance = matches[0] + assert instance.kind == 'spam' + assert instance.key == 'bar' + assert instance in instance_pile + + def string_factory(model_class, instance_pile): + matches = list( + utilities.match( + "data/{kind}/{key}.yml", + (lambda key, kind: model_class(kind, key)), + kind='egg', + key='foo', + ) + ) + assert len(matches) == 1 + instance = matches[0] + assert instance.kind == 'egg' + assert instance.key == 'foo' + assert instance in instance_pile + + def class_factory_wildcard(model_class, instance_pile): + matches = list( + utilities.match( + model_class, + (lambda key, kind: model_class(kind, key)), + kind='spam', + ) + ) + assert len(matches) == 2 + assert all(i.kind == 'spam' for i in matches) + assert all(i in instance_pile for i in matches) + + def string_self_factory_wildcard(model_class, instance_pile): + matches = list( + utilities.match( + "data/{self.kind}/{self.key}.yml", + (lambda key, kind: model_class(kind, key)), + kind='egg', + ) + ) + assert len(matches) == 2 + assert all(i.kind == 'egg' for i in matches) + assert all(i in instance_pile for i in matches) + + def string_factory_wildcard(model_class, instance_pile): + matches = list( + utilities.match( + "data/{kind}/{key}.yml", + (lambda key, kind: model_class(kind, key)), + key='foo', + ) + ) + assert len(matches) == 2 + assert all(i.key == 'foo' for i in matches) + assert all(i in instance_pile for i in matches) def describe_load(): diff --git a/yorm/utilities.py b/yorm/utilities.py index 8b38fdd..e1eeb9a 100644 --- a/yorm/utilities.py +++ b/yorm/utilities.py @@ -2,6 +2,11 @@ import inspect import logging +import string +import glob +import types + +import parse from . import common, exceptions @@ -37,10 +42,81 @@ def find(class_or_instance, *args, create=False, **kwargs): # pylint: disable=r return None -def match(cls, **kwargs): - """Yield all matching mapped objects.""" - log.debug((cls, kwargs)) - raise NotImplementedError +class GlobFormatter(string.Formatter): + """Uses '*' for all unknown fields.""" + + WILDCARD = object() + + def get_field(self, field_name, args, kwargs): + try: + return super().get_field(field_name, args, kwargs) + except (KeyError, IndexError, AttributeError): + return self.WILDCARD, None + + def get_value(self, key, args, kwargs): + try: + return super().get_value(key, args, kwargs) + except (KeyError, IndexError, AttributeError): + return self.WILDCARD + + def convert_field(self, value, conversion): + if value is self.WILDCARD: + return self.WILDCARD + else: + return super().convert_field(value, conversion) + + def format_field(self, value, format_spec): + if value is self.WILDCARD: + return '*' + else: + return super().format_field(value, format_spec) + + +def _unpack_parsed_fields(pathfields): + return { + (k[len('self.'):] if k.startswith('self.') else k): v + for k, v in pathfields.items() + } + + +def match(cls_or_path, _factory=None, **kwargs): + """Yield all matching mapped objects. + + Can be used two ways: + * With a YORM-decorated class, optionally with a factory callable + * With a Python 3-style string template and a factory callable + + The factory callable must accept keyuword arguments, extracted from the file + name merged with those passed to match(). If no factory is given, the class + itself is used as the factory (same signature). + + Keyword arguments are used to filter objects. Filtering is only done by + filename, so only fields that are part of the path_format can be filtered + against. + """ + if isinstance(cls_or_path, type): + path_format = common.path_formats[cls_or_path] + # Let KeyError fail through + if _factory is None: + _factory = cls_or_path + else: + path_format = cls_or_path + if _factory is None: + raise TypeError("Factory must be given if a path format is given") + + gf = GlobFormatter() + mock = types.SimpleNamespace(**kwargs) + + kwargs['self'] = mock + posix_pattern = gf.vformat(path_format, (), kwargs.copy()) + del kwargs['self'] + py_pattern = parse.compile(path_format) + + for filename in glob.iglob(posix_pattern): + pathfields = py_pattern.parse(filename).named + fields = _unpack_parsed_fields(pathfields) + fields.update(kwargs) + yield _factory(**fields) def load(instance):