From fe4ca259db6e7a31c0989a81a0d867be901cf6b3 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 25 Nov 2014 17:52:11 +0200 Subject: [PATCH 01/11] adding type check for ListField --- elasticgit/models.py | 24 ++++++++++++++++++++++++ elasticgit/tests/test_models.py | 13 ++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/elasticgit/models.py b/elasticgit/models.py index 92c05bd..e37d3ab 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -104,6 +104,18 @@ def clean(self, value): return bool(value) +class TypeCheck(object): + + def __init__(self, type_): + self.type_ = type_ + + def get_type(self): + return self.type_ + + def __call__(self, value): + return isinstance(value, self.type_) + + class ListField(ModelField): """ A list field @@ -115,11 +127,23 @@ class ListField(ModelField): 'type': 'string', } + def __init__(self, doc, type_check=None, default=[], static=False, + fallbacks=(), mapping={}): + super(ListField, self).__init__( + doc, default=default, static=static, fallbacks=fallbacks, + mapping=mapping) + self.type_check = type_check + def clean(self, value): if isinstance(value, tuple): value = list(value) if not isinstance(value, list): self.raise_config_error("is not a list.") + if self.type_check is not None: + if not all([self.type_check(v) for v in value]): + self.raise_config_error( + 'Type check %r failed for some values.' % ( + self.type_check,)) return deepcopy(value) diff --git a/elasticgit/tests/test_models.py b/elasticgit/tests/test_models.py index 7834a90..9e76b6f 100644 --- a/elasticgit/tests/test_models.py +++ b/elasticgit/tests/test_models.py @@ -1,6 +1,7 @@ from elasticgit.tests.base import ModelBaseTest from elasticgit.models import ( - ConfigError, IntegerField, TextField, ModelVersionField) + ConfigError, IntegerField, TextField, ModelVersionField, + ListField, TypeCheck) import elasticgit @@ -86,3 +87,13 @@ def test_version_check(self): self.assertTrue(field.compatible_version('0.2.10', '0.2.9')) self.assertTrue(field.compatible_version('0.2.10', '0.2.10')) self.assertFalse(field.compatible_version('0.2.9', '0.2.10')) + + def test_list_field(self): + model_class = self.mk_model({ + 'tags': ListField('list field', type_check=TypeCheck(int)) + }) + self.assertRaises(ConfigError, model_class, {'tags': ['1']}) + self.assertRaises(ConfigError, model_class, {'tags': [2.0]}) + self.assertRaises(ConfigError, model_class, {'tags': [None]}) + self.assertTrue(model_class({'tags': []})) + self.assertTrue(model_class({'tags': [1, 2, 3]})) From 2c1a9f27944dab958d3a091dc79986b71fa4c3a9 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Fri, 28 Nov 2014 21:01:18 +0200 Subject: [PATCH 02/11] work in progress --- elasticgit/commands/avro.py | 6 +++++- elasticgit/commands/tests/test_avro.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index 886be5b..696790e 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -306,7 +306,11 @@ class SchemaDumper(ToolCommand): TextField: 'string', FloatField: 'float', BooleanField: 'boolean', - ListField: 'array', + ListField: { + 'type': 'array', + 'name': 'list', + 'items': 'string', + }, DictField: 'record', UUIDField: 'string', ModelVersionField: { diff --git a/elasticgit/commands/tests/test_avro.py b/elasticgit/commands/tests/test_avro.py index 604ebf3..2731b7e 100644 --- a/elasticgit/commands/tests/test_avro.py +++ b/elasticgit/commands/tests/test_avro.py @@ -47,6 +47,15 @@ class TestModel(models.Model): age = self.get_field(schema, 'age') self.assertEqual(age['aliases'], ['length']) + def test_dump_array(self): + class TestModel(models.Model): + tags = models.ListField('The tags', type_check=str) + + schema_dumper = self.mk_schema_dumper() + schema = json.loads(schema_dumper.dump_schema(TestModel)) + from pprint import pprint + pprint(schema) + class TestLoadSchemaTool(ToolBaseTest): From 2fd28d7f114e84ca998d78aa9745ac9521b264c6 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 11 Feb 2015 15:55:48 +0200 Subject: [PATCH 03/11] break things --- elasticgit/tests/base.py | 4 +++- requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/elasticgit/tests/base.py b/elasticgit/tests/base.py index af1073e..f157e4c 100644 --- a/elasticgit/tests/base.py +++ b/elasticgit/tests/base.py @@ -7,6 +7,8 @@ from unittest import TestCase +import avro.schema + from elasticgit.models import ( IntegerField, TextField, Model, SingleFieldFallback) from elasticgit.workspace import EG @@ -87,7 +89,7 @@ def mk_schema_dumper(self): return schema_dumper def get_schema(self, schema_dumper): - return json.loads(schema_dumper.stdout.getvalue()) + return avro.schema.parse(schema_dumper.stdout) def get_field(self, schema, field_name): return [field diff --git a/requirements.txt b/requirements.txt index 0302d52..ca1cfe0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ elasticutils==0.10.1 GitPython==0.3.5 Jinja2==2.7.3 Unidecode==0.04.16 +avro==1.7.7 From 605c577699773cfd20b3f67d4ce3812f82cac8d3 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 11 Feb 2015 20:12:46 +0200 Subject: [PATCH 04/11] better support for complex types like record & array --- elasticgit/commands/avro.py | 146 ++++++++++++++------ elasticgit/commands/gitmodel.py | 5 +- elasticgit/commands/tests/test_avro.py | 40 ++++-- elasticgit/commands/tests/test_gitmodel.py | 5 +- elasticgit/models.py | 31 ++++- elasticgit/templates/model_generator.py.txt | 1 + elasticgit/tests/base.py | 4 +- elasticgit/tests/test_models.py | 3 +- pytest.ini | 2 +- 9 files changed, 174 insertions(+), 63 deletions(-) diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index 696790e..9ca15c8 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -165,13 +165,21 @@ class SchemaLoader(ToolCommand): action='append', type=RenameType), ) - mapping = { + core_mapping = { 'int': IntegerField, 'string': TextField, 'float': FloatField, 'boolean': BooleanField, - 'array': ListField, - 'record': DictField, + } + + # How avro types map to Python types + core_type_mappings = { + 'string': basestring, + 'null': None, + 'integer': int, + 'number': float, + 'record': dict, + 'array': list, } def run(self, schema_files, field_mappings=None, model_renames=None): @@ -210,17 +218,26 @@ def field_class_for(self, field, field_mapping): if field_name in field_mapping: return field_mapping[field_name].__name__ - if isinstance(field_type, dict): + if field_type == 'record' or isinstance(field_type, dict): return self.field_class_for_complex_type(field) - return self.mapping[field_type].__name__ + return self.core_mapping[field_type].__name__ def field_class_for_complex_type(self, field): field_type = field['type'] - if (field_type['name'] == 'ModelVersionField' and - field_type['namespace'] == 'elasticgit.models'): + handler = getattr( + self, 'field_class_for_complex_%(type)s_type' % field_type) + return handler(field) + + def field_class_for_complex_record_type(self, field): + field_type = field['type'] + if (field_type.get('name') == 'ModelVersionField' and + field_type.get('namespace') == 'elasticgit.models'): return ModelVersionField.__name__ return DictField.__name__ + def field_class_for_complex_array_type(self, field): + return ListField.__name__ + def default_value(self, field): return pprint.pformat(field['default'], indent=8) @@ -275,6 +292,18 @@ def generate_model(self, schema, field_mapping={}, model_renames={}, self.field_class_for, field_mapping=field_mapping) env.globals['default_value'] = self.default_value + def python_types_for(field): + return ', '.join([self.core_type_mappings[type_].__name__ + for type_ in field['type']['items']]) + + env.globals['types_for'] = python_types_for + + def is_complex(field): + return ( + isinstance(field['type'], dict) or field['type'] == 'record') + + env.globals['is_complex'] = is_complex + template = env.get_template('model_generator.py.txt') return template.render( datetime=datetime.utcnow(), @@ -301,45 +330,25 @@ class SchemaDumper(ToolCommand): CommandArgument('class_path', help='python path to Class.'), ) - mapping = { + # How model fields map to types + core_field_mappings = { IntegerField: 'int', TextField: 'string', FloatField: 'float', BooleanField: 'boolean', - ListField: { - 'type': 'array', - 'name': 'list', - 'items': 'string', - }, - DictField: 'record', UUIDField: 'string', - ModelVersionField: { - 'type': 'record', - 'name': 'ModelVersionField', - 'namespace': 'elasticgit.models', - 'fields': [ - { - 'name': 'language', - 'type': 'string', - }, - { - 'name': 'language_version_string', - 'type': 'string', - }, - { - 'name': 'language_version', - 'type': 'string', - }, - { - 'name': 'package', - 'type': 'string', - }, - { - 'name': 'package_version', - 'type': 'string', - } - ] - } + } + + # How python types map to Avro types + core_type_mappings = { + basestring: 'string', + str: 'string', + unicode: 'string', + None: 'null', + int: 'integer', + float: 'number', + dict: 'record', + list: 'array', } def run(self, class_path): @@ -374,6 +383,59 @@ def dump_schema(self, model_class): for name, field in model_class._fields.items()], }, indent=2) + def map_field_to_type(self, field): + if field.__class__ in self.core_field_mappings: + return self.core_field_mappings[field.__class__] + + handler = getattr(self, 'map_%s_type' % (field.__class__.__name__,)) + return handler(field) + + def map_ListField_type(self, field): + avro_types = [self.core_type_mappings[type_] + for type_ in field.type_check.get_types()] + return { + 'type': 'array', + 'items': avro_types + } + + def map_DictField_type(self, field): + avro_types = [self.core_type_mappings[type_] + for type_ in field.type_check.get_types()] + return { + 'type': 'record', + 'items': avro_types, + } + + def map_ModelVersionField_type(self, field): + return { + 'type': 'record', + 'name': 'ModelVersionField', + 'namespace': 'elasticgit.models', + 'items': ['string'], + 'fields': [ + { + 'name': 'language', + 'type': 'string', + }, + { + 'name': 'language_version_string', + 'type': 'string', + }, + { + 'name': 'language_version', + 'type': 'string', + }, + { + 'name': 'package', + 'type': 'string', + }, + { + 'name': 'package_version', + 'type': 'string', + } + ], + } + def get_field_info(self, name, field): """ Return the Avro field object for an @@ -387,7 +449,7 @@ def get_field_info(self, name, field): """ return { 'name': name, - 'type': self.mapping[field.__class__], + 'type': self.map_field_to_type(field), 'doc': field.doc, 'default': field.default, 'aliases': [fallback.field_name for fallback in field.fallbacks] diff --git a/elasticgit/commands/gitmodel.py b/elasticgit/commands/gitmodel.py index 3b8a6b9..1ff830b 100644 --- a/elasticgit/commands/gitmodel.py +++ b/elasticgit/commands/gitmodel.py @@ -153,6 +153,9 @@ def guess_type(self, value): float: 'float', str: 'string', unicode: 'string', - list: 'array', + list: { + 'type': 'array', + 'items': ['string'], + }, None: 'null', }[None if value is None else type(value)] diff --git a/elasticgit/commands/tests/test_avro.py b/elasticgit/commands/tests/test_avro.py index 2731b7e..79f9713 100644 --- a/elasticgit/commands/tests/test_avro.py +++ b/elasticgit/commands/tests/test_avro.py @@ -49,7 +49,8 @@ class TestModel(models.Model): def test_dump_array(self): class TestModel(models.Model): - tags = models.ListField('The tags', type_check=str) + tags = models.ListField('The tags', + type_check=models.TypeCheck(str)) schema_dumper = self.mk_schema_dumper() schema = json.loads(schema_dumper.dump_schema(TestModel)) @@ -133,7 +134,10 @@ def test_boolean_field(self): def test_array_field(self): self.assertFieldCreation({ 'name': 'array', - 'type': 'array', + 'type': { + 'type': 'array', + 'items': ['string'], + }, 'doc': 'The Array', 'default': ['foo', 'bar', 'baz'] }, models.ListField) @@ -141,7 +145,14 @@ def test_array_field(self): def test_dict_field(self): self.assertFieldCreation({ 'name': 'obj', - 'type': 'record', + 'type': { + 'type': 'record', + 'items': ['string'], + 'fields': [{ + 'name': 'hello', + 'type': 'string', + }] + }, 'doc': 'The Object', 'default': {'hello': 'world'}, }, models.DictField) @@ -152,7 +163,12 @@ def test_complex_field(self): 'type': { 'namespace': 'foo.bar', 'name': 'ItIsComplicated', - 'type': 'record' + 'type': 'record', + 'items': ['string'], + 'fields': [{ + 'name': 'foo', + 'type': 'string', + }] }, 'doc': 'Super Complex', 'default': {}, @@ -165,6 +181,7 @@ def test_version_field(self): 'namespace': 'elasticgit.models', 'name': 'ModelVersionField', 'type': 'record', + 'items': ['string'], }, 'doc': 'The Model Version', 'default': elasticgit.version_info, @@ -186,8 +203,8 @@ class DumpAndLoadModel(models.Model): integer = models.IntegerField('the integer') float_ = models.FloatField('the float') boolean = models.BooleanField('the boolean') - list_ = models.ListField('the list') - dict_ = models.DictField('the dict') + list_ = models.ListField('the list', type_check=models.TypeCheck(int)) + dict_ = models.DictField('the dict', type_check=models.TypeCheck(str)) class TestDumpAndLoad(ToolBaseTest): @@ -198,8 +215,13 @@ def test_two_way(self): schema_loader = self.mk_schema_loader() schema = schema_dumper.dump_schema(DumpAndLoadModel) + + print schema + generated_code = schema_loader.generate_model(json.loads(schema)) + print generated_code + GeneratedModel = self.load_class(generated_code, 'DumpAndLoadModel') data = { @@ -207,7 +229,7 @@ def test_two_way(self): 'integer': 1, 'float': 1.1, 'boolean': False, - 'list': ['1', '2', '3'], + 'list_': [1, 2, 3], 'dict_': {'hello': 'world'} } record1 = DumpAndLoadModel(data) @@ -229,8 +251,8 @@ def test_two_way_dict_ints(self): 'integer': 1, 'float': 1.1, 'boolean': False, - 'list': ['1', '2', '3'], - 'dict_': {'hello': 1} + 'list': [1, 2, 3], + 'dict_': {'hello': '1'} } record1 = DumpAndLoadModel(data) record2 = GeneratedModel(data) diff --git a/elasticgit/commands/tests/test_gitmodel.py b/elasticgit/commands/tests/test_gitmodel.py index 522c070..883f1f3 100644 --- a/elasticgit/commands/tests/test_gitmodel.py +++ b/elasticgit/commands/tests/test_gitmodel.py @@ -146,7 +146,10 @@ def test_introspect_page_schema(self): ('created_at', 'string'), ('featured_in_category', 'boolean'), ('modified_at', 'string'), - ('linked_pages', 'array'), + ('linked_pages', { + 'type': 'array', + 'items': ['string'], + }), ('slug', 'string'), ('content', 'string'), ('source', 'string'), # inferred null but default type is string diff --git a/elasticgit/models.py b/elasticgit/models.py index e37d3ab..2e2cf43 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -106,14 +106,18 @@ def clean(self, value): class TypeCheck(object): - def __init__(self, type_): - self.type_ = type_ + def __init__(self, *types): + self.types = types - def get_type(self): - return self.type_ + def get_types(self): + return self.types def __call__(self, value): - return isinstance(value, self.type_) + return any([isinstance(value, type_) for type_ in self.types]) + + def __repr__(self): + return '' % ( + ', '.join([type_.__name__ for type_ in self.types])) class ListField(ModelField): @@ -127,7 +131,7 @@ class ListField(ModelField): 'type': 'string', } - def __init__(self, doc, type_check=None, default=[], static=False, + def __init__(self, doc, type_check, default=[], static=False, fallbacks=(), mapping={}): super(ListField, self).__init__( doc, default=default, static=static, fallbacks=fallbacks, @@ -158,9 +162,21 @@ class DictField(ModelField): 'type': 'string', } + def __init__(self, doc, type_check, default=None, static=False, + fallbacks=(), mapping={}): + super(DictField, self).__init__( + doc, default=default, static=static, fallbacks=fallbacks, + mapping=mapping) + self.type_check = type_check + def clean(self, value): if not isinstance(value, dict): self.raise_config_error("is not a dict.") + if self.type_check is not None: + if not all([self.type_check(v) for v in value.values()]): + self.raise_config_error( + 'Type check %s failed for some values: %r' % ( + self.type_check, value)) return deepcopy(value) @@ -235,7 +251,8 @@ class Model(Config): A dictionary with keys & values to populate this Model instance with. """ - _version = ModelVersionField('Model Version Identifier') + _version = ModelVersionField('Model Version Identifier', + type_check=TypeCheck(basestring)) uuid = UUIDField('Unique Identifier') def __init__(self, config_data, static=False): diff --git a/elasticgit/templates/model_generator.py.txt b/elasticgit/templates/model_generator.py.txt index ad7f4ec..8026f3f 100644 --- a/elasticgit/templates/model_generator.py.txt +++ b/elasticgit/templates/model_generator.py.txt @@ -22,6 +22,7 @@ class {{model_class_for(schema.name)}}(models.Model): {{field.name}} = models.{{ field_class_for(field) }}( {%- if field.doc -%}u"""{{ field.doc }}"""{% else %}"""{{ field.name }}"""{% endif %} {%- if field.default is defined -%}, default={{ default_value(field) }}{% endif %} + {%- if is_complex(field) -%}, type_check=models.TypeCheck({{types_for(field)}}){% endif %} {%- if field.aliases -%}, fallbacks=[{% for alias in field.aliases -%} models.SingleFieldFallback('{{alias}}'), {%- endfor %}]{% endif %}){% endfor %} diff --git a/elasticgit/tests/base.py b/elasticgit/tests/base.py index f157e4c..fdf1b50 100644 --- a/elasticgit/tests/base.py +++ b/elasticgit/tests/base.py @@ -89,7 +89,9 @@ def mk_schema_dumper(self): return schema_dumper def get_schema(self, schema_dumper): - return avro.schema.parse(schema_dumper.stdout) + data = schema_dumper.stdout.getvalue() + schema = avro.schema.parse(data) + return schema.to_json() def get_field(self, schema, field_name): return [field diff --git a/elasticgit/tests/test_models.py b/elasticgit/tests/test_models.py index 9e76b6f..4d9cbf1 100644 --- a/elasticgit/tests/test_models.py +++ b/elasticgit/tests/test_models.py @@ -83,7 +83,8 @@ def test_update(self): self.assertFalse(new_model.is_read_only()) def test_version_check(self): - field = ModelVersionField('ModelVersionField') + field = ModelVersionField('ModelVersionField', + type_check=TypeCheck(basestring)) self.assertTrue(field.compatible_version('0.2.10', '0.2.9')) self.assertTrue(field.compatible_version('0.2.10', '0.2.10')) self.assertFalse(field.compatible_version('0.2.9', '0.2.10')) diff --git a/pytest.ini b/pytest.ini index 68d5566..1fe28e2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules --verbose +addopts = --doctest-modules --verbose -s From 16d0d1b62b2c3aeae5644725b044f8a13ed55cc7 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 11 Feb 2015 20:20:27 +0200 Subject: [PATCH 05/11] use avro.schema.parse more widely --- elasticgit/commands/avro.py | 9 +++++++-- elasticgit/models.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index 9ca15c8..2a36229 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from jinja2 import Environment, PackageLoader from functools import partial import argparse @@ -7,6 +9,8 @@ from datetime import datetime +import avro.schema + from elasticgit import version_info from elasticgit.models import ( Model, IntegerField, TextField, ModelVersionField, FloatField, @@ -17,11 +21,11 @@ from elasticgit.utils import load_class -def deserialize(schema, field_mapping={}, module_name=None): +def deserialize(data, field_mapping={}, module_name=None): """ Deserialize an Avro schema and define it within a module (if specified) - :param dict schema: + :param dict data: The Avro schema :param dict field_mapping: Optional mapping to override the default mapping. @@ -47,6 +51,7 @@ def deserialize(schema, field_mapping={}, module_name=None): """ schema_loader = SchemaLoader() + schema = avro.schema.make_avsc_object(data, avro.schema.Names()).to_json() model_code = schema_loader.generate_model(schema) model_name = schema['name'] diff --git a/elasticgit/models.py b/elasticgit/models.py index 2e2cf43..0ca4710 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -215,6 +215,13 @@ class ModelVersionField(DictField): } } + def __init__(self, doc, type_check=TypeCheck(basestring), + default=None, static=False, + fallbacks=(), mapping={}): + super(ModelVersionField, self).__init__( + doc, type_check=type_check, default=default, static=static, + fallbacks=fallbacks, mapping=mapping) + def compatible_version(self, own_version, check_version): own = map(int, own_version.split('.')) check = map(int, check_version.split('.')) @@ -251,8 +258,7 @@ class Model(Config): A dictionary with keys & values to populate this Model instance with. """ - _version = ModelVersionField('Model Version Identifier', - type_check=TypeCheck(basestring)) + _version = ModelVersionField('Model Version Identifier') uuid = UUIDField('Unique Identifier') def __init__(self, config_data, static=False): From cfeb7c905354a5d33ff79629cfe020c535dc2d4b Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Thu, 12 Feb 2015 20:26:41 +0200 Subject: [PATCH 06/11] kill ModelVersionField add concept of fields for typing of ListField and DictField --- elasticgit/__init__.py | 13 -- elasticgit/commands/avro.py | 77 ++++------- elasticgit/commands/tests/test_avro.py | 34 +++-- elasticgit/commands/tests/test_version.py | 2 +- elasticgit/commands/version.py | 2 +- elasticgit/models.py | 135 +++++++++++--------- elasticgit/templates/model_generator.py.txt | 11 +- elasticgit/tests/base.py | 1 + elasticgit/tests/test_index.py | 7 +- elasticgit/tests/test_manager.py | 4 +- elasticgit/tests/test_models.py | 24 ++-- pytest.ini | 2 +- 12 files changed, 148 insertions(+), 164 deletions(-) diff --git a/elasticgit/__init__.py b/elasticgit/__init__.py index 16f63a3..e480eb9 100644 --- a/elasticgit/__init__.py +++ b/elasticgit/__init__.py @@ -1,19 +1,6 @@ import pkg_resources -import sys from elasticgit.workspace import EG, F, Q __all__ = ['EG', 'F', 'Q'] __version__ = pkg_resources.require('elastic-git')[0].version - -version_info = { - 'language': 'python', - 'language_version_string': sys.version, - 'language_version': '%d.%d.%d' % ( - sys.version_info.major, - sys.version_info.minor, - sys.version_info.micro, - ), - 'package': 'elastic-git', - 'package_version': __version__ -} diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index 2a36229..b26e9a9 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -11,10 +11,10 @@ import avro.schema -from elasticgit import version_info from elasticgit.models import ( - Model, IntegerField, TextField, ModelVersionField, FloatField, - BooleanField, ListField, DictField, UUIDField) + Model, IntegerField, TextField, FloatField, + BooleanField, ListField, DictField, UUIDField, + version_info) from elasticgit.commands.base import ( ToolCommand, ToolCommandError, CommandArgument) @@ -234,10 +234,6 @@ def field_class_for_complex_type(self, field): return handler(field) def field_class_for_complex_record_type(self, field): - field_type = field['type'] - if (field_type.get('name') == 'ModelVersionField' and - field_type.get('namespace') == 'elasticgit.models'): - return ModelVersionField.__name__ return DictField.__name__ def field_class_for_complex_array_type(self, field): @@ -296,18 +292,22 @@ def generate_model(self, schema, field_mapping={}, model_renames={}, env.globals['field_class_for'] = partial( self.field_class_for, field_mapping=field_mapping) env.globals['default_value'] = self.default_value + env.globals['is_complex'] = ( + lambda field: isinstance(field['type'], dict)) + env.globals['core_mapping'] = self.core_mapping - def python_types_for(field): - return ', '.join([self.core_type_mappings[type_].__name__ - for type_ in field['type']['items']]) - - env.globals['types_for'] = python_types_for + # def complex_fields_for(complex_field): + # field_type = complex_field['type'] + # if field_type['type'] == 'array': + # items = field_type['items'] + # elif field_type['type'] == 'record': + # items = [f['type'] for f in field_type['fields']] + # from pprint import pprint + # pprint(items) - def is_complex(field): - return ( - isinstance(field['type'], dict) or field['type'] == 'record') + # return '\n'.join([self.core_mapping[field].__name__ for field in items]) - env.globals['is_complex'] = is_complex + # env.globals['complex_fields_for'] = complex_fields_for template = env.get_template('model_generator.py.txt') return template.render( @@ -396,49 +396,22 @@ def map_field_to_type(self, field): return handler(field) def map_ListField_type(self, field): - avro_types = [self.core_type_mappings[type_] - for type_ in field.type_check.get_types()] return { 'type': 'array', - 'items': avro_types + 'name': field.name, + 'namespace': field.__class__.__module__, + 'items': [self.map_field_to_type(field) for field in field.fields], } def map_DictField_type(self, field): - avro_types = [self.core_type_mappings[type_] - for type_ in field.type_check.get_types()] - return { - 'type': 'record', - 'items': avro_types, - } - - def map_ModelVersionField_type(self, field): return { 'type': 'record', - 'name': 'ModelVersionField', - 'namespace': 'elasticgit.models', - 'items': ['string'], - 'fields': [ - { - 'name': 'language', - 'type': 'string', - }, - { - 'name': 'language_version_string', - 'type': 'string', - }, - { - 'name': 'language_version', - 'type': 'string', - }, - { - 'name': 'package', - 'type': 'string', - }, - { - 'name': 'package_version', - 'type': 'string', - } - ], + 'name': field.name, + 'namespace': field.__class__.__module__, + 'fields': [{ + 'name': field.name, + 'type': self.map_field_to_type(field), + } for field in field.fields], } def get_field_info(self, name, field): diff --git a/elasticgit/commands/tests/test_avro.py b/elasticgit/commands/tests/test_avro.py index 79f9713..7e712a9 100644 --- a/elasticgit/commands/tests/test_avro.py +++ b/elasticgit/commands/tests/test_avro.py @@ -4,8 +4,6 @@ from elasticgit import models from elasticgit.tests.base import ToolBaseTest -import elasticgit - class TestDumpSchemaTool(ToolBaseTest): @@ -50,7 +48,7 @@ class TestModel(models.Model): def test_dump_array(self): class TestModel(models.Model): tags = models.ListField('The tags', - type_check=models.TypeCheck(str)) + fields=(models.IntegerField('doc'),)) schema_dumper = self.mk_schema_dumper() schema = json.loads(schema_dumper.dump_schema(TestModel)) @@ -179,13 +177,13 @@ def test_version_field(self): 'name': 'version', 'type': { 'namespace': 'elasticgit.models', - 'name': 'ModelVersionField', + 'name': 'version', 'type': 'record', 'items': ['string'], }, 'doc': 'The Model Version', - 'default': elasticgit.version_info, - }, models.ModelVersionField) + 'default': models.version_info, + }, models.DictField) def test_mapping_hints(self): self.assertFieldCreation({ @@ -203,8 +201,12 @@ class DumpAndLoadModel(models.Model): integer = models.IntegerField('the integer') float_ = models.FloatField('the float') boolean = models.BooleanField('the boolean') - list_ = models.ListField('the list', type_check=models.TypeCheck(int)) - dict_ = models.DictField('the dict', type_check=models.TypeCheck(str)) + list_ = models.ListField('the list', fields=( + models.IntegerField('the int'), + )) + dict_ = models.DictField('the dict', fields=( + models.TextField('hello', name='hello'), + )) class TestDumpAndLoad(ToolBaseTest): @@ -216,12 +218,8 @@ def test_two_way(self): schema = schema_dumper.dump_schema(DumpAndLoadModel) - print schema - generated_code = schema_loader.generate_model(json.loads(schema)) - print generated_code - GeneratedModel = self.load_class(generated_code, 'DumpAndLoadModel') data = { @@ -251,7 +249,7 @@ def test_two_way_dict_ints(self): 'integer': 1, 'float': 1.1, 'boolean': False, - 'list': [1, 2, 3], + 'list_': [1, 2, 3], 'dict_': {'hello': '1'} } record1 = DumpAndLoadModel(data) @@ -273,7 +271,7 @@ def test_two_way_list_ints(self): 'integer': 1, 'float': 1.1, 'boolean': False, - 'list': [1, 2, 3], + 'list_': [1, 2, 3], 'dict_': {'hello': '1'} } record1 = DumpAndLoadModel(data) @@ -295,7 +293,7 @@ def test_two_way_list_unicode(self): 'integer': 1, 'float': 1.1, 'boolean': False, - 'list': [1, 2, 3], + 'list_': [1, 2, 3], 'dict_': {'hello': '1'} } record1 = DumpAndLoadModel(data) @@ -357,7 +355,7 @@ def test_load_older_version(self): class Foo(models.Model): pass - old_version_info = elasticgit.version_info.copy() + old_version_info = models.version_info.copy() old_version_info['package_version'] = '0.0.1' f = Foo({ @@ -372,9 +370,9 @@ class Foo(models.Model): pass major, minor, micro = map( - int, elasticgit.version_info['package_version'].split('.')) + int, models.version_info['package_version'].split('.')) - new_version = elasticgit.version_info.copy() + new_version = models.version_info.copy() new_version['package_version'] = '%d.%d.%d' % ( major + 1, minor, diff --git a/elasticgit/commands/tests/test_version.py b/elasticgit/commands/tests/test_version.py index e6dbe83..0adc36e 100644 --- a/elasticgit/commands/tests/test_version.py +++ b/elasticgit/commands/tests/test_version.py @@ -1,7 +1,7 @@ from StringIO import StringIO import json -from elasticgit import version_info +from elasticgit.models import version_info from elasticgit.tests.base import ToolBaseTest from elasticgit.commands.version import VersionTool, DEFAULT_FILE_NAME diff --git a/elasticgit/commands/version.py b/elasticgit/commands/version.py index 9ae5ead..c0e1c6f 100644 --- a/elasticgit/commands/version.py +++ b/elasticgit/commands/version.py @@ -1,7 +1,7 @@ import sys import json -from elasticgit import version_info +from elasticgit.models import version_info from elasticgit.commands.base import ToolCommand, CommandArgument diff --git a/elasticgit/models.py b/elasticgit/models.py index 0ca4710..0f28e58 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -1,3 +1,5 @@ +import sys +import pkg_resources from copy import deepcopy from urllib2 import urlparse import uuid @@ -7,7 +9,17 @@ from confmodel.fallbacks import SingleFieldFallback -import elasticgit +version_info = { + 'language': 'python', + 'language_version_string': sys.version, + 'language_version': '%d.%d.%d' % ( + sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro, + ), + 'package': 'elastic-git', + 'package_version': pkg_resources.require('elastic-git')[0].version +} class ModelField(ConfigField): @@ -17,13 +29,18 @@ class ModelField(ConfigField): } def __init__(self, doc, required=False, default=None, static=False, - fallbacks=(), mapping={}): + fallbacks=(), mapping={}, name=None): super(ModelField, self).__init__( doc, required=required, default=default, static=static, fallbacks=fallbacks) + self.name = name self.mapping = self.__class__.default_mapping.copy() self.mapping.update(mapping) + def __repr__(self): + return '<%s.%s %r>' % ( + self.__class__.__module__, self.__class__.__name__, self.name) + class TextField(ModelField): """ @@ -131,23 +148,24 @@ class ListField(ModelField): 'type': 'string', } - def __init__(self, doc, type_check, default=[], static=False, + def __init__(self, doc, fields, default=[], static=False, fallbacks=(), mapping={}): super(ListField, self).__init__( doc, default=default, static=static, fallbacks=fallbacks, mapping=mapping) - self.type_check = type_check + self.fields = fields def clean(self, value): if isinstance(value, tuple): value = list(value) if not isinstance(value, list): self.raise_config_error("is not a list.") - if self.type_check is not None: - if not all([self.type_check(v) for v in value]): - self.raise_config_error( - 'Type check %r failed for some values.' % ( - self.type_check,)) + + if len(value) > 0: + for field in self.fields: + if not any([field.clean(v) for v in value]): + self.raise_config_error( + 'All field checks failed for some values.') return deepcopy(value) @@ -162,23 +180,25 @@ class DictField(ModelField): 'type': 'string', } - def __init__(self, doc, type_check, default=None, static=False, - fallbacks=(), mapping={}): + def __init__(self, doc, fields, default=None, static=False, + fallbacks=(), mapping=()): super(DictField, self).__init__( doc, default=default, static=static, fallbacks=fallbacks, mapping=mapping) - self.type_check = type_check + self.fields = fields def clean(self, value): if not isinstance(value, dict): - self.raise_config_error("is not a dict.") - if self.type_check is not None: - if not all([self.type_check(v) for v in value.values()]): - self.raise_config_error( - 'Type check %s failed for some values: %r' % ( - self.type_check, value)) + self.raise_config_error('is not a dict.') return deepcopy(value) + def validate(self, config): + data = self.get_value(config) + if data: + for key, value in data.items(): + [field] = [field for field in self.fields if field.name == key] + field.clean(value) + class URLField(ModelField): """ @@ -200,46 +220,6 @@ def clean(self, value): return urlparse.urlparse(value) -class ModelVersionField(DictField): - """ - A field holding the version information for a model - """ - default_mapping = { - 'type': 'nested', - 'properties': { - 'language': {'type': 'string'}, - 'language_version_string': {'type': 'string'}, - 'language_version': {'type': 'string'}, - 'package': {'type': 'string'}, - 'package_version': {'type': 'string'} - } - } - - def __init__(self, doc, type_check=TypeCheck(basestring), - default=None, static=False, - fallbacks=(), mapping={}): - super(ModelVersionField, self).__init__( - doc, type_check=type_check, default=default, static=static, - fallbacks=fallbacks, mapping=mapping) - - def compatible_version(self, own_version, check_version): - own = map(int, own_version.split('.')) - check = map(int, check_version.split('.')) - return own >= check - - def validate(self, config): - config._config_data.setdefault( - self.name, elasticgit.version_info.copy()) - value = self.get_value(config) - current_version = elasticgit.version_info['package_version'] - package_version = value['package_version'] - if not self.compatible_version(current_version, package_version): - raise ConfigError( - 'Got a version from the future, expecting: %r got %r' % ( - current_version, package_version)) - return super(ModelVersionField, self).validate(config) - - class UUIDField(TextField): def validate(self, config): @@ -258,7 +238,28 @@ class Model(Config): A dictionary with keys & values to populate this Model instance with. """ - _version = ModelVersionField('Model Version Identifier') + _version = DictField( + 'Model Version Identifier', + default=version_info, + fields=( + TextField('language', name='language'), + TextField('language_version_string', + name='language_version_string'), + TextField('language_version', name='language_version'), + TextField('package', name='package'), + TextField('package_version', name='package_version'), + ), + mapping={ + 'type': 'nested', + 'properties': { + 'language': {'type': 'string'}, + 'language_version_string': {'type': 'string'}, + 'language_version': {'type': 'string'}, + 'package': {'type': 'string'}, + 'package_version': {'type': 'string'} + } + }) + uuid = UUIDField('Unique Identifier') def __init__(self, config_data, static=False): @@ -292,5 +293,21 @@ def __iter__(self): for field in self._get_fields(): yield field.name, field.get_value(self) + def compatible_version(self, own_version, check_version): + own = map(int, own_version.split('.')) + check = map(int, check_version.split('.')) + return own >= check + + def post_validate(self): + value = self._version + current_version = version_info['package_version'] + package_version = value['package_version'] + if not self.compatible_version(current_version, package_version): + raise ConfigError( + 'Got a version from the future, expecting: %r got %r' % ( + current_version, package_version)) + super(Model, self).post_validate() + + ConfigError SingleFieldFallback diff --git a/elasticgit/templates/model_generator.py.txt b/elasticgit/templates/model_generator.py.txt index 8026f3f..3ff78f9 100644 --- a/elasticgit/templates/model_generator.py.txt +++ b/elasticgit/templates/model_generator.py.txt @@ -22,7 +22,16 @@ class {{model_class_for(schema.name)}}(models.Model): {{field.name}} = models.{{ field_class_for(field) }}( {%- if field.doc -%}u"""{{ field.doc }}"""{% else %}"""{{ field.name }}"""{% endif %} {%- if field.default is defined -%}, default={{ default_value(field) }}{% endif %} - {%- if is_complex(field) -%}, type_check=models.TypeCheck({{types_for(field)}}){% endif %} + {%- if is_complex(field) -%}, fields=( + {%- set sub_type = field['type'] %} + {%- if sub_type['type'] == 'array' %} + {% for item in sub_type['items'] %} + models.{{core_mapping[item].__name__}}('{{item}}'),{%- endfor %} + {%- elif sub_type['type'] == 'record' %} + {% for field in sub_type['fields'] %} + models.{{core_mapping[field.type].__name__}}('{{field.name}}', name='{{field.name}}'),{%- endfor %} + {% endif %} + ){% endif %} {%- if field.aliases -%}, fallbacks=[{% for alias in field.aliases -%} models.SingleFieldFallback('{{alias}}'), {%- endfor %}]{% endif %}){% endfor %} diff --git a/elasticgit/tests/base.py b/elasticgit/tests/base.py index fdf1b50..d50b18d 100644 --- a/elasticgit/tests/base.py +++ b/elasticgit/tests/base.py @@ -152,5 +152,6 @@ def load_class_with_field(self, field, field_mapping={}, model_renames={}, model_name, field_mapping=field_mapping, model_renames=model_renames) + print model_code model_name = model_renames.get(model_name, model_name) return self.load_class(model_code, model_name) diff --git a/elasticgit/tests/test_index.py b/elasticgit/tests/test_index.py index f853a78..92e7e22 100644 --- a/elasticgit/tests/test_index.py +++ b/elasticgit/tests/test_index.py @@ -1,9 +1,8 @@ +from elasticgit.models import version_info from elasticgit.tests.base import ModelBaseTest, TestPerson from elasticutils import S -import elasticgit - class TestIndex(ModelBaseTest): @@ -38,7 +37,7 @@ def test_extract_document_with_object(self): 'age': 1, 'name': 'Kees', 'uuid': person.uuid, - '_version': elasticgit.version_info, + '_version': version_info, }) def test_extract_document_with_object_id(self): @@ -53,7 +52,7 @@ def test_extract_document_with_object_id(self): 'age': 1, 'name': 'Kees', 'uuid': person.uuid, - '_version': elasticgit.version_info, + '_version': version_info, }) def test_indexing(self): diff --git a/elasticgit/tests/test_manager.py b/elasticgit/tests/test_manager.py index f82e31e..a6f8209 100644 --- a/elasticgit/tests/test_manager.py +++ b/elasticgit/tests/test_manager.py @@ -3,7 +3,7 @@ import os from elasticgit import EG -from elasticgit.models import IntegerField, ModelVersionField +from elasticgit.models import IntegerField, DictField from elasticgit.tests.base import ModelBaseTest, TestPage, TestPerson from elasticsearch.client import Elasticsearch @@ -50,7 +50,7 @@ def test_indexable(self): 'properties': { 'age': {'type': 'integer'}, 'uuid': {'type': 'string'}, - '_version': ModelVersionField.default_mapping, + '_version': model_class._fields['_version'].mapping, } }) model_instance = model_class({'age': 1}) diff --git a/elasticgit/tests/test_models.py b/elasticgit/tests/test_models.py index 4d9cbf1..a1142db 100644 --- a/elasticgit/tests/test_models.py +++ b/elasticgit/tests/test_models.py @@ -1,9 +1,6 @@ from elasticgit.tests.base import ModelBaseTest from elasticgit.models import ( - ConfigError, IntegerField, TextField, ModelVersionField, - ListField, TypeCheck) - -import elasticgit + ConfigError, IntegerField, TextField, ListField, version_info) class TestModel(ModelBaseTest): @@ -43,7 +40,7 @@ def test_to_dict(self): 'age': IntegerField('An age'), 'name': TextField('A name'), }) - data = {'age': 1, 'name': 'foo', '_version': elasticgit.version_info} + data = {'age': 1, 'name': 'foo', '_version': version_info} model = model_class(data) self.assertEqual(dict(model), data) @@ -83,18 +80,21 @@ def test_update(self): self.assertFalse(new_model.is_read_only()) def test_version_check(self): - field = ModelVersionField('ModelVersionField', - type_check=TypeCheck(basestring)) - self.assertTrue(field.compatible_version('0.2.10', '0.2.9')) - self.assertTrue(field.compatible_version('0.2.10', '0.2.10')) - self.assertFalse(field.compatible_version('0.2.9', '0.2.10')) + model_class = self.mk_model({}) + model = model_class({}) + self.assertTrue(model.compatible_version('0.2.10', '0.2.9')) + self.assertTrue(model.compatible_version('0.2.10', '0.2.10')) + self.assertFalse(model.compatible_version('0.2.9', '0.2.10')) def test_list_field(self): model_class = self.mk_model({ - 'tags': ListField('list field', type_check=TypeCheck(int)) + 'tags': ListField('list field', fields=( + IntegerField('int'), + )) }) - self.assertRaises(ConfigError, model_class, {'tags': ['1']}) + self.assertRaises(ConfigError, model_class, {'tags': ['a']}) self.assertRaises(ConfigError, model_class, {'tags': [2.0]}) self.assertRaises(ConfigError, model_class, {'tags': [None]}) self.assertTrue(model_class({'tags': []})) self.assertTrue(model_class({'tags': [1, 2, 3]})) + self.assertTrue(model_class({'tags': ['1']})) diff --git a/pytest.ini b/pytest.ini index 1fe28e2..18dd171 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules --verbose -s +;addopts = --doctest-modules --verbose -s From 383f05dd00296be1dff9f9b45dee526cfa8ef60d Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Thu, 12 Feb 2015 20:33:07 +0200 Subject: [PATCH 07/11] removing some unused code --- elasticgit/commands/avro.py | 43 ++++--------------------------------- 1 file changed, 4 insertions(+), 39 deletions(-) diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index b26e9a9..6329604 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -177,16 +177,6 @@ class SchemaLoader(ToolCommand): 'boolean': BooleanField, } - # How avro types map to Python types - core_type_mappings = { - 'string': basestring, - 'null': None, - 'integer': int, - 'number': float, - 'record': dict, - 'array': list, - } - def run(self, schema_files, field_mappings=None, model_renames=None): """ Inspect an Avro schema file and write the generated Python code @@ -296,19 +286,6 @@ def generate_model(self, schema, field_mapping={}, model_renames={}, lambda field: isinstance(field['type'], dict)) env.globals['core_mapping'] = self.core_mapping - # def complex_fields_for(complex_field): - # field_type = complex_field['type'] - # if field_type['type'] == 'array': - # items = field_type['items'] - # elif field_type['type'] == 'record': - # items = [f['type'] for f in field_type['fields']] - # from pprint import pprint - # pprint(items) - - # return '\n'.join([self.core_mapping[field].__name__ for field in items]) - - # env.globals['complex_fields_for'] = complex_fields_for - template = env.get_template('model_generator.py.txt') return template.render( datetime=datetime.utcnow(), @@ -344,18 +321,6 @@ class SchemaDumper(ToolCommand): UUIDField: 'string', } - # How python types map to Avro types - core_type_mappings = { - basestring: 'string', - str: 'string', - unicode: 'string', - None: 'null', - int: 'integer', - float: 'number', - dict: 'record', - list: 'array', - } - def run(self, class_path): """ Introspect the given class path and print the schema to @@ -400,7 +365,7 @@ def map_ListField_type(self, field): 'type': 'array', 'name': field.name, 'namespace': field.__class__.__module__, - 'items': [self.map_field_to_type(field) for field in field.fields], + 'items': [self.map_field_to_type(fld) for fld in field.fields], } def map_DictField_type(self, field): @@ -409,9 +374,9 @@ def map_DictField_type(self, field): 'name': field.name, 'namespace': field.__class__.__module__, 'fields': [{ - 'name': field.name, - 'type': self.map_field_to_type(field), - } for field in field.fields], + 'name': fld.name, + 'type': self.map_field_to_type(fld), + } for fld in field.fields], } def get_field_info(self, name, field): From 2817b1087cf3fbebfae47b75fd269d9432d1d9ec Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Thu, 12 Feb 2015 20:53:41 +0200 Subject: [PATCH 08/11] removing obsolete field type check --- elasticgit/commands/avro.py | 2 +- elasticgit/commands/tests/test_avro.py | 6 ++++-- elasticgit/tests/base.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/elasticgit/commands/avro.py b/elasticgit/commands/avro.py index 6329604..bc0d7e2 100644 --- a/elasticgit/commands/avro.py +++ b/elasticgit/commands/avro.py @@ -213,7 +213,7 @@ def field_class_for(self, field, field_mapping): if field_name in field_mapping: return field_mapping[field_name].__name__ - if field_type == 'record' or isinstance(field_type, dict): + if isinstance(field_type, dict): return self.field_class_for_complex_type(field) return self.core_mapping[field_type].__name__ diff --git a/elasticgit/commands/tests/test_avro.py b/elasticgit/commands/tests/test_avro.py index 7e712a9..ff24ff3 100644 --- a/elasticgit/commands/tests/test_avro.py +++ b/elasticgit/commands/tests/test_avro.py @@ -52,8 +52,10 @@ class TestModel(models.Model): schema_dumper = self.mk_schema_dumper() schema = json.loads(schema_dumper.dump_schema(TestModel)) - from pprint import pprint - pprint(schema) + tags = self.get_field(schema, 'tags') + field_type = tags['type'] + self.assertEqual(field_type['type'], 'array') + self.assertEqual(field_type['items'], ['int']) class TestLoadSchemaTool(ToolBaseTest): diff --git a/elasticgit/tests/base.py b/elasticgit/tests/base.py index d50b18d..fdf1b50 100644 --- a/elasticgit/tests/base.py +++ b/elasticgit/tests/base.py @@ -152,6 +152,5 @@ def load_class_with_field(self, field, field_mapping={}, model_renames={}, model_name, field_mapping=field_mapping, model_renames=model_renames) - print model_code model_name = model_renames.get(model_name, model_name) return self.load_class(model_code, model_name) From 9562105753ba57c12beb23d13ead1a5ddc0e0750 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Thu, 12 Feb 2015 21:12:42 +0200 Subject: [PATCH 09/11] removing junk code --- elasticgit/models.py | 16 ---------------- pytest.ini | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/elasticgit/models.py b/elasticgit/models.py index 0f28e58..a1b16e3 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -121,22 +121,6 @@ def clean(self, value): return bool(value) -class TypeCheck(object): - - def __init__(self, *types): - self.types = types - - def get_types(self): - return self.types - - def __call__(self, value): - return any([isinstance(value, type_) for type_ in self.types]) - - def __repr__(self): - return '' % ( - ', '.join([type_.__name__ for type_ in self.types])) - - class ListField(ModelField): """ A list field diff --git a/pytest.ini b/pytest.ini index 18dd171..1fe28e2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -;addopts = --doctest-modules --verbose -s +addopts = --doctest-modules --verbose -s From 9fd2fdba36813c09dffb95cebc8340a214748610 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Fri, 13 Feb 2015 08:20:18 +0200 Subject: [PATCH 10/11] add flake8 check to travis --- .travis.yml | 2 ++ elasticgit/tests/test_manager.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e33de19..af017cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ install: - pip install -r requirements-dev.txt - pip install -e . - pip install coveralls + - pip install flake8 script: + - flake8 elasticgit - py.test elasticgit -s --cov ./elasticgit after_success: - coveralls diff --git a/elasticgit/tests/test_manager.py b/elasticgit/tests/test_manager.py index a6f8209..2191bca 100644 --- a/elasticgit/tests/test_manager.py +++ b/elasticgit/tests/test_manager.py @@ -3,7 +3,7 @@ import os from elasticgit import EG -from elasticgit.models import IntegerField, DictField +from elasticgit.models import IntegerField from elasticgit.tests.base import ModelBaseTest, TestPage, TestPerson from elasticsearch.client import Elasticsearch From 239bba8525de4b682ac3befb1a3613b312a6fade Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Fri, 13 Feb 2015 08:44:08 +0200 Subject: [PATCH 11/11] generate mapping for dictfields --- elasticgit/models.py | 14 +++++++++----- elasticgit/tests/test_models.py | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/elasticgit/models.py b/elasticgit/models.py index a1b16e3..a712bf3 100644 --- a/elasticgit/models.py +++ b/elasticgit/models.py @@ -159,18 +159,22 @@ class DictField(ModelField): """ field_type = 'dict' - #: Mapping for Elasticsearch - default_mapping = { - 'type': 'string', - } - def __init__(self, doc, fields, default=None, static=False, fallbacks=(), mapping=()): + mapping = mapping or self.generate_default_mapping(fields) super(DictField, self).__init__( doc, default=default, static=static, fallbacks=fallbacks, mapping=mapping) self.fields = fields + def generate_default_mapping(self, fields): + field_names = [field.name for field in fields] + return { + 'type': 'nested', + 'properties': dict( + [(name, {'type': 'string'}) for name in field_names]), + } + def clean(self, value): if not isinstance(value, dict): self.raise_config_error('is not a dict.') diff --git a/elasticgit/tests/test_models.py b/elasticgit/tests/test_models.py index a1142db..1aa69ce 100644 --- a/elasticgit/tests/test_models.py +++ b/elasticgit/tests/test_models.py @@ -1,6 +1,7 @@ from elasticgit.tests.base import ModelBaseTest from elasticgit.models import ( - ConfigError, IntegerField, TextField, ListField, version_info) + ConfigError, IntegerField, TextField, ListField, version_info, + DictField) class TestModel(ModelBaseTest): @@ -98,3 +99,19 @@ def test_list_field(self): self.assertTrue(model_class({'tags': []})) self.assertTrue(model_class({'tags': [1, 2, 3]})) self.assertTrue(model_class({'tags': ['1']})) + + def test_dict_field(self): + model_class = self.mk_model({ + 'foo': DictField('dict field', fields=( + TextField('a', name='a'), + TextField('b', name='b'), + )) + }) + self.assertEqual( + model_class._fields['foo'].mapping, { + 'type': 'nested', + 'properties': { + 'a': {'type': 'string'}, + 'b': {'type': 'string'}, + } + })