From 92bf6d2fe7abea79be456e95fb5610c287a50315 Mon Sep 17 00:00:00 2001 From: Valeryi Savich Date: Tue, 25 Oct 2016 00:27:14 +0300 Subject: [PATCH 1/4] Ported models_meta module --- aiorest_ws/db/orm/django/model_meta.py | 178 +++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 aiorest_ws/db/orm/django/model_meta.py diff --git a/aiorest_ws/db/orm/django/model_meta.py b/aiorest_ws/db/orm/django/model_meta.py new file mode 100644 index 0000000..5bc5a9d --- /dev/null +++ b/aiorest_ws/db/orm/django/model_meta.py @@ -0,0 +1,178 @@ +""" +Helper function for returning the field information that is associated +with a model class. This includes returning all the forward and reverse +relationships and their associated metadata. +""" +from collections import OrderedDict + +from aiorest_ws.utils.structures import FieldInfo, RelationInfo +from aiorest_ws.db.orm.django.compat import get_related_model, \ + get_remote_field + +__all__ = ( + '_get_pk', '_get_fields', '_get_to_field', '_get_forward_relationships', + '_get_reverse_relationships', '_merge_fields_and_pk', + '_merge_relationships', 'get_field_info', 'is_abstract_model' +) + + +def _get_pk(opts): + pk = opts.pk + rel = get_remote_field(pk) + + while rel and rel.parent_link: + # If model is a child via multi-table inheritance, use parent's pk. + pk = get_related_model(pk)._meta.pk + rel = get_remote_field(pk) + + return pk + + +def _get_fields(opts): + fields = OrderedDict() + opts_fields = [ + field for field in opts.fields + if field.serialize and not get_remote_field(field) + ] + for field in opts_fields: + fields[field.name] = field + + return fields + + +def _get_to_field(field): + return getattr(field, 'to_fields', None) and field.to_fields[0] + + +def _get_forward_relationships(opts): + """ + Returns an `OrderedDict` of field names to `RelationInfo`. + """ + forward_relations = OrderedDict() + forwards_fields = [ + field for field in opts.fields + if field.serialize and get_remote_field(field) + ] + for field in forwards_fields: + forward_relations[field.name] = RelationInfo( + model_field=field, + related_model=get_related_model(field), + to_many=False, + to_field=_get_to_field(field), + has_through_model=False + ) + + # Deal with forward many-to-many relationships. + many_to_many_fields = [ + field for field in opts.many_to_many + if field.serialize + ] + for field in many_to_many_fields: + forward_relations[field.name] = RelationInfo( + model_field=field, + related_model=get_related_model(field), + to_many=True, + # many-to-many do not have to_fields + to_field=None, + has_through_model=( + not get_remote_field(field).through._meta.auto_created + ) + ) + + return forward_relations + + +def _get_reverse_relationships(opts): + """ + Returns an `OrderedDict` of field names to `RelationInfo`. + """ + # Note that we have a hack here to handle internal API differences for + # this internal API across Django 1.7 -> Django 1.8. + # See: https://code.djangoproject.com/ticket/24208 + + reverse_relations = OrderedDict() + all_related_objects = [ + r for r in opts.related_objects + if not r.field.many_to_many + ] + for relation in all_related_objects: + accessor_name = relation.get_accessor_name() + related = getattr(relation, 'related_model', relation.model) + reverse_relations[accessor_name] = RelationInfo( + model_field=None, + related_model=related, + to_many=get_remote_field(relation.field).multiple, + to_field=_get_to_field(relation.field), + has_through_model=False + ) + + # Deal with reverse many-to-many relationships. + all_related_many_to_many_objects = [ + r for r in opts.related_objects + if r.field.many_to_many + ] + for relation in all_related_many_to_many_objects: + + has_through_model = False + through = getattr(get_remote_field(relation.field), 'through', None) + if through is not None: + remote_field = get_remote_field(relation.field) + has_through_model = not remote_field.through._meta.auto_created + + accessor_name = relation.get_accessor_name() + related = getattr(relation, 'related_model', relation.model) + reverse_relations[accessor_name] = RelationInfo( + model_field=None, + related_model=related, + to_many=True, + # manytomany do not have to_fields + to_field=None, + has_through_model=has_through_model + ) + + return reverse_relations + + +def _merge_fields_and_pk(pk, fields): + fields_and_pk = OrderedDict() + fields_and_pk['pk'] = pk + fields_and_pk[pk.name] = pk + fields_and_pk.update(fields) + + return fields_and_pk + + +def _merge_relationships(forward_relations, reverse_relations): + return OrderedDict( + list(forward_relations.items()) + + list(reverse_relations.items()) + ) + + +def get_field_info(model): + """ + Given a model class, returns a `FieldInfo` instance, which is a + `namedtuple`, containing metadata about the various field types on the + model including information about their relationships. + """ + opts = model._meta.concrete_model._meta + + pk = _get_pk(opts) + fields = _get_fields(opts) + forward_relations = _get_forward_relationships(opts) + reverse_relations = _get_reverse_relationships(opts) + fields_and_pk = _merge_fields_and_pk(pk, fields) + relationships = _merge_relationships(forward_relations, reverse_relations) + + return FieldInfo(pk, fields, forward_relations, reverse_relations, + fields_and_pk, relationships) + + +def is_abstract_model(model): + """ + Given a model class, returns a boolean True if it is abstract and False + if it is not. + """ + has_meta_attribute = hasattr(model, '_meta') + is_abstract = hasattr(model._meta, 'abstract') and model._meta.abstract + return has_meta_attribute and is_abstract From b81a60fba0836f5354582f7de5accfc08398d076 Mon Sep 17 00:00:00 2001 From: Valeryi Savich Date: Tue, 25 Oct 2016 00:49:40 +0300 Subject: [PATCH 2/4] Ported compat module --- aiorest_ws/db/orm/django/compat.py | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 aiorest_ws/db/orm/django/compat.py diff --git a/aiorest_ws/db/orm/django/compat.py b/aiorest_ws/db/orm/django/compat.py new file mode 100644 index 0000000..80ad400 --- /dev/null +++ b/aiorest_ws/db/orm/django/compat.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +""" +The `compat` module provides support for backwards compatibility with older +versions of Django, and compatibility wrappers around optional packages. +""" +import inspect + +import django + +from django.apps import apps +from django.core.exceptions import ImproperlyConfigured +from django.db import models + + +__all__ = [ + '_resolve_model', 'get_related_model', 'get_remote_field' +] + + +try: + # DecimalValidator is unavailable in Django < 1.9 + from django.core.validators import DecimalValidator +except ImportError: + DecimalValidator = None + + +def _resolve_model(obj): + """ + Resolve supplied `obj` to a Django model class. + `obj` must be a Django model class itself, or a string + representation of one. Useful in situations like GH #1225 where + Django may not have resolved a string-based reference to a model in + another model's foreign key definition. + String representations should have the format: + 'appname.ModelName' + """ + if isinstance(obj, str) and len(obj.split('.')) == 2: + app_name, model_name = obj.split('.') + resolved_model = apps.get_model(app_name, model_name) + if resolved_model is None: + msg = "Django did not return a model for {0}.{1}" + raise ImproperlyConfigured(msg.format(app_name, model_name)) + return resolved_model + elif inspect.isclass(obj) and issubclass(obj, models.Model): + return obj + raise ValueError("{0} is not a Django model".format(obj)) + + +def get_related_model(field): + if django.VERSION < (1, 9): + return _resolve_model(field.rel.to) + return field.remote_field.model + + +# field.rel is deprecated from 1.9 onwards +def get_remote_field(field, **kwargs): + if 'default' in kwargs: + if django.VERSION < (1, 9): + return getattr(field, 'rel', kwargs['default']) + return getattr(field, 'remote_field', kwargs['default']) + + if django.VERSION < (1, 9): + return field.rel + return field.remote_field From 0c73a2203f4e914400786f1143e5da360ce6a4c3 Mon Sep 17 00:00:00 2001 From: Valeryi Savich Date: Tue, 25 Oct 2016 20:02:40 +0300 Subject: [PATCH 3/4] Ported field_mapping module --- aiorest_ws/db/orm/django/field_mapping.py | 309 ++++++++++++++++++++++ aiorest_ws/db/orm/django/model_meta.py | 1 + aiorest_ws/db/orm/django/validators.py | 89 +++++++ 3 files changed, 399 insertions(+) create mode 100644 aiorest_ws/db/orm/django/field_mapping.py create mode 100644 aiorest_ws/db/orm/django/validators.py diff --git a/aiorest_ws/db/orm/django/field_mapping.py b/aiorest_ws/db/orm/django/field_mapping.py new file mode 100644 index 0000000..3bb4473 --- /dev/null +++ b/aiorest_ws/db/orm/django/field_mapping.py @@ -0,0 +1,309 @@ +# -*- coding: utf-8 -*- +""" +Helper functions for mapping model fields to a dictionary of default +keyword arguments that should be used for their equivalent serializer fields. +""" +import inspect + +from django.core import validators +from django.db import models +from django.utils.text import capfirst + +from aiorest_ws.db.orm.django.compat import DecimalValidator +from aiorest_ws.db.orm.django.validators import UniqueValidator + + +__all__ = [ + 'NUMERIC_FIELD_TYPES', 'ClassLookupDict', 'needs_label', + 'get_detail_view_name', 'get_field_kwargs', 'get_relation_kwargs', + 'get_nested_relation_kwargs', 'get_url_kwargs' +] + + +NUMERIC_FIELD_TYPES = ( + models.IntegerField, models.FloatField, models.DecimalField +) + + +class ClassLookupDict(object): + """ + Takes a dictionary with classes as keys. + Lookups against this object will traverses the object's inheritance + hierarchy in method resolution order, and returns the first matching value + from the dictionary or raises a KeyError if nothing matches. + """ + def __init__(self, mapping): + self.mapping = mapping + + def __getitem__(self, key): + if hasattr(key, '_proxy_class'): + # Deal with proxy classes. Ie. BoundField behaves as if it + # is a Field instance when using ClassLookupDict. + base_class = key._proxy_class + else: + base_class = key.__class__ + + for cls in inspect.getmro(base_class): + if cls in self.mapping: + return self.mapping[cls] + raise KeyError('Class %s not found in lookup.' % base_class.__name__) + + def __setitem__(self, key, value): + self.mapping[key] = value + + +def needs_label(model_field, field_name): + """ + Returns `True` if the label based on the model's verbose name + is not equal to the default label it would have based on it's field name. + """ + default_label = field_name.replace('_', ' ').capitalize() + return capfirst(model_field.verbose_name) != default_label + + +def get_detail_view_name(model): + """ + Given a model class, return the view name to use for URL relationships + that refer to instances of the model. + """ + return '%(model_name)s-detail' % { + 'app_label': model._meta.app_label, + 'model_name': model._meta.object_name.lower() + } + + +def get_field_kwargs(field_name, model_field): + """ + Creates a default instance of a basic non-relational field. + """ + kwargs = {} + validator_kwarg = list(model_field.validators) + + # The following will only be used by ModelField classes. + # Gets removed for everything else. + kwargs['model_field'] = model_field + + if model_field.verbose_name and needs_label(model_field, field_name): + kwargs['label'] = capfirst(model_field.verbose_name) + + if model_field.help_text: + kwargs['help_text'] = model_field.help_text + + max_digits = getattr(model_field, 'max_digits', None) + if max_digits is not None: + kwargs['max_digits'] = max_digits + + decimal_places = getattr(model_field, 'decimal_places', None) + if decimal_places is not None: + kwargs['decimal_places'] = decimal_places + + if isinstance(model_field, models.AutoField) or not model_field.editable: + # If this field is read-only, then return early. + # Further keyword arguments are not valid. + kwargs['read_only'] = True + return kwargs + + if model_field.has_default() or model_field.blank or model_field.null: + kwargs['required'] = False + + is_nullable_field = not isinstance(model_field, models.NullBooleanField) + if model_field.null and is_nullable_field: + kwargs['allow_null'] = True + + if model_field.blank and (isinstance(model_field, models.CharField) or + isinstance(model_field, models.TextField)): + kwargs['allow_blank'] = True + + if isinstance(model_field, models.FilePathField): + kwargs['path'] = model_field.path + + if model_field.match is not None: + kwargs['match'] = model_field.match + + if model_field.recursive is not False: + kwargs['recursive'] = model_field.recursive + + if model_field.allow_files is not True: + kwargs['allow_files'] = model_field.allow_files + + if model_field.allow_folders is not False: + kwargs['allow_folders'] = model_field.allow_folders + + if model_field.choices: + # If this model field contains choices, then return early. + # Further keyword arguments are not valid. + kwargs['choices'] = model_field.choices + return kwargs + + # Our decimal validation is handled in the field code, not validator code. + # (In Django 1.9+ this differs from previous style) + if isinstance(model_field, models.DecimalField) and DecimalValidator: + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, DecimalValidator) + ] + + # Ensure that max_length is passed explicitly as a keyword arg, + # rather than as a validator. + max_length = getattr(model_field, 'max_length', None) + if max_length is not None and (isinstance(model_field, models.CharField) or + isinstance(model_field, models.TextField)): + kwargs['max_length'] = max_length + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, validators.MaxLengthValidator) + ] + + # Ensure that min_length is passed explicitly as a keyword arg, + # rather than as a validator. + min_length = next(( + validator.limit_value for validator in validator_kwarg + if isinstance(validator, validators.MinLengthValidator) + ), None) + if min_length is not None and isinstance(model_field, models.CharField): + kwargs['min_length'] = min_length + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, validators.MinLengthValidator) + ] + + # Ensure that max_value is passed explicitly as a keyword arg, + # rather than as a validator. + max_value = next(( + validator.limit_value for validator in validator_kwarg + if isinstance(validator, validators.MaxValueValidator) + ), None) + if max_value is not None and isinstance(model_field, NUMERIC_FIELD_TYPES): + kwargs['max_value'] = max_value + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, validators.MaxValueValidator) + ] + + # Ensure that max_value is passed explicitly as a keyword arg, + # rather than as a validator. + min_value = next(( + validator.limit_value for validator in validator_kwarg + if isinstance(validator, validators.MinValueValidator) + ), None) + if min_value is not None and isinstance(model_field, NUMERIC_FIELD_TYPES): + kwargs['min_value'] = min_value + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, validators.MinValueValidator) + ] + + # URLField does not need to include the URLValidator argument, + # as it is explicitly added in. + if isinstance(model_field, models.URLField): + validator_kwarg = [ + validator for validator in validator_kwarg + if not isinstance(validator, validators.URLValidator) + ] + + # EmailField does not need to include the validate_email argument, + # as it is explicitly added in. + if isinstance(model_field, models.EmailField): + validator_kwarg = [ + validator for validator in validator_kwarg + if validator is not validators.validate_email + ] + + # SlugField do not need to include the 'validate_slug' argument + if isinstance(model_field, models.SlugField): + validator_kwarg = [ + validator for validator in validator_kwarg + if validator is not validators.validate_slug + ] + + # for IPAddressField exclude the 'validate_ipv46_address' argument + if isinstance(model_field, models.GenericIPAddressField): + validator_kwarg = [ + validator for validator in validator_kwarg + if validator is not validators.validate_ipv46_address + ] + + if getattr(model_field, 'unique', False): + unique_error_message = model_field.error_messages.get('unique', None) + if unique_error_message: + unique_error_message = unique_error_message % { + 'model_name': model_field.model._meta.verbose_name, + 'field_label': model_field.verbose_name + } + validator = UniqueValidator( + queryset=model_field.model._default_manager, + message=unique_error_message) + validator_kwarg.append(validator) + + if validator_kwarg: + kwargs['validators'] = validator_kwarg + + return kwargs + + +def get_relation_kwargs(field_name, relation_info): + """ + Creates a default instance of a flat relational field. + """ + model_field = relation_info.model_field + related_model = relation_info.related_model + to_many = relation_info.to_many + to_field = relation_info.to_field + has_through_model = relation_info.has_through_model + + kwargs = { + 'queryset': related_model._default_manager, + 'view_name': get_detail_view_name(related_model) + } + + if to_many: + kwargs['many'] = True + + if to_field: + kwargs['to_field'] = to_field + + if has_through_model: + kwargs['read_only'] = True + kwargs.pop('queryset', None) + + if model_field: + if model_field.verbose_name and needs_label(model_field, field_name): + kwargs['label'] = capfirst(model_field.verbose_name) + help_text = model_field.help_text + if help_text: + kwargs['help_text'] = help_text + if not model_field.editable: + kwargs['read_only'] = True + kwargs.pop('queryset', None) + if kwargs.get('read_only', False): + # If this field is read-only, then return early. + # No further keyword arguments are valid. + return kwargs + + if model_field.has_default() or model_field.blank or model_field.null: + kwargs['required'] = False + if model_field.null: + kwargs['allow_null'] = True + if model_field.validators: + kwargs['validators'] = model_field.validators + if getattr(model_field, 'unique', False): + queryset = model_field.model._default_manager + validator = UniqueValidator(queryset=queryset) + kwargs['validators'] = kwargs.get('validators', []) + [validator] + if to_many and not model_field.blank: + kwargs['allow_empty'] = False + + return kwargs + + +def get_nested_relation_kwargs(relation_info): + kwargs = {'read_only': True} + if relation_info.to_many: + kwargs['many'] = True + return kwargs + + +def get_url_kwargs(model_field): + return { + 'view_name': get_detail_view_name(model_field) + } diff --git a/aiorest_ws/db/orm/django/model_meta.py b/aiorest_ws/db/orm/django/model_meta.py index 5bc5a9d..00c9145 100644 --- a/aiorest_ws/db/orm/django/model_meta.py +++ b/aiorest_ws/db/orm/django/model_meta.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Helper function for returning the field information that is associated with a model class. This includes returning all the forward and reverse diff --git a/aiorest_ws/db/orm/django/validators.py b/aiorest_ws/db/orm/django/validators.py new file mode 100644 index 0000000..9a3d3e0 --- /dev/null +++ b/aiorest_ws/db/orm/django/validators.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" +We perform uniqueness checks explicitly on the serializer class, rather +the using Django's `.full_clean()`. + +This gives us better separation of concerns, allows us to use single-step +object creation, and makes it possible to switch between using the implicit +`ModelSerializer` class and an equivalent explicit `Serializer` class. +""" +from django.db import DataError +from django.utils.translation import ugettext_lazy as _ + +from aiorest_ws.db.orm.exceptions import ValidationError +from aiorest_ws.utils.representation import smart_repr + +__all__ = [ + 'qs_exists', 'qs_filter', 'UniqueValidator' +] + + +# Robust filter and exist implementations. Ensures that queryset.exists() for +# an invalid value returns `False`, rather than raising an error. +# Refs https://github.com/tomchristie/django-rest-framework/issues/3381 + +def qs_exists(queryset): + try: + return queryset.exists() + except (TypeError, ValueError, DataError): + return False + + +def qs_filter(queryset, **kwargs): + try: + return queryset.filter(**kwargs) + except (TypeError, ValueError, DataError): + return queryset.none() + + +class UniqueValidator(object): + """ + Validator that corresponds to `unique=True` on a model field. + Should be applied to an individual field on the serializer. + """ + message = _('This field must be unique.') + + def __init__(self, queryset, message=None): + self.queryset = queryset + self.serializer_field = None + self.message = message or self.message + + def set_context(self, serializer_field): + """ + This hook is called by the serializer instance, + prior to the validation call being made. + """ + # Determine the underlying model field name. This may not be the + # same as the serializer field name if `source=<>` is set. + self.field_name = serializer_field.source_attrs[-1] + # Determine the existing instance, if this is an update operation. + self.instance = getattr(serializer_field.parent, 'instance', None) + + def filter_queryset(self, value, queryset): + """ + Filter the queryset to all instances matching the given attribute. + """ + filter_kwargs = {self.field_name: value} + return qs_filter(queryset, **filter_kwargs) + + def exclude_current_instance(self, queryset): + """ + If an instance is being updated, then do not include + that instance itself as a uniqueness conflict. + """ + if self.instance is not None: + return queryset.exclude(pk=self.instance.pk) + return queryset + + def __call__(self, value): + queryset = self.queryset + queryset = self.filter_queryset(value, queryset) + queryset = self.exclude_current_instance(queryset) + if qs_exists(queryset): + raise ValidationError(self.message) + + def __repr__(self): + return repr('<%s(queryset=%s)>' % ( + self.__class__.__name__, + smart_repr(self.queryset) + )) From f68b1f2e405b800fd1c7df39401cc3b7489d9328 Mon Sep 17 00:00:00 2001 From: Valeryi Savich Date: Wed, 26 Oct 2016 22:19:34 +0300 Subject: [PATCH 4/4] Ported fields module --- aiorest_ws/conf/global_settings.py | 1 + aiorest_ws/db/orm/django/compat.py | 9 +- aiorest_ws/db/orm/django/fields.py | 394 +++++++++++++++++++++++++++++ aiorest_ws/db/orm/fields.py | 19 +- aiorest_ws/utils/date/dateparse.py | 46 +++- 5 files changed, 456 insertions(+), 13 deletions(-) create mode 100644 aiorest_ws/db/orm/django/fields.py diff --git a/aiorest_ws/conf/global_settings.py b/aiorest_ws/conf/global_settings.py index bd32280..32c7886 100644 --- a/aiorest_ws/conf/global_settings.py +++ b/aiorest_ws/conf/global_settings.py @@ -32,6 +32,7 @@ UNICODE_JSON = True COMPACT_JSON = True COERCE_DECIMAL_TO_STRING = True +UPLOADED_FILES_USE_URL = True # ----------------------------------------------- # Database diff --git a/aiorest_ws/db/orm/django/compat.py b/aiorest_ws/db/orm/django/compat.py index 80ad400..0831854 100644 --- a/aiorest_ws/db/orm/django/compat.py +++ b/aiorest_ws/db/orm/django/compat.py @@ -13,7 +13,8 @@ __all__ = [ - '_resolve_model', 'get_related_model', 'get_remote_field' + '_resolve_model', 'get_related_model', 'get_remote_field', + 'value_from_object' ] @@ -62,3 +63,9 @@ def get_remote_field(field, **kwargs): if django.VERSION < (1, 9): return field.rel return field.remote_field + + +def value_from_object(field, obj): + if django.VERSION < (1, 9): + return field._get_val_from_obj(obj) + return field.value_from_object(obj) diff --git a/aiorest_ws/db/orm/django/fields.py b/aiorest_ws/db/orm/django/fields.py new file mode 100644 index 0000000..95e27a2 --- /dev/null +++ b/aiorest_ws/db/orm/django/fields.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- +""" +Field entities, implemented for support Django ORM. + +Every class, represented here, is associated with one certain field type of +table relatively to Django ORM. Each of them field also used later for +serializing/deserializing object of ORM. +""" +import datetime +import re +import uuid + +from aiorest_ws.conf import settings +from aiorest_ws.db.orm import fields +from aiorest_ws.db.orm.django.compat import get_remote_field, \ + value_from_object +from aiorest_ws.db.orm.fields import empty +from aiorest_ws.db.orm.validators import MaxLengthValidator +from aiorest_ws.utils.date.dateparse import parse_duration + +from django.forms import FilePathField as DjangoFilePathField +from django.forms import ImageField as DjangoImageField +from django.core.exceptions import ValidationError as DjangoValidationError +from django.core.validators import EmailValidator, RegexValidator, \ + URLValidator, ip_address_validators +from django.utils import six +from django.utils.duration import duration_string +from django.utils.encoding import is_protected_type +from django.utils.ipv6 import clean_ipv6_address + +__all__ = ( + +) + + +class IntegerField(fields.IntegerField): + pass + + +class BooleanField(fields.BooleanField): + pass + + +class CharField(fields.CharField): + pass + + +class ChoiceField(fields.ChoiceField): + pass + + +class MultipleChoiceField(ChoiceField): + default_error_messages = { + 'invalid_choice': u'"{input}" is not a valid choice.', + 'not_a_list': u'Expected a list of items but got type "{input_type}".', + 'empty': u'This selection may not be empty.' + } + + def __init__(self, *args, **kwargs): + self.allow_empty = kwargs.pop('allow_empty', True) + super(MultipleChoiceField, self).__init__(*args, **kwargs) + + def get_value(self, dictionary): + if self.field_name not in dictionary: + if getattr(self.root, 'partial', False): + return empty + return dictionary.get(self.field_name, empty) + + def to_internal_value(self, data): + if isinstance(data, type('')) or not hasattr(data, '__iter__'): + self.raise_error('not_a_list', input_type=type(data).__name__) + if not self.allow_empty and len(data) == 0: + self.raise_error('empty') + + return { + super(MultipleChoiceField, self).to_internal_value(item) + for item in data + } + + def to_representation(self, value): + return { + self.choice_strings_to_values.get(str(item), item) + for item in value + } + + +class FloatField(fields.FloatField): + pass + + +class NullBooleanField(fields.NullBooleanField): + pass + + +class DecimalField(fields.DecimalField): + pass + + +class TimeField(fields.TimeField): + pass + + +class DateField(fields.DateField): + pass + + +class DateTimeField(fields.DateTimeField): + pass + + +class DurationField(fields.AbstractField): + default_error_messages = { + 'invalid': u"Duration has wrong format. Use one of these formats " + u"instead: {format}.", + } + + def to_internal_value(self, value): + if isinstance(value, datetime.timedelta): + return value + parsed = parse_duration(str(value)) + if parsed is not None: + return parsed + self.raise_error('invalid', format='[DD] [HH:[MM:]]ss[.uuuuuu]') + + def to_representation(self, value): + return duration_string(value) + + +class ListField(fields.ListField): + pass + + +class DictField(fields.DictField): + pass + + +class HStoreField(fields.HStoreField): + pass + + +class JSONField(fields.JSONField): + pass + + +class ModelField(fields.ModelField): + default_error_messages = { + 'max_length': u'Ensure this field has no more than {max_length} ' + u'characters.' + } + + def __init__(self, model_field, **kwargs): + self.model_field = model_field + # The `max_length` option is supported by Django's base `Field` class, + # so we'd better support it here. + max_length = kwargs.pop('max_length', None) + super(ModelField, self).__init__(**kwargs) + if max_length is not None: + message = self.error_messages['max_length'].format( + max_length=max_length + ) + self.validators.append( + MaxLengthValidator(max_length, message=message) + ) + + def to_internal_value(self, data): + rel = get_remote_field(self.model_field, default=None) + if rel is not None: + return rel.to._meta.get_field(rel.field_name).to_python(data) + return self.model_field.to_python(data) + + def to_representation(self, obj): + value = value_from_object(self.model_field, obj) + if is_protected_type(value): + return value + return self.model_field.value_to_string(obj) + + +class ReadOnlyField(fields.ReadOnlyField): + pass + + +class SerializerMethodField(fields.SerializerMethodField): + pass + + +class EmailField(CharField): + default_error_messages = { + "invalid": u"Enter a valid email address." + } + + def __init__(self, **kwargs): + super(EmailField, self).__init__(**kwargs) + validator = EmailValidator(message=self.error_messages['invalid']) + self.validators.append(validator) + + +class RegexField(CharField): + default_error_messages = { + 'invalid': u"This value does not match the required pattern." + } + + def __init__(self, regex, **kwargs): + super(RegexField, self).__init__(**kwargs) + validator = RegexValidator( + regex, message=self.error_messages['invalid'] + ) + self.validators.append(validator) + + +class SlugField(CharField): + default_error_messages = { + 'invalid': u'Enter a valid "slug" consisting of letters, numbers, ' + u'underscores or hyphens.' + } + + def __init__(self, **kwargs): + super(SlugField, self).__init__(**kwargs) + slug_regex = re.compile(r'^[-a-zA-Z0-9_]+$') + validator = RegexValidator( + slug_regex, message=self.error_messages['invalid'] + ) + self.validators.append(validator) + + +class URLField(CharField): + default_error_messages = { + 'invalid': u"Enter a valid URL." + } + + def __init__(self, **kwargs): + super(URLField, self).__init__(**kwargs) + validator = URLValidator(message=self.error_messages['invalid']) + self.validators.append(validator) + + +class UUIDField(fields.AbstractField): + valid_formats = ('hex_verbose', 'hex', 'int', 'urn') + + default_error_messages = { + 'invalid': u'"{value}" is not a valid UUID.' + } + + def __init__(self, **kwargs): + self.uuid_format = kwargs.pop('format', 'hex_verbose') + if self.uuid_format not in self.valid_formats: + raise ValueError( + 'Invalid format for uuid representation. ' + 'Must be one of "{0}"'.format('", "'.join(self.valid_formats)) + ) + super(UUIDField, self).__init__(**kwargs) + + def to_internal_value(self, data): + if not isinstance(data, uuid.UUID): + try: + if isinstance(data, six.integer_types): + return uuid.UUID(int=data) + elif isinstance(data, six.string_types): + return uuid.UUID(hex=data) + else: + self.raise_error('invalid', value=data) + except (ValueError): + self.raise_error('invalid', value=data) + return data + + def to_representation(self, value): + if self.uuid_format == 'hex_verbose': + return str(value) + else: + return getattr(value, self.uuid_format) + + +class IPAddressField(CharField): + """Support both IPAddressField and GenericIPAddressField""" + + default_error_messages = { + 'invalid': u"Enter a valid IPv4 or IPv6 address." + } + + def __init__(self, protocol='both', **kwargs): + self.protocol = protocol.lower() + self.unpack_ipv4 = (self.protocol == 'both') + super(IPAddressField, self).__init__(**kwargs) + validators, error_message = ip_address_validators( + protocol, self.unpack_ipv4 + ) + self.validators.extend(validators) + + def to_internal_value(self, data): + if not isinstance(data, six.string_types): + self.raise_error('invalid', value=data) + + if ':' in data: + try: + if self.protocol in ('both', 'ipv6'): + return clean_ipv6_address(data, self.unpack_ipv4) + except DjangoValidationError: + self.raise_error('invalid', value=data) + + return super(IPAddressField, self).to_internal_value(data) + + +class FilePathField(ChoiceField): + default_error_messages = { + 'invalid_choice': u'"{input}" is not a valid path choice.' + } + + def __init__(self, path, match=None, recursive=False, allow_files=True, + allow_folders=False, required=None, **kwargs): + # Defer to Django's FilePathField implementation to get the + # valid set of choices. + field = DjangoFilePathField( + path, match=match, recursive=recursive, allow_files=allow_files, + allow_folders=allow_folders, required=required + ) + kwargs['choices'] = field.choices + super(FilePathField, self).__init__(**kwargs) + + +class FileField(fields.AbstractField): + default_error_messages = { + 'required': u'No file was submitted.', + 'invalid': u'The submitted data was not a file. Check the encoding ' + u'type on the form.', + 'no_name': u'No filename could be determined.', + 'empty': u'The submitted file is empty.', + 'max_length': u'Ensure this filename has at most {max_length} ' + u'characters (it has {length}).', + } + + def __init__(self, *args, **kwargs): + self.max_length = kwargs.pop('max_length', None) + self.allow_empty_file = kwargs.pop('allow_empty_file', False) + if 'use_url' in kwargs: + self.use_url = kwargs.pop('use_url') + super(FileField, self).__init__(*args, **kwargs) + + def to_internal_value(self, data): + try: + # `UploadedFile` objects should have name and size attributes. + file_name = data.name + file_size = data.size + except AttributeError: + self.raise_error('invalid') + + if not file_name: + self.raise_error('no_name') + if not self.allow_empty_file and not file_size: + self.raise_error('empty') + if self.max_length and len(file_name) > self.max_length: + self.raise_error( + 'max_length', max_length=self.max_length, length=len(file_name) + ) + + return data + + def to_representation(self, value): + use_url = getattr(self, 'use_url', settings.UPLOADED_FILES_USE_URL) + + if not value: + return None + + if use_url: + if not getattr(value, 'url', None): + # If the file has not been saved it may not have a URL. + return None + url = value.url + request = self.context.get('request', None) + if request is not None: + return request.build_absolute_uri(url) + return url + return value.name + + +class ImageField(FileField): + default_error_messages = { + 'invalid_image': u'Upload a valid image. The file you uploaded was ' + u'either not an image or a corrupted image.' + } + + def __init__(self, *args, **kwargs): + self._DjangoImageField = kwargs.pop( + '_DjangoImageField', DjangoImageField + ) + super(ImageField, self).__init__(*args, **kwargs) + + def to_internal_value(self, data): + # Image validation is a bit grungy, so we'll just outright + # defer to Django's implementation so we don't need to + # consider it, or treat PIL as a test dependency. + file_object = super(ImageField, self).to_internal_value(data) + django_field = self._DjangoImageField() + django_field.error_messages = self.error_messages + django_field.to_python(file_object) + return file_object diff --git a/aiorest_ws/db/orm/fields.py b/aiorest_ws/db/orm/fields.py index 8078287..2332083 100644 --- a/aiorest_ws/db/orm/fields.py +++ b/aiorest_ws/db/orm/fields.py @@ -221,7 +221,7 @@ def __init__(self, choices, **kwargs): # Allows us to deal with eg. integer choices while supporting either # integer or string input, but still get the correct datatype out. self.choice_strings_to_values = { - key: key for key in self.choices.keys() + str(key): key for key in self.choices.keys() } self.allow_blank = kwargs.pop('allow_blank', False) @@ -232,14 +232,14 @@ def to_internal_value(self, data): return '' try: - return self.choice_strings_to_values[data] + return self.choice_strings_to_values[str(data)] except KeyError: self.raise_error('invalid_choice', input=data) def to_representation(self, value): if value in ('', None): return value - return self.choice_strings_to_values.get(value, value) + return self.choice_strings_to_values.get(str(value), value) class FloatField(AbstractField): @@ -836,7 +836,10 @@ def to_representation(self, data): """ List of object instances -> List of dicts of primitive datatypes. """ - return [self.child.to_representation(item) for item in data] + return [ + self.child.to_representation(item) if item is not None else None + for item in data + ] class DictField(AbstractField): @@ -880,7 +883,7 @@ def to_representation(self, value): List of object instances -> List of dicts of primitive datatypes. """ return { - str(key): self.child.to_representation(val) + str(key): self.child.to_representation(val) if val is not None else None # NOQA for key, val in value.items() } @@ -927,11 +930,6 @@ class ModelField(AbstractField): This is used by `ModelSerializer` when dealing with custom model fields, that do not have a serializer field to be mapped to. """ - default_error_messages = { - 'max_length': "Ensure this field has no more than {max_length} " - "characters." - } - def __init__(self, model_field, **kwargs): self.model_field = model_field super(ModelField, self).__init__(**kwargs) @@ -981,6 +979,7 @@ def bind(self, field_name, parent): # The method name should default to `get_{field_name}`. if self.method_name is None: self.method_name = default_method_name + super(SerializerMethodField, self).bind(field_name, parent) def to_representation(self, value): diff --git a/aiorest_ws/utils/date/dateparse.py b/aiorest_ws/utils/date/dateparse.py index c6f55ac..3404966 100644 --- a/aiorest_ws/utils/date/dateparse.py +++ b/aiorest_ws/utils/date/dateparse.py @@ -12,8 +12,9 @@ from aiorest_ws.utils.date.timezone import utc, get_fixed_timezone __all__ = [ - 'date_re', 'time_re', 'datetime_re', - 'parse_date', 'parse_time', 'parse_datetime', 'parse_timedelta' + 'date_re', 'time_re', 'datetime_re', 'standard_duration_re', + 'iso8601_duration_re', 'parse_date', 'parse_time', 'parse_datetime', + 'parse_timedelta' ] date_re = re.compile( @@ -32,6 +33,30 @@ r'(?PZ|[+-]\d{2}(?::?\d{2})?)?$' ) +standard_duration_re = re.compile( + r'^' + r'(?:(?P-?\d+) (days?, )?)?' + r'((?:(?P\d+):)(?=\d+:\d+))?' + r'(?:(?P\d+):)?' + r'(?P\d+)' + r'(?:\.(?P\d{1,6})\d{0,6})?' + r'$' +) + +# Support the sections of ISO 8601 date representation that are accepted by +# timedelta +iso8601_duration_re = re.compile( + r'^(?P[-+]?)' + r'P' + r'(?:(?P\d+(.\d+)?)D)?' + r'(?:T' + r'(?:(?P\d+(.\d+)?)H)?' + r'(?:(?P\d+(.\d+)?)M)?' + r'(?:(?P\d+(.\d+)?)S)?' + r')?' + r'$' +) + def parse_date(value): """Parses a string and return a datetime.date. @@ -127,3 +152,20 @@ def parse_timedelta(value): d = d.groupdict(0) return datetime.timedelta(**dict(((k, float(v)) for k, v in d.items()))) + + +def parse_duration(value): + """Parses a duration string and returns a datetime.timedelta. + The preferred format for durations in Django is '%d %H:%M:%S.%f'. + Also supports ISO 8601 representation. + """ + match = standard_duration_re.match(value) + if not match: + match = iso8601_duration_re.match(value) + if match: + kw = match.groupdict() + sign = -1 if kw.pop('sign', '+') == '-' else 1 + if kw.get('microseconds'): + kw['microseconds'] = kw['microseconds'].ljust(6, '0') + kw = {k: float(v) for k, v in iter(kw) if v is not None} + return sign * datetime.timedelta(**kw)