Fix #208: Upgrade Django to v1.11

This updates Django to v1.11.10 and and all updates related supporting
requirements and introduces some code changes required to keep things
running smoothly

- django-filters 1.x required updates to all API filter classes
  - Implemented new filter class for Attribute model API viewset,
    replacing `AttributeViewSet.filter_fields` with `.filter_class`
  - All existing `MethodFilter` fields had to be converted to typed
    fields that specified `method=` as an argument
  - All `.filter_foo()` methods had to have a  new `name` argument added
    to their signature
- Custom database model fields have been updated to include the new
  `.from_db_value()` method which differs only slightly from the
  `.to_python()` method for each custom field.
- The `nsot-server upgrade` command no longer calls `syncdb` (which has
  been deprecated) underneath has been updated to call `migrate`
  instead.
- All places where `django.utils.log.getLogger` was being called has
  been replaced with `logging.getLogger`
- Calls to `django.core.handlers.wsgi.STATUS_CODE_TEXT` have been
  replaced with `httplib.responses`
- django-macaddress required all MAC address objects to conform to
  EUIv48 format: `settings.INTERFACE_DEFAULT_MAC` has been explicitly
  set to use EUIv48 format
- `nsot.validators.validate_mac_address()` no longer utilizes
  `macaddress.formfields.MacAddressField.clean()` as a validator and
  instead directly instantitates a `netaddr.EUI()` object to assert
  validity.
- Because pytest-django had to be updated, so did pytest, and new test
  warnings were occuring when `tests.api_tests.fixtures.TestSite` was
  used. It has been renamed to `SiteHelper` to avoid the warnings.

nsot/api/filters.py

@@ -4,18 +4,28 @@
 from django.db.models import Q
 import django_filters
 
-from .. import fields, models
+from .. import models
 from ..util import qpbool
 
 
 log = logging.getLogger(__name__)
 
 
+class AttributeFilter(django_filters.rest_framework.FilterSet):
+    required = django_filters.BooleanFilter()
+    display = django_filters.BooleanFilter()
+    multi = django_filters.BooleanFilter()
+
+    class Meta:
+        model = models.Attribute
+        fields = ['name', 'resource_name', 'required', 'display', 'multi']
+
+
 class ResourceFilter(django_filters.rest_framework.FilterSet):
     """Attribute-aware filtering for Resource objects."""
-    attributes = django_filters.MethodFilter()
+    attributes = django_filters.CharFilter(method='filter_attributes')
 
-    def filter_attributes(self, queryset, value):
+    def filter_attributes(self, queryset, name, value):
         """
         Reads 'attributes' from query params and joins them together as an
         intersection set query.
@@ -50,17 +60,13 @@ class Meta:
 
 class NetworkFilter(ResourceFilter):
     """Filter for Network objects."""
-    include_networks = django_filters.MethodFilter()
-    include_ips = django_filters.MethodFilter()
-    cidr = django_filters.MethodFilter()
-    root_only = django_filters.MethodFilter()
-
-    # Field override for the `network_address` field.
-    filter_overrides = {
-        fields.BinaryIPAddressField: {
-            'filter_class': django_filters.CharFilter,
-        },
-    }
+    include_networks = django_filters.BooleanFilter(
+        method='filter_include_networks'
+    )
+    include_ips = django_filters.BooleanFilter(method='filter_include_ips')
+    cidr = django_filters.CharFilter(method='filter_cidr')
+    root_only = django_filters.BooleanFilter(method='filter_root_only')
+    network_address = django_filters.CharFilter()  # Override type
 
     class Meta:
         model = models.Network
@@ -70,7 +76,7 @@ class Meta:
             'attributes'
         ]
 
-    def filter_include_networks(self, queryset, value):
+    def filter_include_networks(self, queryset, name, value):
         """Converts ``include_networks`` to queryset filters."""
         include_ips = qpbool(self.form.cleaned_data['include_ips'])
         include_networks = qpbool(value)
@@ -83,7 +89,7 @@ def filter_include_networks(self, queryset, value):
 
         return queryset
 
-    def filter_include_ips(self, queryset, value):
+    def filter_include_ips(self, queryset, name, value):
         """Converts ``include_ips`` to queryset filters."""
         include_ips = qpbool(value)
         include_networks = qpbool(self.form.cleaned_data['include_networks'])
@@ -96,7 +102,7 @@ def filter_include_ips(self, queryset, value):
 
         return queryset
 
-    def filter_cidr(self, queryset, value):
+    def filter_cidr(self, queryset, name, value):
         """Converts ``cidr`` to network/prefix filter."""
         if value:
             network_address, _, prefix_length = value.partition('/')
@@ -108,7 +114,7 @@ def filter_cidr(self, queryset, value):
             prefix_length=prefix_length
         )
 
-    def filter_root_only(self, queryset, value):
+    def filter_root_only(self, queryset, name, value):
         """Converts ``root_only`` to null parent filter."""
         if qpbool(value):
             return queryset.filter(parent=None)
@@ -122,13 +128,7 @@ class InterfaceFilter(ResourceFilter):
     Includes a custom override for filtering on mac_address because this is not
     a Django built-in field.
     """
-
-    # Field override for the `mac_address` field.
-    filter_overrides = {
-        fields.MACAddressField: {
-            'filter_class': django_filters.MethodFilter,
-        },
-    }
+    mac_address = django_filters.CharFilter(method='filter_mac_address')
 
     class Meta:
         model = models.Interface
@@ -140,7 +140,7 @@ class Meta:
             'device_hostname'
         ]
 
-    def filter_mac_address(self, queryset, value):
+    def filter_mac_address(self, queryset, name, value):
         """
         Overloads queryset filtering to use built-in.
 
@@ -152,16 +152,16 @@ def filter_mac_address(self, queryset, value):
 
 class CircuitFilter(ResourceFilter):
     """Filter for Circuit objects."""
-    endpoint_a = django_filters.MethodFilter()
-    endpoint_z = django_filters.MethodFilter()
+    endpoint_a = django_filters.CharFilter(method='filter_endpoint_a')
+    endpoint_z = django_filters.CharFilter(method='filter_endpoint_z')
 
     class Meta:
         model = models.Circuit
         fields = ['endpoint_a', 'endpoint_z', 'name', 'attributes']
 
     # FIXME(jathan): The copy/pasted methods can be ripped out once we upgrade
     # filters in support of the V2 API. For now this is quicker and easier.
-    def filter_endpoint_a(self, queryset, value):
+    def filter_endpoint_a(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)
@@ -171,7 +171,7 @@ def filter_endpoint_a(self, queryset, value):
         else:
             return queryset.filter(endpoint_a__name_slug=value)
 
-    def filter_endpoint_z(self, queryset, value):
+    def filter_endpoint_z(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)
@@ -191,16 +191,16 @@ class Meta:
 
 class ProtocolFilter(ResourceFilter):
     """Filter for Protocol objects."""
-    device = django_filters.MethodFilter()
-    type = django_filters.MethodFilter()
-    interface = django_filters.MethodFilter()
-    circuit = django_filters.MethodFilter()
+    device = django_filters.CharFilter(method='filter_device')
+    type = django_filters.CharFilter(method='filter_type')
+    interface = django_filters.CharFilter(method='filter_interface')
+    circuit = django_filters.CharFilter(method='filter_circuit')
 
     class Meta:
         model = models.Protocol
         fields = ['device', 'type', 'interface', 'circuit', 'description']
 
-    def filter_device(self, queryset, value):
+    def filter_device(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)
@@ -210,7 +210,7 @@ def filter_device(self, queryset, value):
         else:
             return queryset.filter(device__hostname=value)
 
-    def filter_type(self, queryset, value):
+    def filter_type(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)
@@ -220,7 +220,7 @@ def filter_type(self, queryset, value):
         else:
             return queryset.filter(type__name=value)
 
-    def filter_interface(self, queryset, value):
+    def filter_interface(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)
@@ -230,7 +230,7 @@ def filter_interface(self, queryset, value):
         else:
             return queryset.filter(interface__name_slug=value)
 
-    def filter_circuit(self, queryset, value):
+    def filter_circuit(self, queryset, name, value):
         """Overload to use natural key."""
         if isinstance(value, int):
             value = str(value)

nsot/api/views.py

@@ -370,7 +370,7 @@ class AttributeViewSet(ResourceViewSet):
     """
     queryset = models.Attribute.objects.all()
     serializer_class = serializers.AttributeSerializer
-    filter_fields = ('name', 'resource_name', 'required', 'display', 'multi')
+    filter_class = filters.AttributeFilter
 
     def get_serializer_class(self):
         if self.request.method == 'POST':

nsot/conf/settings.py

@@ -410,7 +410,9 @@
 INTERFACE_DEFAULT_SPEED = 1000  # In Mbps (e.g. 1Gbps)
 
 # Default MAC address ('00:00:00:00:00:00')
-INTERFACE_DEFAULT_MAC = eui.EUI(0, dialect=macaddress.default_dialect())
+INTERFACE_DEFAULT_MAC = eui.EUI(
+    0, dialect=macaddress.default_dialect(), version=48
+)
 
 # These are mappings to the formal integer types from SNMP IF-MIB::ifType. The
 # types listed here are the most commonly found in the wild.

nsot/fields.py

@@ -43,8 +43,6 @@ def _get_new_connection_tolerant(self, conn_params):
 
 class BinaryIPAddressField(models.Field):
     """IP Address field that stores values as varbinary."""
-    __metaclass__ = models.SubfieldBase
-
     def __init__(self, *args, **kwargs):
         super(BinaryIPAddressField, self).__init__(*args, **kwargs)
         self.editable = True
@@ -60,11 +58,7 @@ def db_type(self, connection):
         data = DictWrapper(self.__dict__, connection.ops.quote_name, "qn_")
         return 'varbinary(%(max_length)s)' % data
 
-    def to_python(self, value):
-        """DB -> Python."""
-        if not value:
-            return value
-
+    def _parse_ip_address(self, value):
         try:
             obj = ipaddress.ip_address(unicode(value))
         except ValueError:
@@ -76,6 +70,23 @@ def to_python(self, value):
 
         return obj.exploded
 
+    def from_db_value(self, value, expression, connection, context):
+        """DB -> Python."""
+        if value is None:
+            return value
+
+        return self._parse_ip_address(value)
+
+    def to_python(self, value):
+        """Object -> Python."""
+        if isinstance(value, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
+            return value
+
+        if value is None:
+            return value
+
+        return self._parse_ip_address(value)
+
     def get_db_prep_value(self, value, connection, prepared=False):
         """Python -> DB."""
         # To account for null defaults when performing migrations
@@ -100,7 +111,23 @@ class MACAddressField(BaseMACAddressField):
     always expect the DRF version, for better consistency in debugging and
     testing.
     """
+    def from_db_value(self, value, expression, connection, context):
+        # If value is an integer that is a string, make it an int
+        if isinstance(value, basestring) and value.isdigit():
+            value = int(value)
+
+        try:
+            return super(MACAddressField, self).from_db_value(
+                value, expression, connection, context
+            )
+        except exc.DjangoValidationError as err:
+            raise exc.ValidationError(err.message)
+
     def to_python(self, value):
+        # If value is an integer that is a string, make it an int
+        if isinstance(value, basestring) and value.isdigit():
+            value = int(value)
+
         try:
             return super(MACAddressField, self).to_python(value)
         except exc.DjangoValidationError as err:

nsot/management/commands/upgrade.py

@@ -22,8 +22,7 @@ def add_arguments(self, parser):
 
     def handle(self, **options):
         call_command(
-            'syncdb',
-            migrate=True,
+            'migrate',
             interactive=(not options['noinput']),
             traceback=options['traceback'],
             verbosity=options['verbosity'],

nsot/middleware/auth.py

@@ -2,18 +2,19 @@
 Middleware for authentication.
 """
 
+import logging
+
 from django.contrib.auth import backends, middleware
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import EmailValidator
-from django.utils.log import getLogger
 from guardian.backends import ObjectPermissionBackend
 from guardian.core import ObjectPermissionChecker
 
 from ..util import normalize_auth_header
 
 
-log = getLogger('nsot_server')
+log = logging.getLogger('nsot_server')
 
 
 class EmailHeaderMiddleware(middleware.RemoteUserMiddleware):
@@ -22,15 +23,18 @@ class EmailHeaderMiddleware(middleware.RemoteUserMiddleware):
 
 class EmailHeaderBackend(backends.RemoteUserBackend):
     """Custom backend that validates username is an email."""
-    def authenticate(self, remote_user):
+    def authenticate(self, request, remote_user):
         """Override default to return None if username is invalid."""
         if not remote_user:
             return
+
         username = self.clean_username(remote_user)
         if not username:
             return
 
-        return super(EmailHeaderBackend, self).authenticate(remote_user)
+        return super(EmailHeaderBackend, self).authenticate(
+            request, remote_user
+        )
 
     def clean_username(self, username):
         """Makes sure that the username is a valid email address."""

nsot/middleware/request_logging.py

@@ -2,14 +2,13 @@
 Middleware to log HTTP requests.
 """
 
+import logging
 from time import time
 
-from django.utils.log import getLogger
-
 
 class LoggingMiddleware(object):
     def __init__(self):
-        self.logger = getLogger('nsot_server')
+        self.logger = logging.getLogger('nsot_server')
 
     def process_request(self, request):
         request.timer = time()

nsot/ui/views.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import unicode_literals
+from httplib import responses
 import logging
 
-from django.core.handlers.wsgi import STATUS_CODE_TEXT as status_text
 from django.shortcuts import render
 from django.views.generic import TemplateView
 
@@ -23,7 +23,7 @@ class FeView(TemplateView):
 
 def render_error(request, status_code, template_name='ui/error.html'):
     """Generic base for rendering error pages."""
-    message = status_text.get(status_code)
+    message = responses[status_code].upper()
     context = {'code': status_code, 'message': message}
     return render(request, template_name, context, status=status_code)