From 9734dc3c7d8e75590f25eb9e97ecb9b9f618a1da Mon Sep 17 00:00:00 2001 From: darius BERNARD Date: Fri, 6 Jan 2017 12:11:09 +0100 Subject: [PATCH] added the check framework integration for remote api models and fields. --- rest_models/backend/__init__.py | 5 +- rest_models/backend/compiler.py | 5 +- rest_models/backend/introspection.py | 5 +- rest_models/checks.py | 81 +++++++++++++++++++++++++++ rest_models/router.py | 10 ++++ rest_models/tests/test_systemcheck.py | 60 ++++++++++++++++++++ rest_models/tests/test_utils.py | 1 + rest_models/tests/tests_routers.py | 2 + testapi/__init__.py | 6 -- testapi/badapi/__init__.py | 0 testapi/badapi/models.py | 51 +++++++++++++++++ testapi/badapi/serializers.py | 39 +++++++++++++ testapi/badapi/urls.py | 19 +++++++ testapi/badapi/viewset.py | 27 +++++++++ testapp/badapp/__init__.py | 0 testapp/badapp/models.py | 51 +++++++++++++++++ testsettings.py | 14 ++++- 17 files changed, 363 insertions(+), 13 deletions(-) create mode 100644 rest_models/checks.py create mode 100644 rest_models/tests/test_systemcheck.py create mode 100644 testapi/badapi/__init__.py create mode 100644 testapi/badapi/models.py create mode 100644 testapi/badapi/serializers.py create mode 100644 testapi/badapi/urls.py create mode 100644 testapi/badapi/viewset.py create mode 100644 testapp/badapp/__init__.py create mode 100644 testapp/badapp/models.py diff --git a/rest_models/backend/__init__.py b/rest_models/backend/__init__.py index 73f4979..1fe9d55 100644 --- a/rest_models/backend/__init__.py +++ b/rest_models/backend/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, absolute_import, print_function -import logging +from rest_models.checks import register_checks -logger = logging.getLogger(__name__) +register_checks() diff --git a/rest_models/backend/compiler.py b/rest_models/backend/compiler.py index ab42115..0839cbf 100644 --- a/rest_models/backend/compiler.py +++ b/rest_models/backend/compiler.py @@ -771,8 +771,9 @@ def raise_on_response(self, url, params, response): if response.status_code == 204: raise EmptyResultSet elif response.status_code != 200: - raise ProgrammingError("the query to the api has failed : GET %s \n=> %s" % - (build_url(url, params), message_from_response(response))) + raise ProgrammingError("the query to the api has failed : GET %s/%s \n=> %s" % + (self.connection.connection.url, build_url(url, params), + message_from_response(response))) def get_meta(self, json, response): """ diff --git a/rest_models/backend/introspection.py b/rest_models/backend/introspection.py index 31ef2a1..ce00d53 100644 --- a/rest_models/backend/introspection.py +++ b/rest_models/backend/introspection.py @@ -39,7 +39,10 @@ def get_table_list(self, cursor): "[%s] %s" % (res.request.url, res.status_code, res.text[:500])) tables = res.json().keys() for table in tables: - option = cursor.options(table).json() + response = cursor.options(table) + if response.status_code != 200: + raise Exception("bad response from api: %s" % response.text) + option = response.json() missing_features = self.features - set(option['features']) if missing_features: raise Exception("the remote api does not provide all required features : %s" % missing_features) diff --git a/rest_models/checks.py b/rest_models/checks.py new file mode 100644 index 0000000..5fe7860 --- /dev/null +++ b/rest_models/checks.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import logging + +from django.apps import apps +from django.core.checks import Error, Tags, register + +from rest_models.backend.compiler import get_resource_path +from rest_models.router import RestModelRouter + +logger = logging.getLogger(__name__) + + +def api_struct_check(app_configs, **kwargs): + errors = [] + + all_models = [] + if app_configs is None: + all_models.extend(apps.get_models()) + else: + for app_config in app_configs: + all_models.extend(app_config.get_models()) + router = RestModelRouter() + models = ((router.get_api_connexion(model).cursor(), model) + for model in all_models if router.is_api_model(model)) + + for db, rest_model in models: + url = get_resource_path(rest_model) + res = db.options(url) + if res.status_code != 200: + errors.append( + Error( + 'the remote api does not respond to us. OPTIONS %s/%s => %s' % (db.url, url, res.status_code), + hint='check the url for the remote api or the resource_path', + obj=rest_model, + id='rest_models.E001' + ) + ) + continue + options = res.json() + missings = { + 'include[]', 'exclude[]', 'filter{}', 'page', 'per_page', 'sort[]' + } - set(options.get("features", [])) + if missings: + errors.append( + Error( + 'the remote api does not support the folowing features: %s' % missings, + hint='is the api on %s/%s running with dynamic-rest ?' % (db.url, url), + obj=rest_model, + id='rest_models.E002' + ) + ) + continue + for field in rest_model._meta.get_fields(): + if field.is_relation: + if router.is_api_model(field.related_model): + if field.name not in options['properties']: + errors.append( + Error( + 'the field %s.%s in not present on the remote serializer' % ( + rest_model.__name__, field.name + ), + obj="%s.%s" % (rest_model.__name__, field.name), + hint='check if the serializer on %s/%s has a field "%s"' % (db.url, url, field.name), + id='rest_models.E003' + ) + ) + elif field.name not in options['properties']: + errors.append( + Error( + 'the field %s.%s in not present on the remote serializer' % (rest_model.__name__, field.name), + hint='check if the serializer on %s/%s has a field "%s"' % (db.url, url, field.name), + id='rest_models.E003' + ) + ) + return errors + + +def register_checks(): + register(api_struct_check, Tags.models) diff --git a/rest_models/router.py b/rest_models/router.py index ea52bd1..f12e27c 100644 --- a/rest_models/router.py +++ b/rest_models/router.py @@ -7,6 +7,7 @@ from django.apps import apps from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from django.db import connections from rest_models.backend.base import DatabaseWrapper @@ -74,6 +75,15 @@ def get_api_database(self, model): self.cache[model] = result return result + def get_api_connexion(self, model): + """ + return the connection to query the api behind this model + :param model: + :return: the api connection + :rtype: rest_models.backend.base.DatabaseWrapper + """ + return connections[self.get_api_database(model)] + def db_for_read(self, model, **hints): return self.get_api_database(model) diff --git a/rest_models/tests/test_systemcheck.py b/rest_models/tests/test_systemcheck.py new file mode 100644 index 0000000..dadd46f --- /dev/null +++ b/rest_models/tests/test_systemcheck.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import logging + +from django.core.management import call_command +from django.core.management.base import SystemCheckError +from django.db.utils import ProgrammingError +from django.test.testcases import TestCase +from django.test.utils import modify_settings, override_settings + +logger = logging.getLogger(__name__) + + +@override_settings(ROOT_URLCONF='testapi.badapi.urls') +@modify_settings(INSTALLED_APPS={ + 'append': ['testapi.badapi', 'testapp.badapp'], +}) +class SystemCheckTest(TestCase): + + fixtures = ['user.json'] + + @classmethod + def setUpClass(cls): + super(SystemCheckTest, cls).setUpClass() + call_command( + 'migrate', + verbosity=0, + interactive=False, + test_flush=True) + + def setUp(self): + from testapp.badapp import models + self.models = models + + def test_query_ok(self): + self.assertEqual(0, self.models.AA.objects.filter(a__name='a').all().count()) + + def test_query_fail(self): + with self.assertRaisesMessage(ProgrammingError, 'Invalid filter field: aa'): + self.assertEqual(0, self.models.A.objects.filter(aa__name='a').all().count()) + + def test_check_all_error(self): + with self.assertRaises(SystemCheckError) as ctx: + call_command('check') + msg = ctx.exception.args[0] + self.assertIn('has a field "aa"', msg) + self.assertIn('has a field "bb"', msg) + self.assertIn('OPTIONS http://localapi/api/v1/c => 404', msg) + + def test_check_one_error(self): + with self.assertRaises(SystemCheckError) as ctx: + call_command('check', 'badapp') + msg = ctx.exception.args[0] + self.assertIn('has a field "aa"', msg) + self.assertIn('has a field "bb"', msg) + self.assertIn('OPTIONS http://localapi/api/v1/c => 404', msg) + + def test_check_one_ok(self): + call_command('check', 'testapp') diff --git a/rest_models/tests/test_utils.py b/rest_models/tests/test_utils.py index 4654323..007ff07 100644 --- a/rest_models/tests/test_utils.py +++ b/rest_models/tests/test_utils.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals + import doctest import os import tempfile diff --git a/rest_models/tests/tests_routers.py b/rest_models/tests/tests_routers.py index db7e63d..963fcaa 100644 --- a/rest_models/tests/tests_routers.py +++ b/rest_models/tests/tests_routers.py @@ -36,6 +36,7 @@ def test_only_one_api_provided(self): DB = copy.deepcopy(settings.DATABASES) del DB['api2'] + del DB['apifail'] router.databases = DB self.assertEqual(router.api_database_name, 'api') @@ -45,6 +46,7 @@ def test_no_api_provided(self): DB = copy.deepcopy(settings.DATABASES) del DB['api2'] del DB['api'] + del DB['apifail'] router.databases = DB self.assertRaises(ImproperlyConfigured, getattr, router, "api_database_name") diff --git a/testapi/__init__.py b/testapi/__init__.py index 73f4979..e69de29 100644 --- a/testapi/__init__.py +++ b/testapi/__init__.py @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals, absolute_import, print_function - -import logging - -logger = logging.getLogger(__name__) diff --git a/testapi/badapi/__init__.py b/testapi/badapi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testapi/badapi/models.py b/testapi/badapi/models.py new file mode 100644 index 0000000..4fb971f --- /dev/null +++ b/testapi/badapi/models.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import logging + +from django.db import models +from django.utils import timezone + +logger = logging.getLogger(__name__) + + +def auto_now_plus_5d(): + return timezone.now() + datetime.timedelta(days=5) + + +class A(models.Model): + name = models.CharField(max_length=135) + + def __str__(self): + return self.name # pragma: no cover + + +class B(models.Model): + name = models.CharField(max_length=135) + + def __str__(self): + return self.name # pragma: no cover + + +class C(models.Model): + name = models.CharField(max_length=135) + + def __str__(self): + return self.name # pragma: no cover + + +class AA(models.Model): + name = models.CharField(max_length=135) + a = models.ForeignKey(A, related_name='aa') + + def __str__(self): + return self.name # pragma: no cover + + +class BB(models.Model): + name = models.CharField(max_length=135) + b = models.ManyToManyField(B, related_name='bb') + + def __str__(self): + return self.name # pragma: no cover diff --git a/testapi/badapi/serializers.py b/testapi/badapi/serializers.py new file mode 100644 index 0000000..5f102ec --- /dev/null +++ b/testapi/badapi/serializers.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +from dynamic_rest.fields.fields import DynamicRelationField +from dynamic_rest.serializers import DynamicModelSerializer + +from testapi.badapi.models import AA, BB, A, B + + +class ASerializer(DynamicModelSerializer): + class Meta: + model = A + name = 'a' + fields = ('id', 'name') + + +class BSerializer(DynamicModelSerializer): + class Meta: + model = B + name = 'b' + fields = ('id', 'name') + + +class AASerializer(DynamicModelSerializer): + a = DynamicRelationField(ASerializer, many=False) + + class Meta: + model = AA + name = 'aa' + fields = ('id', 'name', 'a') + + +class BBSerializer(DynamicModelSerializer): + b = DynamicRelationField(BSerializer, many=True) + + class Meta: + model = BB + name = 'bb' + fields = ('id', 'name', 'b') diff --git a/testapi/badapi/urls.py b/testapi/badapi/urls.py new file mode 100644 index 0000000..1e4a57a --- /dev/null +++ b/testapi/badapi/urls.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +from django.conf.urls import include, url +from dynamic_rest.routers import DynamicRouter + +from .viewset import AAViewSet, AViewSet, BBViewSet, BViewSet + +router = DynamicRouter() +router.register('/a', AViewSet) +router.register('/aa', AAViewSet) +router.register('/b', BViewSet) + +router.register('/bb', BBViewSet) + +urlpatterns = [ + url(r'^api/v1', include(router.urls)), + url(r'', include('testapi.urls')), +] diff --git a/testapi/badapi/viewset.py b/testapi/badapi/viewset.py new file mode 100644 index 0000000..242c4e5 --- /dev/null +++ b/testapi/badapi/viewset.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +from dynamic_rest.viewsets import DynamicModelViewSet + +from .models import AA, BB, A, B +from .serializers import AASerializer, ASerializer, BBSerializer, BSerializer + + +class AViewSet(DynamicModelViewSet): + serializer_class = ASerializer + queryset = A.objects.all() + + +class AAViewSet(DynamicModelViewSet): + serializer_class = AASerializer + queryset = AA.objects.all() + + +class BViewSet(DynamicModelViewSet): + serializer_class = BSerializer + queryset = B.objects.all() + + +class BBViewSet(DynamicModelViewSet): + serializer_class = BBSerializer + queryset = BB.objects.all() diff --git a/testapp/badapp/__init__.py b/testapp/badapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testapp/badapp/models.py b/testapp/badapp/models.py new file mode 100644 index 0000000..9c25fea --- /dev/null +++ b/testapp/badapp/models.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import logging + +from django.db import models +from django.utils import timezone + +logger = logging.getLogger(__name__) + + +def auto_now_plus_5d(): + return timezone.now() + datetime.timedelta(days=5) + + +class A(models.Model): + name = models.CharField(max_length=135) + + class APIMeta: + db_name = 'apifail' + + +class B(models.Model): + name = models.CharField(max_length=135) + + class APIMeta: + db_name = 'apifail' + + +class C(models.Model): + name = models.CharField(max_length=135) + + class APIMeta: + db_name = 'apifail' + + +class AA(models.Model): + name = models.CharField(max_length=135) + a = models.ForeignKey(A, related_name='aa') + + class APIMeta: + db_name = 'apifail' + + +class BB(models.Model): + name = models.CharField(max_length=135) + b = models.ManyToManyField(B, related_name='bb') + + class APIMeta: + db_name = 'apifail' diff --git a/testsettings.py b/testsettings.py index b5139b0..7eb6e36 100644 --- a/testsettings.py +++ b/testsettings.py @@ -1,3 +1,5 @@ +import os + import django.conf.global_settings as DEFAULT_SETTINGS from django.conf.global_settings import PASSWORD_HASHERS @@ -15,6 +17,12 @@ 'PASSWORD': 'admin', 'AUTH': 'rest_models.backend.auth.BasicAuth', }, + 'apifail': { + 'ENGINE': 'rest_models.backend', + 'NAME': 'http://localapi/api/v1', + 'USER': 'admin', + 'PASSWORD': 'admin', + }, 'api2': { 'ENGINE': 'rest_models.backend', 'NAME': 'http://localhost:8080/api/v2', @@ -47,8 +55,12 @@ 'testapi', 'rest_framework', 'dynamic_rest', +) + ( + ('testapi.badapi', 'testapp.badapp') + if os.environ.get('WITH_BADAPP', "false").lower().strip() == 'true' + else tuple() ) - +print(INSTALLED_APPS) DATABASE_ROUTERS = [ 'rest_models.router.RestModelRouter', ]