From 0068cb6b8bef9d9e23c84a7c78c656ad5ba931a9 Mon Sep 17 00:00:00 2001 From: Eduardo Douglas Date: Tue, 30 Apr 2019 18:27:28 -0300 Subject: [PATCH] List results --- Pipfile | 1 - Pipfile.lock | 36 +--- bothub/api/v2/evaluate/filters.py | 31 ++- bothub/api/v2/evaluate/permissions.py | 11 + bothub/api/v2/evaluate/serializers.py | 104 ++++++++++ bothub/api/v2/evaluate/views.py | 36 ++++ bothub/api/v2/routers.py | 2 + .../commands/fill_db_using_fake_data.py | 191 ++++++++++++++++++ ...429_1858.py => 0032_auto_20190430_2125.py} | 34 ++-- bothub/common/models.py | 75 ++++--- 10 files changed, 441 insertions(+), 80 deletions(-) rename bothub/common/migrations/{0032_auto_20190429_1858.py => 0032_auto_20190430_2125.py} (79%) diff --git a/Pipfile b/Pipfile index 22e51ebe..fa680b51 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ requests = "==2.20.1" coreapi = "==2.3.3" whitenoise = "==4.1.2" pytz = "==2018.7" -"psycopg2-binary" = "==2.8.2" [dev-packages] "flake8" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8ce96d57..3077853c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5dabcf0f3f54030f79adcaad8e7e8b11b3e30d161ae29ea38bb28a83b1ff18ea" + "sha256": "075b9dfd76bab20bdff1c52632f8c791d2254a2d9b580b318aa62a16476cc39c" }, "pipfile-spec": 6, "requires": { @@ -138,40 +138,6 @@ ], "version": "==1.1.1" }, - "psycopg2-binary": { - "hashes": [ - "sha256:007ca0df127b1862fc010125bc4100b7a630efc6841047bd11afceadb4754611", - "sha256:03c49e02adf0b4d68f422fdbd98f7a7c547beb27e99a75ed02298f85cb48406a", - "sha256:0a1232cdd314e08848825edda06600455ad2a7adaa463ebfb12ece2d09f3370e", - "sha256:131c80d0958c89273d9720b9adf9df1d7600bb3120e16019a7389ab15b079af5", - "sha256:2de34cc3b775724623f86617d2601308083176a495f5b2efc2bbb0da154f483a", - "sha256:2eddc31500f73544a2a54123d4c4b249c3c711d31e64deddb0890982ea37397a", - "sha256:484f6c62bdc166ee0e5be3aa831120423bf399786d1f3b0304526c86180fbc0b", - "sha256:4c2d9369ed40b4a44a8ccd6bc3a7db6272b8314812d2d1091f95c4c836d92e06", - "sha256:70f570b5fa44413b9f30dbc053d17ef3ce6a4100147a10822f8662e58d473656", - "sha256:7a2b5b095f3bd733aab101c89c0e1a3f0dfb4ebdc26f6374805c086ffe29d5b2", - "sha256:804914a669186e2843c1f7fbe12b55aad1b36d40a28274abe6027deffad9433d", - "sha256:8520c03172da18345d012949a53617a963e0191ccb3c666f23276d5326af27b5", - "sha256:90da901fc33ea393fc644607e4a3916b509387e9339ec6ebc7bfded45b7a0ae9", - "sha256:a582416ad123291a82c300d1d872bdc4136d69ad0b41d57dc5ca3df7ef8e3088", - "sha256:ac8c5e20309f4989c296d62cac20ee456b69c41fd1bc03829e27de23b6fa9dd0", - "sha256:b2cf82f55a619879f8557fdaae5cec7a294fac815e0087c4f67026fdf5259844", - "sha256:b59d6f8cfca2983d8fdbe457bf95d2192f7b7efdb2b483bf5fa4e8981b04e8b2", - "sha256:be08168197021d669b9964bd87628fa88f910b1be31e7010901070f2540c05fd", - "sha256:be0f952f1c365061041bad16e27e224e29615d4eb1fb5b7e7760a1d3d12b90b6", - "sha256:c1c9a33e46d7c12b9c96cf2d4349d783e3127163fd96254dcd44663cf0a1d438", - "sha256:d18c89957ac57dd2a2724ecfe9a759912d776f96ecabba23acb9ecbf5c731035", - "sha256:d7e7b0ff21f39433c50397e60bf0995d078802c591ca3b8d99857ea18a7496ee", - "sha256:da0929b2bf0d1f365345e5eb940d8713c1d516312e010135b14402e2a3d2404d", - "sha256:de24a4962e361c512d3e528ded6c7480eab24c655b8ca1f0b761d3b3650d2f07", - "sha256:e45f93ff3f7dae2202248cf413a87aeb330821bf76998b3cf374eda2fc893dd7", - "sha256:f046aeae1f7a845041b8661bb7a52449202b6c5d3fb59eb4724e7ca088811904", - "sha256:f1dc2b7b2748084b890f5d05b65a47cd03188824890e9a60818721fd492249fb", - "sha256:fcbe7cf3a786572b73d2cd5f34ed452a5f5fac47c9c9d1e0642c457a148f9f88" - ], - "index": "pypi", - "version": "==2.8.2" - }, "python-decouple": { "hashes": [ "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" diff --git a/bothub/api/v2/evaluate/filters.py b/bothub/api/v2/evaluate/filters.py index b3ed308c..4f9fd9ae 100644 --- a/bothub/api/v2/evaluate/filters.py +++ b/bothub/api/v2/evaluate/filters.py @@ -5,8 +5,9 @@ from rest_framework.exceptions import PermissionDenied from rest_framework.exceptions import NotFound -from bothub.common.models import RepositoryEvaluate from bothub.common.models import Repository +from bothub.common.models import RepositoryEvaluate +from bothub.common.models import RepositoryEvaluateResult class EvaluatesFilter(filters.FilterSet): @@ -66,3 +67,31 @@ def filter_label(self, queryset, name, value): def filter_entity(self, queryset, name, value): return queryset.filter(entities__entity__value=value) + + +class ResultsFilter(filters.FilterSet): + + class Meta: + model = RepositoryEvaluateResult + fields = [] + + repository_uuid = filters.CharFilter( + field_name='repository_uuid', + method='filter_repository_uuid', + required=True, + help_text=_('Repository\'s UUID')) + + def filter_repository_uuid(self, queryset, name, value): + request = self.request + try: + repository = Repository.objects.get(uuid=value) + authorization = repository.get_user_authorization(request.user) + + if not authorization.can_read: + raise PermissionDenied() + return repository.evaluations_results(queryset=queryset) + except Repository.DoesNotExist: + raise NotFound( + _('Repository {} does not exist').format(value)) + except DjangoValidationError: + raise NotFound(_('Invalid repository_uuid')) diff --git a/bothub/api/v2/evaluate/permissions.py b/bothub/api/v2/evaluate/permissions.py index e2f9955e..79fbb631 100644 --- a/bothub/api/v2/evaluate/permissions.py +++ b/bothub/api/v2/evaluate/permissions.py @@ -16,3 +16,14 @@ def has_object_permission(self, request, view, obj): return authorization.can_write return authorization.is_admin return False + + +class RepositoryEvaluateResultPermission(permissions.BasePermission): + + def has_object_permission(self, request, view, obj): + authorization = obj.repository_update. \ + repository.get_user_authorization(request.user) + + if request.method in READ_METHODS: + return authorization.can_read + return authorization.can_contribute diff --git a/bothub/api/v2/evaluate/serializers.py b/bothub/api/v2/evaluate/serializers.py index e3a31919..79136eaa 100644 --- a/bothub/api/v2/evaluate/serializers.py +++ b/bothub/api/v2/evaluate/serializers.py @@ -1,9 +1,16 @@ +import json + from django.utils.translation import gettext as _ from rest_framework import serializers from bothub.common.models import Repository from bothub.common.models import RepositoryEvaluate from bothub.common.models import RepositoryEvaluateEntity +from bothub.common.models import RepositoryEvaluateResult +from bothub.common.models import RepositoryEvaluateResultScore +from bothub.common.models import RepositoryEvaluateResultIntent +from bothub.common.models import RepositoryEvaluateResultEntity + from bothub.common.languages import LANGUAGE_CHOICES from ..fields import EntityValueField @@ -93,3 +100,100 @@ def update(self, instance, validated_data): repository_evaluate=instance, **entity) return instance + + +class RepositoryEvaluateResultVersionsSerializer(serializers.ModelSerializer): + + class Meta: + model = RepositoryEvaluateResult + fields = [ + 'id', + 'language' + ] + + language = serializers.SerializerMethodField() + + def get_language(self, obj): + return obj.repository_update.language + + +class RepositoryEvaluateResultScore(serializers.ModelSerializer): + + class Meta: + model = RepositoryEvaluateResultScore + fields = [ + 'precision', + 'f1_score', + 'accuracy', + 'recall', + 'support' + ] + + +class RepositoryEvaluateResultIntentSerializer(serializers.ModelSerializer): + + class Meta: + model = RepositoryEvaluateResultIntent + fields = [ + 'intent', + 'score', + ] + + score = RepositoryEvaluateResultScore(read_only=True) + + +class RepositoryEvaluateResultEntitySerializer(serializers.ModelSerializer): + + class Meta: + model = RepositoryEvaluateResultEntity + fields = [ + 'entity', + 'score', + ] + + score = RepositoryEvaluateResultScore(read_only=True) + entity = serializers.SerializerMethodField() + + def get_entity(self, obj): + return obj.entity.value + + +class RepositoryEvaluateResultSerializer(serializers.ModelSerializer): + + class Meta: + model = RepositoryEvaluateResult + fields = [ + 'id', + 'created_at', + 'matrix_chart', + 'confidence_chart', + 'success_log', + 'error_log', + 'intents_list', + 'entities_list', + 'intent_results', + 'entity_results' + + ] + + success_log = serializers.SerializerMethodField() + error_log = serializers.SerializerMethodField() + intent_results = RepositoryEvaluateResultScore(read_only=True) + entity_results = RepositoryEvaluateResultScore(read_only=True) + + intents_list = serializers.SerializerMethodField() + entities_list = serializers.SerializerMethodField() + + def get_intents_list(self, obj): + return RepositoryEvaluateResultIntentSerializer( + obj.evaluate_result_intent.all(), many=True).data + + def get_entities_list(self, obj): + return RepositoryEvaluateResultEntitySerializer( + obj.evaluate_result_entity.all(), many=True).data + + def get_success_log(self, obj): + return json.loads(obj.success_log) + + def get_error_log(self, obj): + return json.loads(obj.error_log) diff --git a/bothub/api/v2/evaluate/views.py b/bothub/api/v2/evaluate/views.py index 461634ca..816e3f5e 100644 --- a/bothub/api/v2/evaluate/views.py +++ b/bothub/api/v2/evaluate/views.py @@ -7,11 +7,18 @@ from django_filters.rest_framework import DjangoFilterBackend from bothub.common.models import RepositoryEvaluate +from bothub.common.models import RepositoryEvaluateResult from ..metadata import Metadata from .serializers import RepositoryEvaluateSerializer +from .serializers import RepositoryEvaluateResultVersionsSerializer +from .serializers import RepositoryEvaluateResultSerializer + from .filters import EvaluatesFilter +from .filters import ResultsFilter + from .permissions import RepositoryEvaluatePermission +from .permissions import RepositoryEvaluateResultPermission class EvaluateViewSet( @@ -49,3 +56,32 @@ def list(self, request, *args, **kwargs): ] return super().list(request, *args, **kwargs) + + +class ResultsListViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + GenericViewSet): + + queryset = RepositoryEvaluateResult.objects + permission_classes = [ + IsAuthenticatedOrReadOnly, + RepositoryEvaluateResultPermission, + ] + + def list(self, request, *args, **kwargs): + self.serializer_class = RepositoryEvaluateResultVersionsSerializer + self.filter_class = ResultsFilter + self.filter_backends = [ + OrderingFilter, + DjangoFilterBackend, + ] + self.ordering_fields = [ + 'created_at', + ] + + return super().list(request, *args, **kwargs) + + def retrieve(self, request, *args, **kwargs): + self.serializer_class = RepositoryEvaluateResultSerializer + return super().retrieve(request, *args, **kwargs) diff --git a/bothub/api/v2/routers.py b/bothub/api/v2/routers.py index 51b0fdaf..45bfc546 100644 --- a/bothub/api/v2/routers.py +++ b/bothub/api/v2/routers.py @@ -4,10 +4,12 @@ from .repository.views import RepositoriesViewSet from .examples.views import ExamplesViewSet from .evaluate.views import EvaluateViewSet +from .evaluate.views import ResultsListViewSet router = routers.SimpleRouter() router.register('repository', RepositoryViewSet) router.register('repositories', RepositoriesViewSet) router.register('examples', ExamplesViewSet) +router.register('evaluate/results', ResultsListViewSet) router.register('evaluate', EvaluateViewSet) diff --git a/bothub/common/management/commands/fill_db_using_fake_data.py b/bothub/common/management/commands/fill_db_using_fake_data.py index c1df595b..8f36c882 100644 --- a/bothub/common/management/commands/fill_db_using_fake_data.py +++ b/bothub/common/management/commands/fill_db_using_fake_data.py @@ -10,6 +10,13 @@ from bothub.common.models import RepositoryExampleEntity from bothub.common.models import RepositoryTranslatedExample from bothub.common.models import RepositoryTranslatedExampleEntity +from bothub.common.models import RepositoryEvaluate +from bothub.common.models import RepositoryEvaluateEntity +from bothub.common.models import RepositoryEvaluateResult +from bothub.common.models import RepositoryEvaluateResultScore +from bothub.common.models import RepositoryEvaluateResultIntent +from bothub.common.models import RepositoryEvaluateResultEntity + from bothub.common import languages @@ -100,6 +107,12 @@ def handle(self, *args, **kwargs): # Example Entity + RepositoryExampleEntity.objects.create( + repository_example=example_1, + start=8, + end=15, + entity='cuisine') + RepositoryExampleEntity.objects.create( repository_example=example_5, start=8, @@ -135,3 +148,181 @@ def handle(self, *args, **kwargs): start=23, end=29, entity='cuisine') + + # Evaluates + + evalute_1 = RepositoryEvaluate.objects.create( + repository_update=repository_1.current_update(), + text='show me chinese restaurants', + intent='restaurant_search') + + evalute_2 = RepositoryEvaluate.objects.create( + repository_update=repository_1.current_update(), + text='hello', + intent='greet') + + RepositoryEvaluate.objects.create( + repository_update=repository_1.current_update(), + text='yes', + intent='affirm') + + RepositoryEvaluate.objects.create( + repository_update=repository_1.current_update(), + text='yep', + intent='affirm') + + RepositoryEvaluateEntity.objects.create( + repository_evaluate=evalute_1, + start=23, + end=29, + entity='cuisine') + + RepositoryEvaluateEntity.objects.create( + repository_evaluate=evalute_2, + start=0, + end=5, + entity='greet') + + # Evaluate Report + + intent_results = RepositoryEvaluateResultScore.objects.create( + f1_score=0.976, + precision=0.978, + accuracy=0.976, + ) + + entity_results = RepositoryEvaluateResultScore.objects.create( + f1_score=0.977, + precision=0.978, + accuracy=0.978, + ) + + success_log = ''' + [ + { + "text": "hey", + "intent": "greet", + "intent_prediction": { + "name": "greet", + "confidence": 0.9263743763408538 + } + }, + { + "text": "howdy", + "intent": "greet", + "intent_prediction": { + "name": "greet", + "confidence": 0.8099720606047796 + } + }, + { + "text": "hey there", + "intent": "greet", + "intent_prediction": { + "name": "greet", + "confidence": 0.8227075176309955 + } + } + ] + ''' + + error_log = ''' + [ + { + "text": "test with nlu", + "intent": "restaurant_search", + "intent_prediction": { + "name": "goodbye", + "confidence": 0.3875259420712092 + } + } + ] + ''' + + evaluate_result = RepositoryEvaluateResult.objects.create( + repository_update=repository_1.current_update(), + intent_results=intent_results, + entity_results=entity_results, + matrix_chart='https://s3.amazonaws.com/bothub-sample/confmat.png', + confidence_chart='https://s3.amazonaws.com/bothub-sample/hist.png', + success_log=success_log, + error_log=error_log, + ) + + intent_score_1 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=1.0, + f1_score=1.0, + support=11, + ) + + intent_score_2 = RepositoryEvaluateResultScore.objects.create( + precision=0.89, + recall=1.0, + f1_score=0.94, + support=8, + ) + + intent_score_3 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=1.0, + f1_score=1.0, + support=8, + ) + + intent_score_4 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.93, + f1_score=0.97, + support=15, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='affirm', + score=intent_score_1, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='goodbye', + score=intent_score_2, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='greet', + score=intent_score_3, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='restaurant_search', + score=intent_score_4, + ) + + entity_score_1 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.90, + f1_score=0.95, + support=10, + ) + + entity_score_2 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.75, + f1_score=0.86, + support=8, + ) + + RepositoryEvaluateResultEntity.objects.create( + evaluate_result=evaluate_result, + entity='cuisine', + score=entity_score_1, + ) + + RepositoryEvaluateResultEntity.objects.create( + evaluate_result=evaluate_result, + entity='greet', + score=entity_score_2, + ) diff --git a/bothub/common/migrations/0032_auto_20190429_1858.py b/bothub/common/migrations/0032_auto_20190430_2125.py similarity index 79% rename from bothub/common/migrations/0032_auto_20190429_1858.py rename to bothub/common/migrations/0032_auto_20190430_2125.py index 8b835f98..38e5dced 100644 --- a/bothub/common/migrations/0032_auto_20190429_1858.py +++ b/bothub/common/migrations/0032_auto_20190430_2125.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.5 on 2019-04-29 18:58 +# Generated by Django 2.1.5 on 2019-04-30 21:25 import django.core.validators from django.db import migrations, models @@ -17,8 +17,10 @@ class Migration(migrations.Migration): name='RepositoryEvaluateResult', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('intent_chart', models.URLField(verbose_name='Intenties chart')), - ('entity_chart', models.URLField(verbose_name='Entities chart')), + ('matrix_chart', models.URLField(editable=False, verbose_name='Intent Confusion Matrix Chart')), + ('confidence_chart', models.URLField(editable=False, verbose_name='Intent Prediction Confidence Distribution')), + ('success_log', models.TextField(blank=True, editable=False, verbose_name='Success Log')), + ('error_log', models.TextField(blank=True, editable=False, verbose_name='Error Log')), ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), ], options={ @@ -32,7 +34,7 @@ class Migration(migrations.Migration): name='RepositoryEvaluateResultEntity', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('entity', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='entity_results', to='common.RepositoryEntity')), + ('entity', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='entity', to='common.RepositoryEntity')), ('evaluate_result', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='evaluate_result_entity', to='common.RepositoryEvaluateResult')), ], options={ @@ -51,14 +53,14 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name='RepositoryEvaluationResultScore', + name='RepositoryEvaluateResultScore', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('precision', models.DecimalField(decimal_places=2, max_digits=3)), - ('f1_score', models.DecimalField(decimal_places=2, max_digits=3)), - ('accuracy', models.DecimalField(decimal_places=2, max_digits=3)), - ('recall', models.DecimalField(decimal_places=2, max_digits=3)), - ('support', models.DecimalField(decimal_places=2, max_digits=3)), + ('precision', models.DecimalField(decimal_places=2, max_digits=3, null=True)), + ('f1_score', models.DecimalField(decimal_places=2, max_digits=3, null=True)), + ('accuracy', models.DecimalField(decimal_places=2, max_digits=3, null=True)), + ('recall', models.DecimalField(decimal_places=2, max_digits=3, null=True)), + ('support', models.IntegerField(null=True)), ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), ], options={ @@ -69,26 +71,26 @@ class Migration(migrations.Migration): migrations.AddField( model_name='repositoryevaluateresultintent', name='score', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_intenties_score', to='common.RepositoryEvaluationResultScore'), + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_intenties_score', to='common.RepositoryEvaluateResultScore'), ), migrations.AddField( model_name='repositoryevaluateresultentity', name='score', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_entities_score', to='common.RepositoryEvaluationResultScore'), + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_entities_score', to='common.RepositoryEvaluateResultScore'), ), migrations.AddField( model_name='repositoryevaluateresult', name='entity_results', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='entity_results', to='common.RepositoryEvaluationResultScore'), + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='entity_results', to='common.RepositoryEvaluateResultScore'), ), migrations.AddField( model_name='repositoryevaluateresult', name='intent_results', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='intent_results', to='common.RepositoryEvaluationResultScore'), + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='intent_results', to='common.RepositoryEvaluateResultScore'), ), migrations.AddField( model_name='repositoryevaluateresult', - name='repository_evaluate', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='results', to='common.RepositoryEvaluate'), + name='repository_update', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='results', to='common.RepositoryUpdate'), ), ] diff --git a/bothub/common/models.py b/bothub/common/models.py index 3a8d316b..bbdc3b1d 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -12,7 +12,6 @@ from django.template.loader import render_to_string from django.dispatch import receiver from django.core.exceptions import ValidationError -from django.contrib.postgres.fields import JSONField from bothub.authentication.models import User @@ -348,6 +347,13 @@ def evaluations(self, language=None, exclude_deleted=True, queryset=None): return query.exclude(deleted_in__isnull=False) return query + def evaluations_results(self, queryset=None): + if queryset is None: + queryset = RepositoryEvaluateResult.objects + query = queryset.filter( + repository_update__repository=self) + return query + def language_status(self, language): is_base_language = self.language == language examples = self.examples(language) @@ -878,11 +884,15 @@ class EntityBaseQueryset(models.QuerySet): def create(self, entity, **kwargs): if type(entity) is not RepositoryEntity: instance = self.model(**kwargs) - if instance.example: - repository = instance.example.repository_update.repository - elif instance.repository_evaluate: + if 'repository_evaluate_id' in instance.__dict__: evaluate = instance.repository_evaluate repository = evaluate.repository_update.repository + elif 'evaluate_result_id' in instance.__dict__: + result = instance.evaluate_result + repository = result.repository_update.repository + else: + repository = instance.example.repository_update.repository + entity = RepositoryEntity.objects.get( repository=repository, value=entity) @@ -1290,30 +1300,33 @@ def get_evaluate(self): return self.repository_evaluate -class RepositoryEvaluationResultScore(models.Model): +class RepositoryEvaluateResultScore(models.Model): class Meta: db_table = 'common_repository_evaluate_result_score' ordering = ['-created_at'] precision = models.DecimalField( max_digits=3, - decimal_places=2) + decimal_places=2, + null=True) f1_score = models.DecimalField( max_digits=3, - decimal_places=2) + decimal_places=2, + null=True) accuracy = models.DecimalField( max_digits=3, - decimal_places=2) + decimal_places=2, + null=True) recall = models.DecimalField( max_digits=3, - decimal_places=2) + decimal_places=2, + null=True) - support = models.DecimalField( - max_digits=3, - decimal_places=2) + support = models.IntegerField( + null=True) created_at = models.DateTimeField( _('created at'), @@ -1327,38 +1340,44 @@ class Meta: verbose_name_plural = _('evaluate results') ordering = ['-created_at'] - repository_evaluate = models.ForeignKey( - RepositoryEvaluate, + repository_update = models.ForeignKey( + RepositoryUpdate, models.CASCADE, editable=False, related_name='results') intent_results = models.ForeignKey( - RepositoryEvaluationResultScore, + RepositoryEvaluateResultScore, models.CASCADE, editable=False, related_name='intent_results') entity_results = models.ForeignKey( - RepositoryEvaluationResultScore, + RepositoryEvaluateResultScore, models.CASCADE, editable=False, related_name='entity_results') - intent_chart = models.URLField( - verbose_name=_('Intenties chart') + matrix_chart = models.URLField( + verbose_name=_('Intent Confusion Matrix Chart'), + editable=False, ) - entity_chart = models.URLField( - verbose_name=_('Entities chart') + confidence_chart = models.URLField( + verbose_name=_('Intent Prediction Confidence Distribution'), + editable=False, ) - success_log = JSONField( - verbose_name=_('Success Log') + success_log = models.TextField( + verbose_name=_('Success Log'), + blank=True, + editable=False, ) - error_log = JSONField( - verbose_name=_('Error Log') + error_log = models.TextField( + verbose_name=_('Error Log'), + blank=True, + editable=False, ) created_at = models.DateTimeField( @@ -1383,7 +1402,7 @@ class Meta: validators=[validate_item_key]) score = models.ForeignKey( - RepositoryEvaluationResultScore, + RepositoryEvaluateResultScore, models.CASCADE, related_name='evaluation_intenties_score', editable=False) @@ -1402,15 +1421,17 @@ class Meta: entity = models.ForeignKey( RepositoryEntity, models.CASCADE, - related_name='entity_results', + related_name='entity', editable=False) score = models.ForeignKey( - RepositoryEvaluationResultScore, + RepositoryEvaluateResultScore, models.CASCADE, related_name='evaluation_entities_score', editable=False) + objects = EntityBaseManager() + @receiver(models.signals.pre_save, sender=RequestRepositoryAuthorization) def set_user_role_on_approved(instance, **kwargs):