diff --git a/bothub/api/v2/evaluate/filters.py b/bothub/api/v2/evaluate/filters.py index 4f9fd9ae..56f82412 100644 --- a/bothub/api/v2/evaluate/filters.py +++ b/bothub/api/v2/evaluate/filters.py @@ -1,5 +1,6 @@ from django.utils.translation import gettext as _ from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import Q from django_filters import rest_framework as filters from rest_framework.exceptions import PermissionDenied @@ -11,6 +12,7 @@ class EvaluatesFilter(filters.FilterSet): + class Meta: model = RepositoryEvaluate fields = [ @@ -69,7 +71,7 @@ def filter_entity(self, queryset, name, value): return queryset.filter(entities__entity__value=value) -class ResultsFilter(filters.FilterSet): +class EvaluateResultsFilter(filters.FilterSet): class Meta: model = RepositoryEvaluateResult @@ -95,3 +97,42 @@ def filter_repository_uuid(self, queryset, name, value): _('Repository {} does not exist').format(value)) except DjangoValidationError: raise NotFound(_('Invalid repository_uuid')) + + +class EvaluateResultFilter(filters.FilterSet): + + class Meta: + model = RepositoryEvaluateResult + fields = [] + + text = filters.CharFilter( + field_name='text', + method='filter_evaluate_text', + required=False, + help_text=_('Evaluate Text')) + + repository_uuid = filters.CharFilter( + field_name='repository_uuid', + method='filter_repository_uuid', + required=True, + help_text=_('Repository\'s UUID')) + + def filter_evaluate_text(self, queryset, name, value): + return queryset.filter( + Q(success_log__icontains=value) | Q(error_log__contains=value) + ) + + def filter_repository_uuid(self, queryset, name, value): + request = self.request + try: + repository = Repository.objects.get(uuid=value) + authorization = repository.get_user_authorization(request.user) + + if not authorization.can_read: + raise PermissionDenied() + return repository.evaluations_results(queryset=queryset) + except Repository.DoesNotExist: + raise NotFound( + _('Repository {} does not exist').format(value)) + except DjangoValidationError: + raise NotFound(_('Invalid repository_uuid')) diff --git a/bothub/api/v2/evaluate/serializers.py b/bothub/api/v2/evaluate/serializers.py index 79136eaa..9a13665e 100644 --- a/bothub/api/v2/evaluate/serializers.py +++ b/bothub/api/v2/evaluate/serializers.py @@ -108,7 +108,9 @@ class Meta: model = RepositoryEvaluateResult fields = [ 'id', - 'language' + 'language', + 'created_at', + 'version', ] language = serializers.SerializerMethodField() diff --git a/bothub/api/v2/evaluate/views.py b/bothub/api/v2/evaluate/views.py index 816e3f5e..1987872c 100644 --- a/bothub/api/v2/evaluate/views.py +++ b/bothub/api/v2/evaluate/views.py @@ -15,7 +15,8 @@ from .serializers import RepositoryEvaluateResultSerializer from .filters import EvaluatesFilter -from .filters import ResultsFilter +from .filters import EvaluateResultsFilter +from .filters import EvaluateResultFilter from .permissions import RepositoryEvaluatePermission from .permissions import RepositoryEvaluateResultPermission @@ -64,24 +65,21 @@ class ResultsListViewSet( GenericViewSet): queryset = RepositoryEvaluateResult.objects + serializer_class = RepositoryEvaluateResultVersionsSerializer permission_classes = [ IsAuthenticatedOrReadOnly, RepositoryEvaluateResultPermission, ] - - def list(self, request, *args, **kwargs): - self.serializer_class = RepositoryEvaluateResultVersionsSerializer - self.filter_class = ResultsFilter - self.filter_backends = [ - OrderingFilter, - DjangoFilterBackend, - ] - self.ordering_fields = [ - 'created_at', - ] - - return super().list(request, *args, **kwargs) + filter_class = EvaluateResultsFilter + filter_backends = [ + OrderingFilter, + DjangoFilterBackend, + ] + ordering_fields = [ + 'created_at', + ] def retrieve(self, request, *args, **kwargs): self.serializer_class = RepositoryEvaluateResultSerializer + self.filter_class = EvaluateResultFilter return super().retrieve(request, *args, **kwargs) diff --git a/bothub/common/management/commands/fill_db_using_fake_data.py b/bothub/common/management/commands/fill_db_using_fake_data.py index 8f36c882..0d5a207f 100644 --- a/bothub/common/management/commands/fill_db_using_fake_data.py +++ b/bothub/common/management/commands/fill_db_using_fake_data.py @@ -1,4 +1,5 @@ import random +import json from django.core.management.base import BaseCommand from django.conf import settings @@ -185,20 +186,20 @@ def handle(self, *args, **kwargs): # Evaluate Report - intent_results = RepositoryEvaluateResultScore.objects.create( - f1_score=0.976, - precision=0.978, - accuracy=0.976, - ) + for x in range(0, 2): + intent_results = RepositoryEvaluateResultScore.objects.create( + f1_score=0.976, + precision=0.978, + accuracy=0.976, + ) - entity_results = RepositoryEvaluateResultScore.objects.create( - f1_score=0.977, - precision=0.978, - accuracy=0.978, - ) + entity_results = RepositoryEvaluateResultScore.objects.create( + f1_score=0.977, + precision=0.978, + accuracy=0.978, + ) - success_log = ''' - [ + success_log = [ { "text": "hey", "intent": "greet", @@ -223,11 +224,9 @@ def handle(self, *args, **kwargs): "confidence": 0.8227075176309955 } } - ] - ''' + ] - error_log = ''' - [ + error_log = [ { "text": "test with nlu", "intent": "restaurant_search", @@ -237,92 +236,92 @@ def handle(self, *args, **kwargs): } } ] - ''' - evaluate_result = RepositoryEvaluateResult.objects.create( - repository_update=repository_1.current_update(), - intent_results=intent_results, - entity_results=entity_results, - matrix_chart='https://s3.amazonaws.com/bothub-sample/confmat.png', - confidence_chart='https://s3.amazonaws.com/bothub-sample/hist.png', - success_log=success_log, - error_log=error_log, - ) - - intent_score_1 = RepositoryEvaluateResultScore.objects.create( - precision=1.0, - recall=1.0, - f1_score=1.0, - support=11, - ) - - intent_score_2 = RepositoryEvaluateResultScore.objects.create( - precision=0.89, - recall=1.0, - f1_score=0.94, - support=8, - ) - - intent_score_3 = RepositoryEvaluateResultScore.objects.create( - precision=1.0, - recall=1.0, - f1_score=1.0, - support=8, - ) - - intent_score_4 = RepositoryEvaluateResultScore.objects.create( - precision=1.0, - recall=0.93, - f1_score=0.97, - support=15, - ) - - RepositoryEvaluateResultIntent.objects.create( - evaluate_result=evaluate_result, - intent='affirm', - score=intent_score_1, - ) - - RepositoryEvaluateResultIntent.objects.create( - evaluate_result=evaluate_result, - intent='goodbye', - score=intent_score_2, - ) - - RepositoryEvaluateResultIntent.objects.create( - evaluate_result=evaluate_result, - intent='greet', - score=intent_score_3, - ) - - RepositoryEvaluateResultIntent.objects.create( - evaluate_result=evaluate_result, - intent='restaurant_search', - score=intent_score_4, - ) - - entity_score_1 = RepositoryEvaluateResultScore.objects.create( - precision=1.0, - recall=0.90, - f1_score=0.95, - support=10, - ) - - entity_score_2 = RepositoryEvaluateResultScore.objects.create( - precision=1.0, - recall=0.75, - f1_score=0.86, - support=8, - ) - - RepositoryEvaluateResultEntity.objects.create( - evaluate_result=evaluate_result, - entity='cuisine', - score=entity_score_1, - ) - - RepositoryEvaluateResultEntity.objects.create( - evaluate_result=evaluate_result, - entity='greet', - score=entity_score_2, - ) + sample_url = 'https://s3.amazonaws.com/bothub-sample' + evaluate_result = RepositoryEvaluateResult.objects.create( + repository_update=repository_1.current_update(), + intent_results=intent_results, + entity_results=entity_results, + matrix_chart='{}/confmat.png'.format(sample_url), + confidence_chart='{}/hist.png'.format(sample_url), + success_log=json.dumps(success_log), + error_log=json.dumps(error_log), + ) + + intent_score_1 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=1.0, + f1_score=1.0, + support=11, + ) + + intent_score_2 = RepositoryEvaluateResultScore.objects.create( + precision=0.89, + recall=1.0, + f1_score=0.94, + support=8, + ) + + intent_score_3 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=1.0, + f1_score=1.0, + support=8, + ) + + intent_score_4 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.93, + f1_score=0.97, + support=15, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='affirm', + score=intent_score_1, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='goodbye', + score=intent_score_2, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='greet', + score=intent_score_3, + ) + + RepositoryEvaluateResultIntent.objects.create( + evaluate_result=evaluate_result, + intent='restaurant_search', + score=intent_score_4, + ) + + entity_score_1 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.90, + f1_score=0.95, + support=10, + ) + + entity_score_2 = RepositoryEvaluateResultScore.objects.create( + precision=1.0, + recall=0.75, + f1_score=0.86, + support=8, + ) + + RepositoryEvaluateResultEntity.objects.create( + evaluate_result=evaluate_result, + entity='cuisine', + score=entity_score_1, + ) + + RepositoryEvaluateResultEntity.objects.create( + evaluate_result=evaluate_result, + entity='greet', + score=entity_score_2, + ) diff --git a/bothub/common/migrations/0033_repositoryevaluateresult_version.py b/bothub/common/migrations/0033_repositoryevaluateresult_version.py new file mode 100644 index 00000000..8119ce12 --- /dev/null +++ b/bothub/common/migrations/0033_repositoryevaluateresult_version.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.5 on 2019-05-02 16:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0032_auto_20190430_2125'), + ] + + operations = [ + migrations.AddField( + model_name='repositoryevaluateresult', + name='version', + field=models.IntegerField(default=0, editable=False, verbose_name='Version'), + ), + ] diff --git a/bothub/common/models.py b/bothub/common/models.py index bbdc3b1d..9adbb984 100644 --- a/bothub/common/models.py +++ b/bothub/common/models.py @@ -1360,30 +1360,39 @@ class Meta: matrix_chart = models.URLField( verbose_name=_('Intent Confusion Matrix Chart'), - editable=False, - ) + editable=False) confidence_chart = models.URLField( verbose_name=_('Intent Prediction Confidence Distribution'), - editable=False, - ) + editable=False) success_log = models.TextField( verbose_name=_('Success Log'), blank=True, - editable=False, - ) + editable=False) error_log = models.TextField( verbose_name=_('Error Log'), blank=True, - editable=False, - ) + editable=False) + + version = models.IntegerField( + verbose_name=_('Version'), + blank=False, + default=0, + editable=False) created_at = models.DateTimeField( _('created at'), auto_now_add=True) + def save(self, *args, **kwargs): + version_number = RepositoryEvaluateResult.objects.filter( + repository_update=self.repository_update + ).count() + self.version = version_number + 1 + return super().save(*args, **kwargs) + class RepositoryEvaluateResultIntent(models.Model): class Meta: