Skip to content

Commit

Permalink
Merge cd529e8 into b0498d4
Browse files Browse the repository at this point in the history
  • Loading branch information
@dyohan9
dyohan9 committed Jul 17, 2019
2 parents b0498d4 + cd529e8 commit 123539c
Show file tree
Hide file tree
Showing 2 changed files with 285 additions and 1 deletion.
53 changes: 52 additions & 1 deletion bothub/api/v2/evaluate/serializers.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from decimal import Decimal, ROUND_DOWN

from django.utils.translation import gettext as _
from rest_framework import serializers
Expand Down Expand Up @@ -214,4 +215,54 @@ def get_entities_list(self, obj):
obj.evaluate_result_entity.all(), many=True).data

def get_log(self, obj):
return json.loads(obj.log)
intent = self.context.get('request'). \
query_params.get('intent')
min_confidence = self.context.get('request'). \
query_params.get('min')
max_confidence = self.context.get('request'). \
query_params.get('max')

def filter_intent(log, intent, min_confidence, max_confidence):
if not intent and not min_confidence and not max_confidence:
return log

confidence = float(
Decimal(
log.get('intent_prediction').get('confidence')
).quantize(Decimal('0.00'), rounding=ROUND_DOWN)
)

has_intent = False

if min_confidence and max_confidence:
min_confidence = float(min_confidence) / 100
max_confidence = float(max_confidence) / 100

has_intent = True if \
min_confidence <= \
confidence <= \
max_confidence else False

if intent and log.get('intent') != intent:
has_intent = False

if has_intent:
return log

results = filter(
None,
list(
map(
lambda log:
filter_intent(
log,
intent,
min_confidence,
max_confidence
),
json.loads(obj.log)
)
)
)

return results
233 changes: 233 additions & 0 deletions bothub/api/v2/tests/test_evaluate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,3 +520,236 @@ def test_okay(self):
self.assertEqual(
response.status_code,
status.HTTP_200_OK)


class ListEvaluateResultTestFilterCase(TestCase):
def setUp(self):
self.factory = RequestFactory()

self.owner, self.owner_token = create_user_and_token('owner')
self.user, self.token = create_user_and_token()

self.repository = Repository.objects.create(
owner=self.owner,
name='Testing',
slug='test',
language=languages.LANGUAGE_EN
)

intent_results = RepositoryEvaluateResultScore.objects.create(
f1_score=0.976,
precision=0.978,
accuracy=0.976,
)

entity_results = RepositoryEvaluateResultScore.objects.create(
f1_score=0.977,
precision=0.978,
accuracy=0.978,
)

evaluate_log = [
{
"text": "hey",
"intent": "greet",
"intent_prediction": {
"name": "greet",
"confidence": 0.9263743763408538
},
"status": "success"
},
{
"text": "howdy",
"intent": "greet",
"intent_prediction": {
"name": "greet",
"confidence": 0.8099720606047796
},
"status": "success"
},
{
"text": "hey there",
"intent": "greet",
"intent_prediction": {
"name": "greet",
"confidence": 0.8227075176309955
},
"status": "success"
},
{
"text": "test with nlu",
"intent": "restaurant_search",
"intent_prediction": {
"name": "goodbye",
"confidence": 0.3875259420712092
},
"status": "error"
}
]

sample_url = 'https://s3.amazonaws.com/bothub-sample'
self.evaluate_result = RepositoryEvaluateResult.objects.create(
repository_update=self.repository.current_update(),
intent_results=intent_results,
entity_results=entity_results,
matrix_chart='{}/confmat.png'.format(sample_url),
confidence_chart='{}/hist.png'.format(sample_url),
log=json.dumps(evaluate_log),
)

intent_score_1 = RepositoryEvaluateResultScore.objects.create(
precision=1.0,
recall=1.0,
f1_score=1.0,
support=11,
)

intent_score_2 = RepositoryEvaluateResultScore.objects.create(
precision=0.89,
recall=1.0,
f1_score=0.94,
support=8,
)

intent_score_3 = RepositoryEvaluateResultScore.objects.create(
precision=1.0,
recall=1.0,
f1_score=1.0,
support=8,
)

intent_score_4 = RepositoryEvaluateResultScore.objects.create(
precision=1.0,
recall=0.93,
f1_score=0.97,
support=15,
)

RepositoryEvaluateResultIntent.objects.create(
evaluate_result=self.evaluate_result,
intent='affirm',
score=intent_score_1,
)

RepositoryEvaluateResultIntent.objects.create(
evaluate_result=self.evaluate_result,
intent='goodbye',
score=intent_score_2,
)

RepositoryEvaluateResultIntent.objects.create(
evaluate_result=self.evaluate_result,
intent='greet',
score=intent_score_3,
)

RepositoryEvaluateResultIntent.objects.create(
evaluate_result=self.evaluate_result,
intent='restaurant_search',
score=intent_score_4,
)

entity_score_1 = RepositoryEvaluateResultScore.objects.create(
precision=1.0,
recall=0.90,
f1_score=0.95,
support=10,
)

entity_score_2 = RepositoryEvaluateResultScore.objects.create(
precision=1.0,
recall=0.75,
f1_score=0.86,
support=8,
)

RepositoryEvaluateResultEntity.objects.create(
evaluate_result=self.evaluate_result,
entity='cuisine',
score=entity_score_1,
)

RepositoryEvaluateResultEntity.objects.create(
evaluate_result=self.evaluate_result,
entity='greet',
score=entity_score_2,
)

def request(self, token, params):
authorization_header = {
'HTTP_AUTHORIZATION': 'Token {}'.format(token.key),
}
request = self.factory.get(
'/api/v2/evaluate/results/{}/{}'.format(
self.evaluate_result.id,
params
), **authorization_header
)
response = ResultsListViewSet.as_view({'get': 'retrieve'})(
request,
pk=self.evaluate_result.id,
repository_uuid=self.repository.uuid
)
response.render()
content_data = json.loads(response.content)
return (response, content_data,)

def test_okay(self):
response, content_data = self.request(
self.owner_token,
'?repository_uuid={}'.format(
self.repository.uuid
)
)
self.assertEqual(len(content_data['log']), 4)
self.assertEqual(
response.status_code,
status.HTTP_200_OK)

def test_okay_intent_filter(self):
response, content_data = self.request(
self.owner_token,
'?repository_uuid={}&intent=greet&min=0&max=100'.format(
self.repository.uuid
)
)
self.assertEqual(len(content_data['log']), 3)
self.assertEqual(
response.status_code,
status.HTTP_200_OK)

def test_okay_without_intent_filter(self):
response, content_data = self.request(
self.owner_token,
'?repository_uuid={}&min=0&max=100'.format(
self.repository.uuid
)
)
self.assertEqual(len(content_data['log']), 4)
self.assertEqual(
response.status_code,
status.HTTP_200_OK)

def test_okay_range_without_intent_filter(self):
response, content_data = self.request(
self.owner_token,
'?repository_uuid={}&min=50&max=80'.format(
self.repository.uuid
)
)
self.assertEqual(len(content_data['log']), 1)
self.assertEqual(
response.status_code,
status.HTTP_200_OK)

def test_okay_range_with_intent_filter(self):
response, content_data = self.request(
self.owner_token,
'?repository_uuid={}&intent=greet&min=50&max=80'.format(
self.repository.uuid
)
)
self.assertEqual(len(content_data['log']), 1)
self.assertEqual(
response.status_code,
status.HTTP_200_OK)

0 comments on commit 123539c

Please sign in to comment.