Skip to content
This repository has been archived by the owner on Jun 13, 2020. It is now read-only.

Commit

Permalink
Adds demo option for handling json
Browse files Browse the repository at this point in the history 
Since we're switching to handling straight json instead of jsonp,
this PR needs to be coordinated with the `json-handling` branch
of CCDB4/complaint-randomizer.

The original WordPress page pulled the json straight from S3, which
required formatting the narratives as jsonp to avoid CORS restrictions.
We can drop all that now, because a Django view pulls from S3.
  • Loading branch information
@higs4281
higs4281 committed Sep 2, 2016
1 parent de9c3b1 commit c9b57f2
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 51 deletions.
35 changes: 25 additions & 10 deletions complaintdatabase/tests.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import collections
from mock import patch, Mock
from mock import patch, Mock, MagicMock, mock_open

from requests.exceptions import ConnectionError
from django.test import RequestFactory, TestCase
Expand Down Expand Up @@ -32,13 +32,27 @@ def test_get_context_data_exist(self):
class NarrativeJsonTest(TestCase):

@patch('complaintdatabase.views.requests.get')
def test_get_narratives_json(self, mock_requests_get):
def test_get_narratives_json(self, mock_get):
# Using namedtuple to mock out the attribute text in response
# not sure if this is the best way though
Response = collections.namedtuple('Response', 'text')
mock_requests_get.return_value = Response(text="narratives({});")
mock_return = MagicMock()
mock_return.json.return_value = {}
mock_get.return_value = mock_return
res_json = get_narratives_json()
self.assertEqual(res_json, {})
self.assertTrue(mock_get.call_count == 1)

@patch('complaintdatabase.views.requests.get')
def test_get_demo_narratives_json(self, mock_get):
# Using namedtuple to mock out the attribute text in response
# not sure if this is the best way though
mock_return = MagicMock()
mock_return.json.return_value = {}
mock_get.return_value = mock_return
m = mock_open(read_data='{"mock_data": ""}')
with patch("__builtin__.open", m, create=True):
res_json = get_narratives_json(demo_json='/fake/path')
self.assertEqual(res_json, {"mock_data": ""})

@patch('complaintdatabase.views.requests.get')
def test_request_exception_get_narratives_json(self, mock_requests_get):
Expand All @@ -50,14 +64,15 @@ def test_request_exception_get_narratives_json(self, mock_requests_get):
fakeOutput.getvalue().strip())

@patch('complaintdatabase.views.requests.get')
def test_incorrect_text_get_narratives_json(self, mock_requests_get):
Response = collections.namedtuple('Response', 'text')
mock_requests_get.return_value = Response(text=("This is not a correct"
" set of narratives"))
with patch('sys.stdout', new=StringIO()) as fakeOutput:
def test_incorrect_text_get_narratives_json(self, mock_get):
mock_return = MagicMock()
mock_return.json.return_value = {}
mock_get.return_value = mock_return
with patch('sys.stdout', new=StringIO('ValueError')) as fakeOutput:
res_json = get_narratives_json()
self.assertEqual(res_json, {})
self.assertIn('ValueError', fakeOutput.getvalue().strip())
self.assertIn('ValueError', fakeOutput.getvalue())
self.assertTrue(mock_get.call_count == 1)


class FormatNarrativesTest(TestCase):
Expand Down
141 changes: 101 additions & 40 deletions complaintdatabase/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,20 @@
else: # pragma: no cover
BASE_TEMPLATE = "front/base_update.html"


class LandingView(TemplateView):
template_name = "landing-page.html"

def get_context_data(self, **kwargs):
context = super(LandingView, self).get_context_data(**kwargs)
context['base_template'] = BASE_TEMPLATE

res_json = get_narratives_json()

context['narratives'] = format_narratives(res_json)
context['stats'] = get_stats(res_json)
context['total_complaints'], context['timely_responses'] = get_count_info()

context['data_down'], context['narratives_down'] = is_data_not_updated(res_json)


(context['total_complaints'],
context['timely_responses']) = get_count_info()
(context['data_down'],
context['narratives_down']) = is_data_not_updated(res_json)
return context


Expand All @@ -42,83 +40,139 @@ def get_context_data(self, **kwargs):
context['base_template'] = BASE_TEMPLATE
return context

def get_narratives_json():

def get_narratives_json(demo_json=None):
"""
Main handler to deliver sample narratives to the landing page.
To demo a local json file, pass in a file path via the 'demo_json' kwarg
"""

if demo_json:
with open(demo_json, 'r') as f:
res_json = json.loads(f.read())
return res_json

s3json = "http://files.consumerfinance.gov/ccdb/narratives.json"
try:
response = requests.get("http://files.consumerfinance.gov/ccdb/narratives.json")
res_json = json.loads(response.text[11:-2]) # This is to parse out the 'narratives();' that wrapped around the json
res_json = requests.get(s3json).json()
except requests.exceptions.RequestException as e:
print("get_narratives_json:requests.exceptions.RequestException")
print("There is a problem with getting data from the URL")
print(e)
res_json = json.loads('{}')
res_json = {}
except ValueError as e:
print("get_narratives_json:ValueError")
print("The text from the response doesn't follow the correct format to be parse as json")
print("The text from the response doesn't follow "
"the correct format to be parse as json")
print(e)
res_json = json.loads('{}')
res_json = {}
return res_json


def format_narratives(res_json):

"""Set up label and css values for page."""
narratives = []

# Additional data needed to output the narratives on the page
# title, css, and icon are required
# optional 'tooltip' determines text that displays on section button hover (defaults to title)
# optional 'tooltip' determines text that displays on section button hover
# 'tooltip' defaults to title
narrative_types = [
{'key': 'bank_accounts', 'title': 'Bank account', 'css': 'bank-account', 'icon':'bank-account'},
{'key':'credit_cards', 'title':'Credit card', 'css':'credit-card', 'icon':'credit-card'},
{'key':'credit_reporting', 'title':'Credit reporting', 'css':'credit-reporting', 'icon':'loan'},
{'key':'debt_collection', 'title':'Debt collection', 'css':'debt-collection', 'icon': 'debt-collection'},
{'key':'money_transfers', 'title':'Money transfer or virtual currency', 'tooltip': 'Money transfer/virtual currency', 'css':'money-transfer', 'icon': 'money-transfer'},
{'key':'mortgages', 'title':'Mortgage', 'css':'mortgage', 'icon': 'owning-home'},
{'key':'other_financial_services', 'title':'Other financial service', 'css':'other', 'icon': 'money'},
{'key':'payday_loans', 'title':'Payday loan', 'css':'payday-loan', 'icon': 'payday-loan'},
{'key':'prepaid_cards', 'title':'Prepaid card', 'css':'prepaid-card', 'icon': 'prepaid-cards'},
{'key':'student_loans', 'title':'Student loan', 'css':'student-loan', 'icon': 'paying-college'},
{'key':'other_consumer_loans', 'title':'Vehicle / consumer loan', 'css':'consumer-loan', 'icon': 'buying-car'}
{'key': 'bank_accounts',
'title': 'Bank account',
'css': 'bank-account',
'icon': 'bank-account'},
{'key': 'credit_cards',
'title': 'Credit card',
'css': 'credit-card',
'icon': 'credit-card'},
{'key': 'credit_reporting',
'title': 'Credit reporting',
'css': 'credit-reporting',
'icon': 'loan'},
{'key': 'debt_collection',
'title': 'Debt collection',
'css': 'debt-collection',
'icon': 'debt-collection'},
{'key': 'money_transfers',
'title': 'Money transfer or virtual currency',
'css': 'money-transfer',
'icon': 'money-transfer',
'tooltip': 'Money transfer/virtual currency'},
{'key': 'mortgages',
'title': 'Mortgage',
'css': 'mortgage',
'icon': 'owning-home'},
{'key': 'other_financial_services',
'title': 'Other financial service',
'css': 'other',
'icon': 'money'},
{'key': 'payday_loans',
'title': 'Payday loan',
'css': 'payday-loan',
'icon': 'payday-loan'},
{'key': 'prepaid_cards',
'title': 'Prepaid card',
'css': 'prepaid-card',
'icon': 'prepaid-cards'},
{'key': 'student_loans',
'title': 'Student loan',
'css': 'student-loan',
'icon': 'paying-college'},
{'key': 'other_consumer_loans',
'title': 'Vehicle / consumer loan',
'css': 'consumer-loan',
'icon': 'buying-car'}
]

try:
try:

for index, item in enumerate(narrative_types):
# get json data for this type
narrative = res_json[item['key']]

# extend it with the additional title/css/icon/tooltip data
narrative.update(item)

# format date
narrative['date'] = datetime.strptime(narrative['date_received'], "%Y-%m-%dT%H:%M:%S")
narrative['date'] = datetime.strptime(narrative['date_received'],
"%Y-%m-%dT%H:%M:%S")

# add data for next item
narrative['next'] = narrative_types[(index + 1) % len(narrative_types)]
narrative['next'] = narrative_types[(index + 1) %
len(narrative_types)]

narratives.append(narrative)

except KeyError as e:
print("format_narratives:KeyError")
print("There is problem accessing with the given key, which probably means the json has missing data")
print("There is problem accessing with the given key, "
"which probably means the json has missing data")
print(e)

return narratives


def get_stats(res_json):
res_stat = {}
try:
res_stat = res_json['stats']
except KeyError as e:
print("get_stats:KeyError")
print("There is problem accessing with the given key, which probably means the json has missing data")
print("There is problem accessing with the given key, "
"which probably means the json has missing data")
print(e)

return res_stat


def get_count_info():
total_complaints = 0
timely_responses = 0
try:
count_response = requests.get('https://data.consumerfinance.gov/resource/u473-64qt.json')
count_response = requests.get('https://data.consumerfinance.gov/'
'resource/u473-64qt.json')
count_json = json.loads(count_response.text)

for item in count_json:
Expand All @@ -134,39 +188,46 @@ def get_count_info():

except ValueError as e:
print("get_count_info:ValueError")
print("The text from the response doesn't follow the correct format to be parse as json")
print("The text from the response doesn't follow "
"the correct format to be parsed as json.")
print(e)

except KeyError as e:
print("get_count_info:KeyError")
print("There is problem accessing with the given key, which probably means the json has missing data")
print("There is problem accessing with the given key, "
"which probably means the json has missing data")
print(e)
total_complaints = 0
timely_responses = 0

return (total_complaints, timely_responses)


def get_now():
return datetime.now()


def is_data_not_updated(res_json):
data_down = False
narratives_down = False
# show notification starting fifth business day data has not been updated
# M-Th, data needs to have been updated 6 days ago; F-S, preceding Monday
weekday = datetime.weekday(get_now())
delta = weekday if weekday > 3 else 6
four_business_days_ago = (get_now() - timedelta(delta)).strftime("%Y-%m-%d")

four_business_days_ago = (get_now() -
timedelta(delta)).strftime("%Y-%m-%d")

try:

if res_json['stats']['last_updated'] < four_business_days_ago:
if res_json['stats']['last_updated'] < four_business_days_ago:
data_down = True
elif res_json['stats']['last_updated_narratives'] < four_business_days_ago:
elif (res_json['stats']['last_updated_narratives'] <
four_business_days_ago):
narratives_down = True
except KeyError as e:
print("is_data_not_updated:KeyError")
print("There is problem accessing with the given key, which probably means the json has missing data")
print("There is problem accessing with the given key, "
"which probably means the json has missing data")
print(e)

return (data_down, narratives_down)
Loading

0 comments on commit c9b57f2

Please sign in to comment.