From e23893a57d4f2f8bb96eba721168fcb5b989357b Mon Sep 17 00:00:00 2001 From: Vanessa Qian Date: Mon, 18 Jun 2018 10:32:43 -0400 Subject: [PATCH 1/7] officeid filter; closes #320 --- CHANGELOG.rst | 7 ++++++- docs/cli.rst | 3 ++- elex/api/models.py | 7 +++++++ elex/api/utils.py | 1 - elex/cli/app.py | 5 +++++ elex/cli/hooks.py | 12 +++++++++++- tests/__init__.py | 42 ++++++++++++++++++++++++++++++++++++++++ tests/test_ap_network.py | 18 +++++++++++++++++ 8 files changed, 91 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd86c77..9a309a3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +2.4.1 (hold) - June 13, 2018 +---------------------------- + +* Adds a :code:`officeids` feature. :code:`elex races 2016-11-08 --officeids P,H,S,G` passes the :code:`officeids` as params in the API request to retrieve data for specific offices. Particularly useful when raceid is unknown. + 2.4.0 - October 23, 2016 ------------------------ @@ -41,7 +46,7 @@ Fixes a bug related to national / local flags on races. Running :code:`--local-o * Adds a :code:`raceids` feature. :code:`elex races 2016-03-15 --raceids 10675,14897` still downloads the full JSON file but only parses the races passed in the :code:`raceids` argument. Particularly effective when used with the :code:`local-only` flag to grab a subset of non-national races, e.g., every NY state race. 2.0.5 - 2.0.6 - June 6, 2016 ---------------------- +---------------------------- * Fixes a small bug in the ME reporting for the upcoming 6-14 primary. 2.0.1 - 2.0.4 - April 26, 2016 diff --git a/docs/cli.rst b/docs/cli.rst index 794d52f..16e9b24 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -88,7 +88,8 @@ Commands and flags --batch-name BATCH_NAME Specify a value for a `batchname` column to append to each row. - + --officeids OFFICEIDS Specify officeids to parse + ----------------- Command reference ----------------- diff --git a/elex/api/models.py b/elex/api/models.py index 4e4a3c5..5dc9a19 100644 --- a/elex/api/models.py +++ b/elex/api/models.py @@ -876,6 +876,7 @@ def __init__(self, **kwargs): self.setzerocounts = kwargs.get('setzerocounts', False) self.raceids = kwargs.get('raceids', []) + self.officeids = kwargs.get('officeids', "") self.set_id_field() @@ -1049,8 +1050,10 @@ def races(self): level="ru", test=self.testresults, national=self.national, + officeID=self.officeids, apiKey=self.api_key ) + race_objs = self.get_race_objects(raw_races) races, reporting_units, candidate_reporting_units = self.get_units( race_objs @@ -1067,6 +1070,7 @@ def reporting_units(self): level="ru", test=self.testresults, national=self.national, + officeID=self.officeids, apiKey=self.api_key ) race_objs = self.get_race_objects(raw_races) @@ -1085,6 +1089,7 @@ def candidate_reporting_units(self): level="ru", test=self.testresults, national=self.national, + officeID=self.officeids, apiKey=self.api_key ) race_objs = self.get_race_objects(raw_races) @@ -1104,6 +1109,7 @@ def results(self): setzerocounts=self.setzerocounts, test=self.testresults, national=self.national, + officeID=self.officeids, apiKey=self.api_key ) race_objs = self.get_race_objects(raw_races) @@ -1122,6 +1128,7 @@ def candidates(self): level="ru", test=self.testresults, national=self.national, + officeID=self.officeids, apiKey=self.api_key ) race_objs = self.get_race_objects(raw_races) diff --git a/elex/api/utils.py b/elex/api/utils.py index b2eb31a..6f3c07f 100644 --- a/elex/api/utils.py +++ b/elex/api/utils.py @@ -98,7 +98,6 @@ def api_request(path, **params): params = sorted(params.items()) # Sort for consistent caching url = '{0}{1}'.format(elex.BASE_URL, path.replace('//', '/')) - response = cache.get(url, params=params) response.raise_for_status() diff --git a/elex/cli/app.py b/elex/cli/app.py index 6637e6f..2e419db 100644 --- a/elex/cli/app.py +++ b/elex/cli/app.py @@ -90,6 +90,11 @@ class Meta: action='store', help='Specify a value for a `batchname` column to append to each row.', )), + (['--officeids'], dict( + action='store', + help='Specify officeids to parse', + default=[] + )), ] @expose(hide=True) diff --git a/elex/cli/hooks.py b/elex/cli/hooks.py index 39c0ca5..71608dc 100644 --- a/elex/cli/hooks.py +++ b/elex/cli/hooks.py @@ -1,6 +1,7 @@ import logging from elex.api import Election from elex.cli.constants import LOG_FORMAT +from elex.api import maps def add_election_hook(app): @@ -13,7 +14,8 @@ def add_election_hook(app): resultslevel=app.pargs.results_level, setzerocounts=app.pargs.set_zero_counts, is_test=False, - raceids=[] + raceids=[], + officeids="" ) if app.pargs.data_file: @@ -28,6 +30,14 @@ def add_election_hook(app): if app.pargs.raceids: app.election.raceids = [x.strip() for x in app.pargs.raceids.split(',')] + if app.pargs.officeids: + invalid_officeids = [x.strip() for x in app.pargs.officeids.split(',') if x.strip() not in maps.OFFICE_NAMES] + if invalid_officeids: + text = '{0} is/are invalid officeID(s). Here is a list of valid officeIDs: {1}' + app.log.error(text.format(", ".join(invalid_officeids), ", ".join(maps.OFFICE_NAMES.keys()))) + app.close(1) + app.election.officeids = app.pargs.officeids + def cachecontrol_logging_hook(app): """ diff --git a/tests/__init__.py b/tests/__init__.py index f407cf8..3c84a2a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -29,6 +29,48 @@ def setUp(self, **kwargs): self.senate_trends = USSenateTrendReport(self.senate_file) +class ElectionResultsParseValidOfficeIdsTestCase(unittest.TestCase): + def setUp(self, **kwargs): + e = Election( + electiondate='2016-11-08', + testresults=False, + liveresults=True, + is_test=False, + officeids='P,H,G' + ) + self.election = e + self.resultslevel = e.resultslevel + self.raw_races = e.get_raw_races() + self.race_objs = e.get_race_objects(self.raw_races) + self.ballot_measures = e.ballot_measures + self.candidate_reporting_units = e.candidate_reporting_units + self.candidates = e.candidates + self.races = e.races + self.reporting_units = e.reporting_units + self.results = e.results + + +class ElectionResultsParseInvalidOfficeIdsTestCase(unittest.TestCase): + def setUp(self, **kwargs): + e = Election( + electiondate='2016-11-08', + testresults=False, + liveresults=True, + is_test=False, + officeids='N,ABC,PS' + ) + self.election = e + self.resultslevel = e.resultslevel + self.raw_races = e.get_raw_races() + self.race_objs = e.get_race_objects(self.raw_races) + self.ballot_measures = e.ballot_measures + self.candidate_reporting_units = e.candidate_reporting_units + self.candidates = e.candidates + self.races = e.races + self.reporting_units = e.reporting_units + self.results = e.results + + class ElectionResultsParseIdsTestCase(unittest.TestCase): data_url = 'tests/data/20151103_national.json' diff --git a/tests/test_ap_network.py b/tests/test_ap_network.py index 1f31815..aae123f 100644 --- a/tests/test_ap_network.py +++ b/tests/test_ap_network.py @@ -1,5 +1,6 @@ import os import unittest +import tests from elex.cli.app import ElexApp from requests.exceptions import HTTPError @@ -37,6 +38,23 @@ def test_nonexistent_param(self): self.assertEqual(response.status_code, 400) +class TestRaceResultsOfficeIdParsing(tests.ElectionResultsParseValidOfficeIdsTestCase): + + @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) + def test_officeid_number_of_races(self): + self.assertEqual(len(self.races), 536) + + @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) + def test_officeid_number_of_results(self): + self.assertEqual(len(self.results), 55601) + + +class TestRaceResultsInvalidOfficeIdParsing(tests.ElectionResultsParseInvalidOfficeIdsTestCase): + @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) + def test_invalid_officeid_number_of_races(self): + self.assertEqual(len(self.races), 14) + + class ElexNetworkCacheTestCase(NetworkTestCase): @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) def test_elex_cache_miss(self): From 7610c9b3b7b6c24f08007c10a69839b2830f4623 Mon Sep 17 00:00:00 2001 From: Miles Wingfield Watkins Date: Mon, 18 Jun 2018 10:56:07 -0400 Subject: [PATCH 2/7] Finish updating mentions of 3.5 to 3.6 --- Dockerfile | 2 +- docs/conf.py | 2 +- docs/contributing.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb1f9f6..0a38024 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.5 +FROM python:3.6 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app COPY . /usr/src/app diff --git a/docs/conf.py b/docs/conf.py index a88ba9c..ed7f263 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ autodoc_member_order = 'bysource' intersphinx_mapping = { - 'python': ('http://docs.python.org/3.5', None) + 'python': ('http://docs.python.org/3.6', None) } # Templates diff --git a/docs/contributing.rst b/docs/contributing.rst index 59da0f5..1e7f3f0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -45,7 +45,7 @@ Make sure all tests are passing in your environment by running the nose2 tests. make test -If you have Python 2.7, 3.5, and pypy installed, run can run :code:`tox` to test in multiple environments. +If you have Python 2.7, 3.6, and pypy installed, run can run :code:`tox` to test in multiple environments. Writing docs ============ From 122944b0931aa3c6a7539fc3bd5766b6d73bfc71 Mon Sep 17 00:00:00 2001 From: Miles Wingfield Watkins Date: Mon, 18 Jun 2018 11:27:26 -0400 Subject: [PATCH 3/7] Update tox to use test makefile and Python 3.6 --- tox.ini | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index b540b2a..2225eee 100644 --- a/tox.ini +++ b/tox.ini @@ -4,18 +4,15 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, pypy +envlist = py27, py36, pypy [testenv] deps = -r{toxinidir}/requirements-dev.txt +whitelist_externals = make -commands = pyflakes elex - pep8 elex - pyflakes tests - pep8 tests - nose2 tests +commands = make test -[pep8] +[flake8] # E731: Ignore the lambda def errors since they are an excusable UnicodeMixin hack # E501: Ignore line length errors. There are long lines in docstrings that are perfectly cromulent. ignore = E731,E501 From c3bfc1bdb17ac144cd8407d47566f73f46a55ee6 Mon Sep 17 00:00:00 2001 From: Miles Wingfield Watkins Date: Mon, 18 Jun 2018 11:49:31 -0400 Subject: [PATCH 4/7] Remove flake8 exceptions from the tox configuration, since they're now in setup.cfg --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index 2225eee..b3a931d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,3 @@ deps = -r{toxinidir}/requirements-dev.txt whitelist_externals = make commands = make test - -[flake8] -# E731: Ignore the lambda def errors since they are an excusable UnicodeMixin hack -# E501: Ignore line length errors. There are long lines in docstrings that are perfectly cromulent. -ignore = E731,E501 From 9ec4af6bdf9bcbdacec31f4a2ba18265e7093fe7 Mon Sep 17 00:00:00 2001 From: Ben Welsh Date: Mon, 18 Jun 2018 08:54:55 -0700 Subject: [PATCH 5/7] Added pypy to Travis builds --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d70dcef..50e4cf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ language: python python: - "2.7" - "3.6" + - "pypy" install: - pip install -r requirements.txt From bea913dbe720db60aa9870d6c6d66c25004b46d3 Mon Sep 17 00:00:00 2001 From: Miles Wingfield Watkins Date: Mon, 18 Jun 2018 16:42:51 -0400 Subject: [PATCH 6/7] Clarify the logic around fetching the API key in a request --- elex/api/utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/elex/api/utils.py b/elex/api/utils.py index b2eb31a..491776f 100644 --- a/elex/api/utils.py +++ b/elex/api/utils.py @@ -84,12 +84,7 @@ def api_request(path, **params): `apiKey="`, your AP API key, or `national=True`, for national-only results. """ - if not params.get('apiKey', None): - if elex.API_KEY != '': - params['apiKey'] = elex.API_KEY - else: - params['apiKey'] = None - + params['apiKey'] = params.get('apiKey') or elex.API_KEY if not params['apiKey']: raise APAPIKeyException() From 63cad58cdb4e52c06dfc2dbcee90e7da5f869780 Mon Sep 17 00:00:00 2001 From: Vanessa Qian Date: Mon, 18 Jun 2018 18:23:48 -0400 Subject: [PATCH 7/7] add officeid filter, edits based on comments; closes #320 --- AUTHORS.rst | 1 + CHANGELOG.rst | 5 ----- docs/cli.rst | 6 +++--- elex/api/models.py | 2 +- elex/api/utils.py | 1 - elex/cli/app.py | 14 ++++++------- elex/cli/hooks.py | 8 +++++--- tests/__init__.py | 43 +--------------------------------------- tests/test_ap_network.py | 32 +++++++++++++++++++++++------- 9 files changed, 43 insertions(+), 69 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index b66d620..836498d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,3 +11,4 @@ These individuals have contributed code, tests, documentation, and troubleshooti * Ben Welsh * Tom Giratikanon * Ryan Pitts +* Vanessa Qian diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a309a3..74a3c46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,8 +1,3 @@ -2.4.1 (hold) - June 13, 2018 ----------------------------- - -* Adds a :code:`officeids` feature. :code:`elex races 2016-11-08 --officeids P,H,S,G` passes the :code:`officeids` as params in the API request to retrieve data for specific offices. Particularly useful when raceid is unknown. - 2.4.0 - October 23, 2016 ------------------------ diff --git a/docs/cli.rst b/docs/cli.rst index 16e9b24..4fa3f9e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -76,8 +76,9 @@ Commands and flags --format-json Pretty print JSON when using `-o json`. -v, --version show program's version number and exit --results-level RESULTS_LEVEL - Specify reporting level for results - --raceids RACEIDS Specify raceids to parse + Specify reporting level for results. + --officeids OFFICEIDS Specify officeids to parse. + --raceids RACEIDS Specify raceids to parse. --set-zero-counts Override results with zeros; omits the winner indicator.Sets the vote, delegate, and reporting precinct counts to zero. @@ -88,7 +89,6 @@ Commands and flags --batch-name BATCH_NAME Specify a value for a `batchname` column to append to each row. - --officeids OFFICEIDS Specify officeids to parse ----------------- Command reference diff --git a/elex/api/models.py b/elex/api/models.py index 5dc9a19..7ab73a6 100644 --- a/elex/api/models.py +++ b/elex/api/models.py @@ -876,7 +876,7 @@ def __init__(self, **kwargs): self.setzerocounts = kwargs.get('setzerocounts', False) self.raceids = kwargs.get('raceids', []) - self.officeids = kwargs.get('officeids', "") + self.officeids = kwargs.get('officeids', None) self.set_id_field() diff --git a/elex/api/utils.py b/elex/api/utils.py index 6f3c07f..5d07fcd 100644 --- a/elex/api/utils.py +++ b/elex/api/utils.py @@ -102,7 +102,6 @@ def api_request(path, **params): response.raise_for_status() write_recording(response.json()) - return response diff --git a/elex/cli/app.py b/elex/cli/app.py index 2e419db..f9071b1 100644 --- a/elex/cli/app.py +++ b/elex/cli/app.py @@ -57,12 +57,17 @@ class Meta: )), (['--results-level'], dict( action='store', - help='Specify reporting level for results', + help='Specify reporting level for results.', default='ru' )), + (['--officeids'], dict( + action='store', + help='Specify officeids to parse.', + default=None + )), (['--raceids'], dict( action='store', - help='Specify raceids to parse', + help='Specify raceids to parse.', default=[] )), (['--set-zero-counts'], dict( @@ -90,11 +95,6 @@ class Meta: action='store', help='Specify a value for a `batchname` column to append to each row.', )), - (['--officeids'], dict( - action='store', - help='Specify officeids to parse', - default=[] - )), ] @expose(hide=True) diff --git a/elex/cli/hooks.py b/elex/cli/hooks.py index 71608dc..ffc85a5 100644 --- a/elex/cli/hooks.py +++ b/elex/cli/hooks.py @@ -15,7 +15,7 @@ def add_election_hook(app): setzerocounts=app.pargs.set_zero_counts, is_test=False, raceids=[], - officeids="" + officeids=None ) if app.pargs.data_file: @@ -31,12 +31,14 @@ def add_election_hook(app): app.election.raceids = [x.strip() for x in app.pargs.raceids.split(',')] if app.pargs.officeids: - invalid_officeids = [x.strip() for x in app.pargs.officeids.split(',') if x.strip() not in maps.OFFICE_NAMES] + invalid_officeids = [x for x in app.pargs.officeids.split(',') if x not in maps.OFFICE_NAMES] if invalid_officeids: text = '{0} is/are invalid officeID(s). Here is a list of valid officeIDs: {1}' app.log.error(text.format(", ".join(invalid_officeids), ", ".join(maps.OFFICE_NAMES.keys()))) app.close(1) - app.election.officeids = app.pargs.officeids + else: + app.election.officeids = app.pargs.officeids + # kept as a comma-delimited string so officeID as a param always appears once in request url (e.g. officeID=P%2CH%2CG) def cachecontrol_logging_hook(app): diff --git a/tests/__init__.py b/tests/__init__.py index 3c84a2a..d9180e3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -29,48 +29,6 @@ def setUp(self, **kwargs): self.senate_trends = USSenateTrendReport(self.senate_file) -class ElectionResultsParseValidOfficeIdsTestCase(unittest.TestCase): - def setUp(self, **kwargs): - e = Election( - electiondate='2016-11-08', - testresults=False, - liveresults=True, - is_test=False, - officeids='P,H,G' - ) - self.election = e - self.resultslevel = e.resultslevel - self.raw_races = e.get_raw_races() - self.race_objs = e.get_race_objects(self.raw_races) - self.ballot_measures = e.ballot_measures - self.candidate_reporting_units = e.candidate_reporting_units - self.candidates = e.candidates - self.races = e.races - self.reporting_units = e.reporting_units - self.results = e.results - - -class ElectionResultsParseInvalidOfficeIdsTestCase(unittest.TestCase): - def setUp(self, **kwargs): - e = Election( - electiondate='2016-11-08', - testresults=False, - liveresults=True, - is_test=False, - officeids='N,ABC,PS' - ) - self.election = e - self.resultslevel = e.resultslevel - self.raw_races = e.get_raw_races() - self.race_objs = e.get_race_objects(self.raw_races) - self.ballot_measures = e.ballot_measures - self.candidate_reporting_units = e.candidate_reporting_units - self.candidates = e.candidates - self.races = e.races - self.reporting_units = e.reporting_units - self.results = e.results - - class ElectionResultsParseIdsTestCase(unittest.TestCase): data_url = 'tests/data/20151103_national.json' @@ -144,6 +102,7 @@ def setUp(self, **kwargs): class NetworkTestCase(unittest.TestCase): + def api_request(self, *args, **kwargs): response = utils.api_request(*args, **kwargs) sleep(10) diff --git a/tests/test_ap_network.py b/tests/test_ap_network.py index aae123f..aa8f63d 100644 --- a/tests/test_ap_network.py +++ b/tests/test_ap_network.py @@ -38,24 +38,42 @@ def test_nonexistent_param(self): self.assertEqual(response.status_code, 400) -class TestRaceResultsOfficeIdParsing(tests.ElectionResultsParseValidOfficeIdsTestCase): +class TestRaceResultsOfficeIdParsing(NetworkTestCase): @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) def test_officeid_number_of_races(self): - self.assertEqual(len(self.races), 536) + valid_officeids = self.api_request('/elections/2016-11-08/', officeID='P,H,G') + data = valid_officeids.json() + self.assertEqual(len(data['races']), 502) @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) - def test_officeid_number_of_results(self): - self.assertEqual(len(self.results), 55601) + def test_races_with_officeids_vs_no_officeids(self): + w_officeids = self.api_request('/elections/2016-11-08/', officeID='P,H') + all_races = self.api_request('/elections/2016-11-08/') + data_w_officeids = w_officeids.json() + data_all = all_races.json() + self.assertLess(len(data_w_officeids['races']), len(data_all['races'])) + raceids_filter_ph_in_all = [elem['raceID'] for elem in data_all['races'] if elem['officeID'] == 'P' or elem['officeID'] == 'H'] + raceids_w_officeids = [elem['raceID'] for elem in data_w_officeids['races']] + self.assertEqual(raceids_filter_ph_in_all, raceids_w_officeids) -class TestRaceResultsInvalidOfficeIdParsing(tests.ElectionResultsParseInvalidOfficeIdsTestCase): @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) - def test_invalid_officeid_number_of_races(self): - self.assertEqual(len(self.races), 14) + def test_raceid_zero_with_officeid_p(self): + raceid_req = self.api_request('/elections/2016-11-08/') + officeid_req = self.api_request('/elections/2016-11-08/', officeID='P') + data_raceid = raceid_req.json() + data_officeid = officeid_req.json() + + len_data_raceid_zero = sum([1 for elem in data_raceid['races'] if elem['raceID'] == '0']) + self.assertEqual(len(data_officeid['races']), len_data_raceid_zero) + + len_data_officeid_zero = sum([1 for elem in data_officeid['races'] if elem['raceID'] == '0']) + self.assertEqual(len_data_officeid_zero, len_data_raceid_zero) class ElexNetworkCacheTestCase(NetworkTestCase): + @unittest.skipUnless(os.environ.get('AP_API_KEY', None), API_MESSAGE) def test_elex_cache_miss(self): from elex import cache