Skip to content
This repository has been archived by the owner on Feb 2, 2022. It is now read-only.

Commit

Permalink
Fix double deceased bug
Browse files Browse the repository at this point in the history
Prevents the parser to crash if the deceased details cannot be found.

Drive-by:
* Updates to python 3.8
  • Loading branch information
rgreinho committed Jul 10, 2020
1 parent 8e92702 commit 2ebc36d
Show file tree
Hide file tree
Showing 5 changed files with 1,305 additions and 4 deletions.
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -18,6 +18,7 @@
.idea/
.installed.cfg
.nox/
.python-version
.tox/
.vscode/
__pycache__/
Expand Down
6 changes: 3 additions & 3 deletions noxfile.py
Expand Up @@ -88,23 +88,23 @@ def pylint(session):
run_pylint(session)


@nox.session(python='python3.7')
@nox.session(python='python3.8')
def test(session):
"""Run all the tests."""
session.install('-rrequirements-dev.txt')
session.install('-e', '.')
run_pytest(session)


@nox.session(python='python3.7', name='test-units')
@nox.session(python='python3.8', name='test-units')
def test_units(session):
"""Run the unit tests."""
session.install('-rrequirements-dev.txt')
session.install('-e', '.')
run_pytest_units(session)


@nox.session(python='python3.7', name='test-integrations')
@nox.session(python='python3.8', name='test-integrations')
def test_integrations(session):
"""Run the integration tests."""
session.install('-rrequirements-dev.txt')
Expand Down
4 changes: 3 additions & 1 deletion scrapd/core/article.py
Expand Up @@ -61,7 +61,9 @@ def parse_deceased_tag(deceased_tag_p):
deceased_text = deceased_tag_p.get_text()
# NOTE(rgreinho): Where do the 20 and 100 numbers come from? There should not be magic numbers.
if 20 < len(deceased_text) < 100 and "preliminary" not in deceased_text:
return re.split(r'Deceased(?: \d)?:', deceased_text, maxsplit=1)[1].strip()
split_text = re.split(r'Deceased(?: \d)?:', deceased_text, maxsplit=1)
if len(split_text) > 1:
return split_text[1].strip()

deceased_field_str = ''
starting_tag = deceased_tag_p.find("strong") or deceased_tag_p
Expand Down
16 changes: 16 additions & 0 deletions tests/core/test_article.py
Expand Up @@ -143,6 +143,22 @@
),
'errors': None,
},
{
'id': 'double-deceased',
'page': 'fatality-crash-41-2',
'expected': model.Report(
case='20-1850221',
crash=41,
date=datetime.date(2020, 7, 3),
fatalities=[
model.Fatality(),
],
location='9600 Block of E. U.S. 290 eastbound',
notes='',
time=datetime.time(5, 49),
),
'errors': 2,
},
]

deceased_tag_scenarios = [
Expand Down

0 comments on commit 2ebc36d

Please sign in to comment.