Skip to content
Permalink
Browse files

Move remaining functions from helpers.py which is not related to rela…

…tions

Tested-by: Travis
  • Loading branch information
vmiklos authored and Travis CI committed Dec 9, 2019
1 parent 68309ee commit 8faffba67f87d548d2e1eca442983e82632bdf4e
Showing with 177 additions and 177 deletions.
  1. +1 −1 cron.py
  2. +6 −111 helpers.py
  3. +3 −3 tests/test_get_reference_housenumbers.py
  4. +3 −3 tests/test_get_reference_streets.py
  5. +12 −55 tests/test_helpers.py
  6. +43 −0 tests/test_util.py
  7. +105 −0 util.py
  8. +4 −4 wsgi.py
@@ -128,7 +128,7 @@ def main() -> None:
config.read(config_path)

datadir = util.get_abspath("data")
workdir = helpers.get_workdir(config)
workdir = util.get_workdir(config)
relations = helpers.Relations(datadir, workdir)
logpath = os.path.join(workdir, "cron.log")
logging.basicConfig(filename=logpath,
@@ -6,8 +6,6 @@

"""The helpers module contains functionality shared between other modules."""

import configparser
import re
import os
from typing import Any
from typing import Dict
@@ -439,8 +437,8 @@ def get_missing_housenumbers(
for street_name in street_names:
ref_house_numbers = all_ref_house_numbers[street_name]
osm_house_numbers = self.get_osm_housenumbers(street_name)
only_in_reference = get_only_in_first(ref_house_numbers, osm_house_numbers)
in_both = get_in_both(ref_house_numbers, osm_house_numbers)
only_in_reference = util.get_only_in_first(ref_house_numbers, osm_house_numbers)
in_both = util.get_in_both(ref_house_numbers, osm_house_numbers)
if only_in_reference:
ongoing_streets.append((street_name, only_in_reference))
if in_both:
@@ -502,9 +500,9 @@ def get_missing_streets(self) -> Tuple[List[str], List[str]]:
street_blacklist = self.get_config().get_street_filters()
osm_streets = [self.get_ref_street_from_osm_street(street) for street in self.get_osm_streets()]

only_in_reference = get_only_in_first(reference_streets, osm_streets)
only_in_reference = util.get_only_in_first(reference_streets, osm_streets)
only_in_reference = [i for i in only_in_reference if i not in street_blacklist]
in_both = get_in_both(reference_streets, osm_streets)
in_both = util.get_in_both(reference_streets, osm_streets)

return only_in_reference, in_both

@@ -609,109 +607,6 @@ def reftelepules_get_name(self, refmegye_name: str, reftelepules: str) -> str:
return cast(str, refmegye[reftelepules])


def get_only_in_first(first: List[Any], second: List[Any]) -> List[Any]:
"""
Returns items which are in first, but not in second.
Any means util.HouseNumber or str.
"""
# Strip suffix that is ignored.
if not first:
return []

if isinstance(first[0], util.HouseNumber):
first_stripped = [re.sub(r"\*$", "", i.get_number()) for i in first]
second_stripped = [re.sub(r"\*$", "", i.get_number()) for i in second]
else:
first_stripped = [re.sub(r"\*$", "", i) for i in first]
second_stripped = [re.sub(r"\*$", "", i) for i in second]

ret = []
for index, item in enumerate(first_stripped):
if item not in second_stripped:
ret.append(first[index])
return ret


def get_in_both(first: List[Any], second: List[Any]) -> List[Any]:
"""
Returns items which are in both first and second.
Any means util.HouseNumber or str.
"""
# Strip suffix that is ignored.
if not first:
return []

if isinstance(first[0], util.HouseNumber):
first_stripped = [re.sub(r"\*$", "", i.get_number()) for i in first]
second_stripped = [re.sub(r"\*$", "", i.get_number()) for i in second]
else:
first_stripped = [re.sub(r"\*$", "", i) for i in first]
second_stripped = [re.sub(r"\*$", "", i) for i in second]

ret = []
for index, item in enumerate(first_stripped):
if item in second_stripped:
ret.append(first[index])
return ret


def get_workdir(config: configparser.ConfigParser) -> str:
"""Gets the directory which is writable."""
return util.get_abspath(config.get('wsgi', 'workdir').strip())


def get_content(workdir: str, path: str = "") -> str:
"""Gets the content of a file in workdir."""
ret = ""
if path:
path = os.path.join(workdir, path)
else:
path = workdir
with open(path) as sock:
ret = sock.read()
return ret


def get_normalizer(street_name: str, normalizers: Dict[str, ranges.Ranges]) -> ranges.Ranges:
"""Determines the normalizer for a given street."""
if street_name in normalizers.keys():
# Have a custom filter.
normalizer = normalizers[street_name]
else:
# Default sanity checks.
default = [ranges.Range(1, 999), ranges.Range(2, 998)]
normalizer = ranges.Ranges(default)
return normalizer


def split_house_number_by_separator(
house_numbers: str,
separator: str,
normalizer: ranges.Ranges
) -> Tuple[List[int], List[int]]:
"""Splits a house number string (possibly a range) by a given separator.
Returns a filtered and a not filtered list of ints."""
ret_numbers = []
# Same as ret_numbers, but if the range is 2-6 and we filter for 2-4, then 6 would be lost, so
# in-range 4 would not be detected, so this one does not drop 6.
ret_numbers_nofilter = []

for house_number in house_numbers.split(separator):
try:
number = int(re.sub(r"([0-9]+).*", r"\1", house_number))
except ValueError:
continue

ret_numbers_nofilter.append(number)

if number not in normalizer:
continue

ret_numbers.append(number)

return ret_numbers, ret_numbers_nofilter


def normalize(relation: Relation, house_numbers: str, street_name: str,
normalizers: Dict[str, ranges.Ranges]) -> List[util.HouseNumber]:
"""Strips down string input to bare minimum that can be interpreted as an
@@ -726,9 +621,9 @@ def normalize(relation: Relation, house_numbers: str, street_name: str,
if house_numbers.endswith("*"):
suffix = house_numbers[-1]

normalizer = get_normalizer(street_name, normalizers)
normalizer = util.get_normalizer(street_name, normalizers)

ret_numbers, ret_numbers_nofilter = split_house_number_by_separator(house_numbers, separator, normalizer)
ret_numbers, ret_numbers_nofilter = util.split_house_number_by_separator(house_numbers, separator, normalizer)

street_is_even_odd = relation.get_config().get_street_is_even_odd(street_name)
if separator == "-" and util.should_expand_range(ret_numbers_nofilter, street_is_even_odd):
@@ -9,8 +9,8 @@
import os
import unittest
import unittest.mock
import helpers
import get_reference_housenumbers
import util


class TestMain(unittest.TestCase):
@@ -22,13 +22,13 @@ def get_abspath(path: str) -> str:
return path
return os.path.join(os.path.dirname(__file__), path)
with unittest.mock.patch('util.get_abspath', get_abspath):
expected = helpers.get_content(get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))
expected = util.get_content(get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))

argv = ["", "gazdagret"]
with unittest.mock.patch('sys.argv', argv):
get_reference_housenumbers.main()

actual = helpers.get_content(get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))
actual = util.get_content(get_abspath("workdir/street-housenumbers-reference-gazdagret.lst"))
self.assertEqual(actual, expected)


@@ -10,7 +10,7 @@
import unittest
import unittest.mock
import get_reference_streets
import helpers
import util


class TestMain(unittest.TestCase):
@@ -22,13 +22,13 @@ def get_abspath(path: str) -> str:
return path
return os.path.join(os.path.dirname(__file__), path)
with unittest.mock.patch('util.get_abspath', get_abspath):
expected = helpers.get_content(get_abspath("workdir/streets-reference-gazdagret.lst"))
expected = util.get_content(get_abspath("workdir/streets-reference-gazdagret.lst"))

argv = ["", "gazdagret"]
with unittest.mock.patch('sys.argv', argv):
get_reference_streets.main()

actual = helpers.get_content(get_abspath("workdir/streets-reference-gazdagret.lst"))
actual = util.get_content(get_abspath("workdir/streets-reference-gazdagret.lst"))
self.assertEqual(actual, expected)


@@ -6,7 +6,6 @@

"""The test_helpers module covers the helpers module."""

import configparser
import os
from typing import List
import unittest
@@ -25,20 +24,6 @@ def get_relations() -> helpers.Relations:
return helpers.Relations(datadir, workdir)


class TestInBoth(unittest.TestCase):
"""Tests get_in_both()."""
def test_happy(self) -> None:
"""Tests that happy path."""
self.assertEqual(helpers.get_in_both(["1", "2", "3"], ["2", "3", "4"]), ["2", "3"])


class TestOnlyInFirst(unittest.TestCase):
"""Tests get_only_in_first()."""
def test_happy(self) -> None:
"""Tests the happy path."""
self.assertEqual(helpers.get_only_in_first(["1", "2", "3"], ["3", "4"]), ["1", "2"])


class TestRelationGetOsmStreets(unittest.TestCase):
"""Tests Relation.get_osm_streets()."""
def test_happy(self) -> None:
@@ -58,17 +43,6 @@ def test_no_house_number(self) -> None:
self.assertEqual(actual, expected)


class TestGetWorkdir(unittest.TestCase):
"""Tests get_workdir()."""
def test_happy(self) -> None:
"""Tests the happy path."""
config = configparser.ConfigParser()
config.read_dict({"wsgi": {"workdir": "/path/to/workdir"}})
actual = helpers.get_workdir(config)
expected = "/path/to/workdir"
self.assertEqual(actual, expected)


class TestRelationGetOsmStreetsQuery(unittest.TestCase):
"""Tests Relation.get_osm_streets_query()."""
def test_happy(self) -> None:
@@ -100,9 +74,9 @@ def test_happy(self) -> None:
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
result_from_overpass = "@id\tname\n1\tTűzkő utca\n2\tTörökugrató utca\n3\tOSM Name 1\n4\tHamzsabégi út\n"
expected = helpers.get_content(relations.get_workdir(), "streets-gazdagret.csv")
expected = util.get_content(relations.get_workdir(), "streets-gazdagret.csv")
relation.get_files().write_osm_streets(result_from_overpass)
actual = helpers.get_content(relations.get_workdir(), "streets-gazdagret.csv")
actual = util.get_content(relations.get_workdir(), "streets-gazdagret.csv")
self.assertEqual(actual, expected)


@@ -120,27 +94,10 @@ def test_happy(self) -> None:
result_from_overpass += "1\tOSM Name 1\t1\n"
result_from_overpass += "1\tOSM Name 1\t2\n"
result_from_overpass += "1\tOnly In OSM utca\t1\n"
expected = helpers.get_content(relations.get_workdir(), "street-housenumbers-gazdagret.csv")
expected = util.get_content(relations.get_workdir(), "street-housenumbers-gazdagret.csv")
relation = relations.get_relation(relation_name)
relation.get_files().write_osm_housenumbers(result_from_overpass)
actual = helpers.get_content(relations.get_workdir(), "street-housenumbers-gazdagret.csv")
self.assertEqual(actual, expected)


class TestGetContent(unittest.TestCase):
"""Tests get_content()."""
def test_happy(self) -> None:
"""Tests the happy path."""
workdir = os.path.join(os.path.dirname(__file__), "workdir")
actual = helpers.get_content(workdir, "gazdagret.percent")
expected = "54.55"
self.assertEqual(actual, expected)

def test_one_arg(self) -> None:
"""Tests the case when only one argument is given."""
workdir = os.path.join(os.path.dirname(__file__), "workdir")
actual = helpers.get_content(os.path.join(workdir, "gazdagret.percent"))
expected = "54.55"
actual = util.get_content(relations.get_workdir(), "street-housenumbers-gazdagret.csv")
self.assertEqual(actual, expected)


@@ -483,7 +440,7 @@ def test_happy(self) -> None:
relations = get_relations()
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
expected = helpers.get_content(relations.get_workdir(), "gazdagret.percent")
expected = util.get_content(relations.get_workdir(), "gazdagret.percent")
ret = relation.write_missing_housenumbers()
todo_street_count, todo_count, done_count, percent, table = ret
self.assertEqual(todo_street_count, 3)
@@ -495,7 +452,7 @@ def test_happy(self) -> None:
['Törökugrató utca', '2', '7<br />10'],
['Tűzkő utca', '2', '1<br />2'],
['Hamzsabégi út', '1', '1']])
actual = helpers.get_content(relations.get_workdir(), "gazdagret.percent")
actual = util.get_content(relations.get_workdir(), "gazdagret.percent")
self.assertEqual(actual, expected)

def test_empty(self) -> None:
@@ -529,14 +486,14 @@ def test_happy(self) -> None:
relations = get_relations()
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
expected = helpers.get_content(relations.get_workdir(), "gazdagret-streets.percent")
expected = util.get_content(relations.get_workdir(), "gazdagret-streets.percent")
ret = relation.write_missing_streets()
todo_count, done_count, percent, streets = ret
self.assertEqual(todo_count, 1)
self.assertEqual(done_count, 4)
self.assertEqual(percent, '80.00')
self.assertEqual(streets, ['Only In Ref utca'])
actual = helpers.get_content(relations.get_workdir(), "gazdagret-streets.percent")
actual = util.get_content(relations.get_workdir(), "gazdagret-streets.percent")
self.assertEqual(actual, expected)

def test_empty(self) -> None:
@@ -613,10 +570,10 @@ def test_happy(self) -> None:
refpath2 = os.path.join(refdir, "hazszamok_kieg_20190808.tsv")
relations = get_relations()
relation_name = "gazdagret"
expected = helpers.get_content(relations.get_workdir(), "street-housenumbers-reference-gazdagret.lst")
expected = util.get_content(relations.get_workdir(), "street-housenumbers-reference-gazdagret.lst")
relation = relations.get_relation(relation_name)
relation.write_ref_housenumbers([refpath, refpath2])
actual = helpers.get_content(relations.get_workdir(), "street-housenumbers-reference-gazdagret.lst")
actual = util.get_content(relations.get_workdir(), "street-housenumbers-reference-gazdagret.lst")
self.assertEqual(actual, expected)

def test_nosuchrefmegye(self) -> None:
@@ -653,9 +610,9 @@ def test_happy(self) -> None:
relations = get_relations()
relation_name = "gazdagret"
relation = relations.get_relation(relation_name)
expected = helpers.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
expected = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
relation.write_ref_streets(refpath)
actual = helpers.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
actual = util.get_content(relations.get_workdir(), "streets-reference-gazdagret.lst")
self.assertEqual(actual, expected)


0 comments on commit 8faffba

Please sign in to comment.
You can’t perform that action at this time.