Skip to content

Commit

Permalink
Agrego defaults de nueva parametrizacion en validacion de catalogos, …
Browse files Browse the repository at this point in the history 
…arreglando tests
  • Loading branch information
@FScaccheri
FScaccheri committed Nov 27, 2019
1 parent dcbd391 commit c7bac62
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 21 deletions.
1 change: 0 additions & 1 deletion pydatajson/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,4 @@
INVALID_STATUS_CODES_REGEX = ["^4[0-9]+$", "^5[0-9]+$"]
EXCEPTION_STATUS_CODES = [429]

CANT_THREADS_BROKEN_URL_VALIDATOR = 10
DEFAULT_CHECK_TIMEOUT = 1
19 changes: 12 additions & 7 deletions pydatajson/validation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,12 @@ def init_jsonschema_validator(self, schema_dir, schema_filename):
schema=schema, resolver=resolver, format_checker=format_checker)

def is_valid(self, catalog, broken_links=False, verify_ssl=True,
url_check_timeout=1):
url_check_timeout=1, broken_links_threads=1):
return not self._get_errors(catalog,
broken_links=broken_links,
verify_ssl=verify_ssl,
url_check_timeout=url_check_timeout)
url_check_timeout=url_check_timeout,
broken_links_threads=broken_links_threads)

def validate_catalog(self, catalog, only_errors=False,
broken_links=False, verify_ssl=True,
Expand All @@ -83,17 +84,19 @@ def validate_catalog(self, catalog, only_errors=False,
return response

def _get_errors(self, catalog, broken_links=False, verify_ssl=True,
url_check_timeout=1):
url_check_timeout=1, broken_links_threads=1):
errors = list(
self.jsonschema_validator.iter_errors(catalog)
)
try:
for error in self._custom_errors(
catalog, broken_links=broken_links,
verify_ssl=verify_ssl,
url_check_timeout=url_check_timeout):
url_check_timeout=url_check_timeout,
broken_links_threads=broken_links_threads):
errors.append(error)
except Exception as e:
print(e)
logger.warning("Error de validación")
return errors

Expand Down Expand Up @@ -124,7 +127,7 @@ def _default_response(self, catalog):

# noinspection PyTypeChecker
def _custom_errors(self, catalog, broken_links=False, verify_ssl=True,
url_check_timeout=1):
url_check_timeout=1, broken_links_threads=1):
"""Realiza validaciones sin usar el jsonschema.
En esta función se agregan bloques de código en python que realizan
Expand All @@ -133,9 +136,11 @@ def _custom_errors(self, catalog, broken_links=False, verify_ssl=True,
validators = self._validators_for_catalog(catalog)
if broken_links:
validators.append(LandingPagesValidator(catalog, verify_ssl,
url_check_timeout))
url_check_timeout,
broken_links_threads))
validators.append(DistributionUrlsValidator(catalog, verify_ssl,
url_check_timeout))
url_check_timeout,
broken_links_threads))

for validator in validators:
for error in validator.validate():
Expand Down
26 changes: 13 additions & 13 deletions tests/test_validators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,27 +46,27 @@ def setUp(self):
self.tinr_validator = \
ThemeIdsNotRepeatedValidator(self.catalog)
self.ddu_validator = \
DistributionUrlsValidator(self.catalog, True, 1)
DistributionUrlsValidator(self.catalog, True, 1, 10)
self.lp_validator = \
LandingPagesValidator(self.catalog, True, 1)
LandingPagesValidator(self.catalog, True, 1, 10)

@requests_mock.Mocker()
def test_is_working_url_valid_url(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
req_mock.head(self.test_url)
self.assertEqual(
(True, 200), url_validator.is_working_url(self.test_url))

@requests_mock.Mocker()
def test_is_working_url_invalid_url(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
req_mock.head(self.test_url, status_code=400)
self.assertEqual(
(False, 400), url_validator.is_working_url(self.test_url))

@requests_mock.Mocker()
def test_is_working_url_too_many_requests_response(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
too_many_request_status_code = 429
req_mock.head(self.test_url,
status_code=too_many_request_status_code)
Expand All @@ -76,20 +76,20 @@ def test_is_working_url_too_many_requests_response(self, req_mock):

@requests_mock.Mocker()
def test_is_working_url_url_with_exception(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
req_mock.head(self.test_url, exc=ConnectionError)
self.assertEqual(
(False, None), url_validator.is_working_url(self.test_url))

@requests_mock.Mocker()
def test_is_working_url_url_with_timeout(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
req_mock.head(self.test_url, exc=Timeout)
self.assertEqual(
(False, 408), url_validator.is_working_url(self.test_url))

def test_is_working_url_malformed_values(self):
url_validator = UrlValidator(self.catalog, True, 1)
url_validator = UrlValidator(self.catalog, True, 1, 10)
self.assertEqual(
(False, None), url_validator.is_working_url('malformed_value'))
self.assertEqual(
Expand All @@ -99,7 +99,7 @@ def test_is_working_url_malformed_values(self):

def test_valid_landing_page_validator(self):
lp_validator = \
LandingPagesValidator(self.catalog, True, 1)
LandingPagesValidator(self.catalog, True, 1, 10)
with mock.patch(
'pydatajson'
'.validators'
Expand All @@ -110,7 +110,7 @@ def test_valid_landing_page_validator(self):

def test_invalid_landing_page_validator(self):
lp_validator = \
LandingPagesValidator(self.catalog, True, 1)
LandingPagesValidator(self.catalog, True, 1, 10)
with mock.patch(
'pydatajson'
'.validators'
Expand All @@ -121,7 +121,7 @@ def test_invalid_landing_page_validator(self):

def test_valid_distribution_url_validator(self):
ddu_validator = \
DistributionUrlsValidator(self.catalog, True, 1)
DistributionUrlsValidator(self.catalog, True, 1, 10)
with mock.patch(
'pydatajson'
'.validators'
Expand All @@ -132,7 +132,7 @@ def test_valid_distribution_url_validator(self):

def test_invalid_distribution_url_validator(self):
ddu_validator = \
DistributionUrlsValidator(self.catalog, True, 1)
DistributionUrlsValidator(self.catalog, True, 1, 10)
with mock.patch(
'pydatajson'
'.validators'
Expand Down Expand Up @@ -191,7 +191,7 @@ def test_invalid_theme_ids_not_repeated_validator(self):

@requests_mock.Mocker()
def test_url_check_timeout(self, req_mock):
url_validator = UrlValidator(self.catalog, True, 100)
url_validator = UrlValidator(self.catalog, True, 100, 10)
req_mock.head(self.test_url)
url_validator.is_working_url(self.test_url)
self.assertEqual(100, req_mock.request_history[0].timeout)

0 comments on commit c7bac62

Please sign in to comment.