From d34fce4ee823da1edf6a5d3d0d4aaa780ce79c22 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Tue, 30 Mar 2021 20:02:28 -0400 Subject: [PATCH 1/2] Removed unreachable code --- .gitignore | 1 + scraper/src/config/config_loader.py | 1 - scraper/src/documentation_spider.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c1a3c4c..9747edd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ src/strategies/__pycache__/ *.pyc .env update.sh +.python-version *yarn.lock diff --git a/scraper/src/config/config_loader.py b/scraper/src/config/config_loader.py index 1aea2ad..99ffe88 100644 --- a/scraper/src/config/config_loader.py +++ b/scraper/src/config/config_loader.py @@ -102,7 +102,6 @@ def _load_config(self, config): return data except ValueError as value_error: raise ValueError('CONFIG is not a valid JSON') from value_error - sys.exit(EXIT_CODE_WRONG_CONFIG) def _parse(self): # Parse Env diff --git a/scraper/src/documentation_spider.py b/scraper/src/documentation_spider.py index ac840e7..0d55f35 100644 --- a/scraper/src/documentation_spider.py +++ b/scraper/src/documentation_spider.py @@ -160,7 +160,6 @@ def add_records(self, response, from_sitemap): self.reason_to_stop = "Too much hits, Docs-Scraper only handle {} records".format( int(self.nb_hits_max)) raise ValueError(self.reason_to_stop) - sys.exit(EXIT_CODE_EXCEEDED_RECORDS) def parse_from_sitemap(self, response): if self.reason_to_stop is not None: From 86e689a25d82c1ddc613f12071d6c3b57df45ca7 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Tue, 30 Mar 2021 20:11:35 -0400 Subject: [PATCH 2/2] Removing unused imports --- scraper/src/config/config_loader.py | 1 - scraper/src/documentation_spider.py | 1 - 2 files changed, 2 deletions(-) diff --git a/scraper/src/config/config_loader.py b/scraper/src/config/config_loader.py index 99ffe88..4060098 100644 --- a/scraper/src/config/config_loader.py +++ b/scraper/src/config/config_loader.py @@ -8,7 +8,6 @@ from collections import OrderedDict import json import os -import sys import copy from .config_validator import ConfigValidator diff --git a/scraper/src/documentation_spider.py b/scraper/src/documentation_spider.py index 0d55f35..88bd125 100644 --- a/scraper/src/documentation_spider.py +++ b/scraper/src/documentation_spider.py @@ -10,7 +10,6 @@ from scrapy.spiders.sitemap import regex import re import os -import sys # End of import for the sitemap behavior