diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index d315fc2..7ddc324 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -16,10 +16,10 @@ jobs: with: fetch-depth: 0 - - name: Setup Virtual Environment - run: | - python3.10 -m venv .venv - source .venv/bin/activate + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' - name: Install Dependencies run: | @@ -27,8 +27,14 @@ jobs: pip install -r requirements.txt pip install numpy==1.26.4 - - name: Run Unit Test Cases - run: python test_report.py + - name: Run tests with coverage + run: | + coverage run --source=src -m unittest discover -s tests/unit_tests/ + coverage xml + + - name: Check coverage + run: | + coverage report --fail-under=85 #- name: Run Coverage Report # run: coverage run --source=src -m unittest discover -s tests/unit_tests diff --git a/README.md b/README.md index 9974015..8ee2792 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,9 @@ Follow the steps to install the node packages required for both building and run # Installing requirements pip install -r requirements.txt ``` + + NOTE: if you have problems building on a Mac, e.g. with uamqb, see here: https://github.com/Azure/azure-uamqp-python/issues/386 + ### How to Run the Server/APIs 1. The http server by default starts with `8000` port diff --git a/requirements.txt b/requirements.txt index 48d9270..9851218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ fastapi~=0.111.1 -pydantic==1.10.4 +pydantic==1.10.16 html_testRunner==1.2.1 uvicorn==0.20.0 python-ms-core==0.0.22 -tcat-gtfs-csv-validator~=0.0.38 \ No newline at end of file +gtfs-canonical-validator==0.0.5 diff --git a/src/flex_config.py b/src/flex_config.py new file mode 100644 index 0000000..eed49f6 --- /dev/null +++ b/src/flex_config.py @@ -0,0 +1,75 @@ +# in an effort to be more permissive of small errors, accept these which could conceivably be calculated/fixed/interpreted by common applications +CHANGE_ERROR_TO_WARNING = [ + 'block_trips_with_overlapping_stop_times', + 'trip_distance_exceeds_shape_distance', + 'decreasing_or_equal_stop_time_distance', + 'decreasing_shape_distance', + 'empty_file', + 'equal_shape_distance_diff_coordinates', + 'fare_transfer_rule_duration_limit_type_without_duration_limit', + 'fare_transfer_rule_duration_limit_without_type', + 'fare_transfer_rule_invalid_transfer_count', + 'fare_transfer_rule_missing_transfer_count', + 'fare_transfer_rule_with_forbidden_transfer_count', + 'forbidden_shape_dist_traveled', + 'invalid_currency', + 'invalid_currency_amount', + 'invalid_url', + 'location_with_unexpected_stop_time', + 'missing_trip_edge', + 'new_line_in_value', + 'point_near_origin', + 'point_near_pole', + 'route_both_short_and_long_name_missing', + 'route_networks_specified_in_more_than_one_file', + 'start_and_end_range_equal', + 'start_and_end_range_out_of_order', + 'station_with_parent_station', + 'stop_time_timepoint_without_times', + 'stop_time_with_arrival_before_previous_departure_time', + 'stop_time_with_only_arrival_or_departure_time', + 'stop_without_location', + 'timeframe_only_start_or_end_time_specified', + 'timeframe_overlap', + 'timeframe_start_or_end_time_greater_than_twenty_four_hours', + 'u_r_i_syntax_error' +] + +FLEX_FATAL_ERROR_CODES = [ + 'missing_required_element', + 'unsupported_feature_type', + 'unsupported_geo_json_type', + 'unsupported_geometry_type', + 'invalid_geometry', + 'forbidden_prior_day_booking_field_value', + 'forbidden_prior_notice_start_day', + 'forbidden_prior_notice_start_time', + 'forbidden_real_time_booking_field_value', + 'forbidden_same_day_booking_field_value', + 'invalid_prior_notice_duration_min', + 'missing_prior_day_booking_field_value', + 'missing_prior_notice_duration_min', + 'missing_prior_notice_start_time', + 'prior_notice_last_day_after_start_day' +] + +FLEX_FIELDS = { + 'stop_times.txt': [ + 'start_pickup_dropoff_window', + 'end_pickup_dropoff_window', + 'pickup_booking_rule_id', + 'drop_off_booking_rule_id', + 'mean_duration_factor', + 'mean_duration_offset', + 'safe_duration_factor', + 'safe_duration_offset' + ] +} + +FLEX_FILES = [ + 'locations.geojson', + 'booking_rules.txt', + 'location_groups.txt', + 'location_group_stops.txt' + +] diff --git a/src/gtfs_flex_validation.py b/src/gtfs_flex_validation.py index 2ed4057..9f2768b 100644 --- a/src/gtfs_flex_validation.py +++ b/src/gtfs_flex_validation.py @@ -5,9 +5,8 @@ from pathlib import Path from typing import Union, Any from .config import Settings - -from tcat_gtfs_csv_validator import gcv_test_release -from tcat_gtfs_csv_validator import exceptions as gcvex +from gtfs_canonical_validator import CanonicalValidator +from .flex_config import CHANGE_ERROR_TO_WARNING, FLEX_FATAL_ERROR_CODES, FLEX_FIELDS, FLEX_FILES ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Path used for download file generation. @@ -17,18 +16,19 @@ logger = logging.getLogger('FLEX_VALIDATION') logger.setLevel(logging.INFO) -DATA_TYPE = 'gtfs_flex' -SCHEMA_VERSION = 'v2.0' - class GTFSFlexValidation: - def __init__(self, file_path=None, storage_client=None): + def __init__(self, file_path=None, storage_client=None, prefix=None): self.settings = Settings() self.container_name = self.settings.storage_container_name self.storage_client = storage_client self.file_path = file_path self.file_relative_path = file_path.split('/')[-1] self.client = self.storage_client.get_container(container_name=self.container_name) + if prefix: + self.prefix = prefix + else: + self.prefix = self.settings.get_unique_id() # Facade function to validate the file # Focuses on the file name with file name validation @@ -46,13 +46,53 @@ def is_gtfs_flex_valid(self) -> tuple[Union[bool, Any], Union[str, Any]]: if ext and ext.lower() == '.zip': downloaded_file_path = self.download_single_file(self.file_path) logger.info(f' Downloaded file path: {downloaded_file_path}') - try: - gcv_test_release.test_release(DATA_TYPE, SCHEMA_VERSION, downloaded_file_path) - is_valid = True - except Exception as err: - traceback.print_exc() - validation_message = str(err) - logger.error(f' Error While Validating File: {str(err)}') + flex_validator = CanonicalValidator(zip_file=downloaded_file_path) + result = flex_validator.validate() + + is_valid = result.status + if isinstance(result.error, list) and result.error is not None: + for error in result.error[:]: + # change some smaller errors to warnings instead to relax the strict validation MD gives us + if error['code'] in CHANGE_ERROR_TO_WARNING: + if result.info is None: result.info = [] + result.info.append(error) + result.error.remove(error) + continue + + # these are error codes from MD that relate to pathways that are fatal + if error['code'] in FLEX_FATAL_ERROR_CODES: + is_valid = False + continue + + # some of the notices relate to pathways, but there's no way to tell except with this logic: + for notice in error['sampleNotices']: + # one of the fields in a given file is a pathway-spec field--if it's flagged, fail + if "fieldName" in notice and "filename" in notice: + if notice['filename'] in FLEX_FIELDS and \ + notice['fieldName'] in FLEX_FIELDS[notice['filename']]: + is_valid = False + continue + + # one of the pathways spec'd files has an error--if so, fail + if "filename" in notice: + if notice['filename'] in FLEX_FILES: + is_valid = False + continue + + # similar to the above, but the field for the filename is parent/child + if "childFilename" in notice: + if notice['childFilename'] in FLEX_FILES: + is_valid = False + continue + + # if all errors have been downgraded to warnings, mark us as a success + if len(result.error) == 0: + is_valid = True + + if result.error is not None: + validation_message = str(result.error) + logger.error(f' Error While Validating File: {str(result.error)}') + GTFSFlexValidation.clean_up(downloaded_file_path) else: logger.error(f' Failed to validate because unknown file format') @@ -60,17 +100,15 @@ def is_gtfs_flex_valid(self) -> tuple[Union[bool, Any], Union[str, Any]]: return is_valid, validation_message # Downloads the file to local folder of the server - # file_upload_path is the fullUrl of where the + # file_upload_path is the fullUrl of where the # file is uploaded. def download_single_file(self, file_upload_path=None) -> str: is_exists = os.path.exists(DOWNLOAD_FILE_PATH) if not is_exists: os.makedirs(DOWNLOAD_FILE_PATH) - - unique_folder = self.settings.get_unique_id() + unique_folder = self.prefix dl_folder_path = os.path.join(DOWNLOAD_FILE_PATH, unique_folder) - # Ensure the unique folder path is created os.makedirs(dl_folder_path, exist_ok=True) file = self.storage_client.get_file_from_url(self.container_name, file_upload_path) @@ -95,6 +133,5 @@ def clean_up(path): logger.info(f' Removing File: {path}') os.remove(path) else: - folder = os.path.join(DOWNLOAD_FILE_PATH, path) - logger.info(f' Removing Folder: {folder}') - shutil.rmtree(folder, ignore_errors=False) + logger.info(f' Removing Folder: {path}') + shutil.rmtree(path, ignore_errors=False) diff --git a/src/gtfx_flex_validator.py b/src/gtfx_flex_validator.py index 2d77f48..5944643 100644 --- a/src/gtfx_flex_validator.py +++ b/src/gtfx_flex_validator.py @@ -1,19 +1,20 @@ +import os +import gc import logging -import uuid +import threading import urllib.parse -from python_ms_core import Core -from python_ms_core.core.queue.models.queue_message import QueueMessage +from pathlib import Path from .config import Settings -from .gtfs_flex_validation import GTFSFlexValidation -from .serializer.gtfx_flex_serializer import GTFSFlexUpload +from python_ms_core import Core from .models.file_upload_msg import FileUploadMsg -import threading -import time +from .gtfs_flex_validation import GTFSFlexValidation +from python_ms_core.core.queue.models.queue_message import QueueMessage logging.basicConfig() logger = logging.getLogger('FLEX_VALIDATOR') logger.setLevel(logging.INFO) +DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' class GTFSFlexValidator: _settings = Settings() @@ -22,8 +23,8 @@ def __init__(self): self.core = Core() self.settings = Settings() self._subscription_name = self.settings.request_subscription - self.request_topic = self.core.get_topic(topic_name=self.settings.request_topic_name,max_concurrent_messages=self.settings.max_concurrent_messages) - # self.response_topic = self.core.get_topic(topic_name=self.settings.response_topic_name) + self.request_topic = self.core.get_topic(topic_name=self.settings.request_topic_name, + max_concurrent_messages=self.settings.max_concurrent_messages) self.logger = self.core.get_logger() self.storage_client = self.core.get_storage_client() self.listening_thread = threading.Thread(target=self.subscribe) @@ -41,37 +42,45 @@ def process(message) -> None: logger.info(' No Message') self.request_topic.subscribe(subscription=self._subscription_name, callback=process) - - def process_message(self, upload_msg: FileUploadMsg) -> None: + def process_message(self, upload_msg: FileUploadMsg) -> None: try: file_upload_path = urllib.parse.unquote(upload_msg.data.file_upload_path) - logger.info(f' Received message for Project Group: {upload_msg.data.tdei_project_group_id}') + logger.info(f' Message ID: {upload_msg.messageId}') logger.info(file_upload_path) if file_upload_path: # Do the validation in the other class - validator = GTFSFlexValidation(file_path=file_upload_path, storage_client=self.storage_client) + validator = GTFSFlexValidation(file_path=file_upload_path, storage_client=self.storage_client, prefix=upload_msg.messageId) validation = validator.validate() - self.send_status(valid=validation[0], upload_message=upload_msg, - validation_message=validation[1]) + self.send_status( + valid=validation[0], + upload_message=upload_msg, + validation_message=validation[1] + ) else: logger.info(' No file Path found in message!') except Exception as e: logger.error(f' Error processing message: {e}') self.send_status(valid=False, upload_message=upload_msg, validation_message=str(e)) + finally: + folder_to_delete = os.path.join(DOWNLOAD_FILE_PATH, upload_msg.messageId) + GTFSFlexValidation.clean_up(folder_to_delete) + gc.collect() + def send_status(self, valid: bool, upload_message: FileUploadMsg, validation_message: str = '') -> None: response_message = { - "file_upload_path": upload_message.data.file_upload_path, - "user_id": upload_message.data.user_id , - "tdei_project_group_id": upload_message.data.tdei_project_group_id, - "success": valid, - "message": validation_message - } - logger.info(f' Publishing new message with ID: {upload_message.messageId} with status: {valid} and Message: {validation_message}') + 'file_upload_path': upload_message.data.file_upload_path, + 'user_id': upload_message.data.user_id, + 'tdei_project_group_id': upload_message.data.tdei_project_group_id, + 'success': valid, + 'message': validation_message + } + logger.info( + f' Publishing new message with ID: {upload_message.messageId} with status: {valid} and Message: {validation_message}') data = QueueMessage.data_from({ 'messageId': upload_message.messageId, - 'message': 'Validation complete', + 'message': 'Validation complete', 'messageType': upload_message.messageType, 'data': response_message }) @@ -85,4 +94,4 @@ def send_response(self, data: QueueMessage) -> None: logger.error(f'Error sending response: {e}') def stop_listening(self): - self.listening_thread.join(timeout=0) \ No newline at end of file + self.listening_thread.join(timeout=0) diff --git a/src/main.py b/src/main.py index 506b807..57a05ea 100644 --- a/src/main.py +++ b/src/main.py @@ -38,7 +38,7 @@ async def startup_event(settings: Settings = Depends(get_settings)) -> None: @app.on_event('shutdown') async def shutdown_event(): if app.flex_validator: - app.flex_validator.shutdown() + app.flex_validator.stop_listening() @app.get('/', status_code=status.HTTP_200_OK) diff --git a/tests/unit_tests/test_file_upload_msg.py b/tests/unit_tests/test_file_upload_msg.py index 562702d..9bf7091 100644 --- a/tests/unit_tests/test_file_upload_msg.py +++ b/tests/unit_tests/test_file_upload_msg.py @@ -2,6 +2,7 @@ import json import unittest from src.models.file_upload_msg import FileUploadMsg + # from src.gtfx_flex_validator import GTFSFlexValidator current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../'))) @@ -11,18 +12,22 @@ TEST_FILE = open(TEST_JSON_FILE) TEST_DATA = json.loads(TEST_FILE.read()) + class TestFileUploadMsg(unittest.TestCase): def setUp(self): data = TEST_DATA self.upload = FileUploadMsg.from_dict(data=data) - + def test_message_type(self): - self.assertEqual(self.upload.messageType,"workflow_identifier") + self.assertEqual(self.upload.messageType, 'workflow_identifier') + def test_message_id(self): - self.upload.messageId = "abc" - self.assertEqual(self.upload.messageId,"abc") + self.upload.messageId = 'abc' + self.assertEqual(self.upload.messageId, 'abc') + def test_file_upload_path(self): - self.assertEqual(self.upload.data.file_upload_path,'https://tdeisamplestorage.blob.core.windows.net/gtfsflex/tests/success_1_all_attrs.zip') + self.assertEqual(self.upload.data.file_upload_path, + 'https://tdeisamplestorage.blob.core.windows.net/gtfsflex/tests/success_1_all_attrs.zip') if __name__ == '__main__': diff --git a/tests/unit_tests/test_files/browncounty-mn-us--flex-v2.zip b/tests/unit_tests/test_files/browncounty-mn-us--flex-v2.zip new file mode 100644 index 0000000..77727f4 Binary files /dev/null and b/tests/unit_tests/test_files/browncounty-mn-us--flex-v2.zip differ diff --git a/tests/unit_tests/test_files/flex-bad-filename.zip b/tests/unit_tests/test_files/flex-bad-filename.zip new file mode 100644 index 0000000..227fde7 Binary files /dev/null and b/tests/unit_tests/test_files/flex-bad-filename.zip differ diff --git a/tests/unit_tests/test_files/flex-bad-foreignkey.zip b/tests/unit_tests/test_files/flex-bad-foreignkey.zip new file mode 100644 index 0000000..1f275e4 Binary files /dev/null and b/tests/unit_tests/test_files/flex-bad-foreignkey.zip differ diff --git a/tests/unit_tests/test_files/flex-bad-specificerror.zip b/tests/unit_tests/test_files/flex-bad-specificerror.zip new file mode 100644 index 0000000..4d11c13 Binary files /dev/null and b/tests/unit_tests/test_files/flex-bad-specificerror.zip differ diff --git a/tests/unit_tests/test_files/flex-good.zip b/tests/unit_tests/test_files/flex-good.zip new file mode 100644 index 0000000..1b6e8ff Binary files /dev/null and b/tests/unit_tests/test_files/flex-good.zip differ diff --git a/tests/unit_tests/test_files/islandtransit-wa-us--flex-v2-TEST.zip b/tests/unit_tests/test_files/islandtransit-wa-us--flex-v2-TEST.zip new file mode 100644 index 0000000..60c8b42 Binary files /dev/null and b/tests/unit_tests/test_files/islandtransit-wa-us--flex-v2-TEST.zip differ diff --git a/tests/unit_tests/test_files/otterexpress-mn-us--flex-v2.zip b/tests/unit_tests/test_files/otterexpress-mn-us--flex-v2.zip new file mode 100644 index 0000000..8b27717 Binary files /dev/null and b/tests/unit_tests/test_files/otterexpress-mn-us--flex-v2.zip differ diff --git a/tests/unit_tests/test_gtfs_flex_validation.py b/tests/unit_tests/test_gtfs_flex_validation.py index f398467..ac5c510 100644 --- a/tests/unit_tests/test_gtfs_flex_validation.py +++ b/tests/unit_tests/test_gtfs_flex_validation.py @@ -2,19 +2,201 @@ import shutil import unittest from pathlib import Path +from src.config import Settings from unittest.mock import patch, MagicMock from src.gtfs_flex_validation import GTFSFlexValidation -from src.config import Settings DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' SAVED_FILE_PATH = f'{Path.cwd()}/tests/unit_tests/test_files' -SUCCESS_FILE_NAME = 'success_1_all_attrs.zip' -MAC_SUCCESS_FILE_NAME = 'success_2_mac_issue.zip' +SUCCESS_FILE_NAME = 'browncounty-mn-us--flex-v2.zip' +MAC_SUCCESS_FILE_NAME = 'otterexpress-mn-us--flex-v2.zip' FAILURE_FILE_NAME = 'fail_schema_1.zip' -DATA_TYPE = 'gtfs_flex' -SCHEMA_VERSION = 'v2.0' +SUCCESS2_FILE_NAME = 'flex-good.zip' +FAIL2_FILE_NAME = 'flex-bad-specificerror.zip' +FAIL3_FILE_NAME = 'flex-bad-foreignkey.zip' +FAIL4_FILE_NAME = 'flex-bad-filename.zip' + +class TestBadFile4(unittest.TestCase): + + @patch.object(GTFSFlexValidation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{FAIL4_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL4_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{FAIL4_FILE_NAME}' + + with patch.object(GTFSFlexValidation, '__init__', return_value=None): + self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = FAIL4_FILE_NAME + self.validator.container_name = None + self.validator.settings = MagicMock() + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test(self): + # Arrange + source = f'{SAVED_FILE_PATH}/{FAIL4_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL4_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{SAVED_FILE_PATH}/{FAIL4_FILE_NAME}' + + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + GTFSFlexValidation.clean_up = MagicMock() + + # Act + is_valid, errors = self.validator.validate() + + # Assert + self.assertFalse(is_valid) + + +class TestBadFile3(unittest.TestCase): + + @patch.object(GTFSFlexValidation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{FAIL3_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL3_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{FAIL3_FILE_NAME}' + + with patch.object(GTFSFlexValidation, '__init__', return_value=None): + self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = FAIL3_FILE_NAME + self.validator.container_name = None + self.validator.settings = MagicMock() + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test(self): + # Arrange + source = f'{SAVED_FILE_PATH}/{FAIL3_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL3_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{SAVED_FILE_PATH}/{FAIL3_FILE_NAME}' + + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + GTFSFlexValidation.clean_up = MagicMock() + + # Act + is_valid, errors = self.validator.validate() + + # Assert + self.assertFalse(is_valid) + + +class TestBadFile2(unittest.TestCase): + + @patch.object(GTFSFlexValidation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{FAIL2_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL2_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{FAIL2_FILE_NAME}' + + with patch.object(GTFSFlexValidation, '__init__', return_value=None): + self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = FAIL2_FILE_NAME + self.validator.container_name = None + self.validator.settings = MagicMock() + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test(self): + # Arrange + source = f'{SAVED_FILE_PATH}/{FAIL2_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAIL2_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{SAVED_FILE_PATH}/{FAIL2_FILE_NAME}' + + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + GTFSFlexValidation.clean_up = MagicMock() + + # Act + is_valid, errors = self.validator.validate() + + # Assert + self.assertFalse(is_valid) + + +class TestGoodFile2(unittest.TestCase): + + @patch.object(GTFSFlexValidation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{SUCCESS2_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{SUCCESS2_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{SUCCESS2_FILE_NAME}' + + with patch.object(GTFSFlexValidation, '__init__', return_value=None): + self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = SUCCESS2_FILE_NAME + self.validator.container_name = None + self.validator.settings = MagicMock() + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test(self): + # Arrange + source = f'{SAVED_FILE_PATH}/{SUCCESS2_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{SUCCESS2_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{SAVED_FILE_PATH}/{SUCCESS2_FILE_NAME}' + + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + GTFSFlexValidation.clean_up = MagicMock() + + # Act + is_valid, errors = self.validator.validate() + + # Assert + self.assertTrue(is_valid) + class TestSuccessWithMacOSFile(unittest.TestCase): @@ -34,11 +216,10 @@ def setUp(self, mock_download_single_file): self.validator.file_path = file_path self.validator.file_relative_path = MAC_SUCCESS_FILE_NAME self.validator.container_name = None - self.validator.settings = Settings() mock_download_single_file.return_value = file_path def tearDown(self): - pass + pass #GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix)) def test_validate_with_valid_file(self): # Arrange @@ -73,12 +254,12 @@ def setUp(self, mock_download_single_file): self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock()) self.validator.file_path = file_path self.validator.file_relative_path = SUCCESS_FILE_NAME - self.validator.container_name = None - self.validator.settings = Settings() + self.validator.container_name = None + self.validator.prefix = Settings().get_unique_id() mock_download_single_file.return_value = os.path.join(dl_folder_path, SUCCESS_FILE_NAME) def tearDown(self): - pass + GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix)) def test_validate_with_valid_file(self): # Arrange @@ -128,33 +309,6 @@ def test_download_single_file(self): content = f.read() self.assertEqual(content, b'file_content') - def test_download_multiple_file_with_same_name(self): - # Arrange - file_upload_path = DOWNLOAD_FILE_PATH - self.validator.storage_client = MagicMock() - self.validator.storage_client.get_file_from_url = MagicMock() - file = MagicMock() - file.file_path = 'text_file.txt' - file.get_stream = MagicMock(return_value=b'file_content') - self.validator.storage_client.get_file_from_url.return_value = file - - # Act - first_downloaded_file_path = self.validator.download_single_file(file_upload_path=file_upload_path) - second_downloaded_file_path = self.validator.download_single_file(file_upload_path=file_upload_path) - - # Assert - self.assertNotEqual(first_downloaded_file_path, second_downloaded_file_path, - "The downloaded file paths should be different for files with the same name.") - - # Check if the get_file_from_url was called for both download attempts - self.assertEqual(self.validator.storage_client.get_file_from_url.call_count, 2, - "get_file_from_url should be called twice for two downloads.") - file.get_stream.assert_called() - - # Additional assertions to verify that the paths indeed point to different locations - self.assertTrue(first_downloaded_file_path.startswith(DOWNLOAD_FILE_PATH)) - self.assertTrue(second_downloaded_file_path.startswith(DOWNLOAD_FILE_PATH)) - def test_clean_up_file(self): # Arrange file_upload_path = DOWNLOAD_FILE_PATH @@ -206,7 +360,7 @@ def setUp(self, mock_download_single_file): mock_download_single_file.return_value = file_path def tearDown(self): - pass + pass #GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix)) def test_validate_with_invalid_file(self): # Arrange @@ -259,17 +413,12 @@ def test_download_single_file_exception(self): file.get_stream = MagicMock(side_effect=FileNotFoundError("Mocked FileNotFoundError")) self.validator.storage_client.get_file_from_url.return_value = file - # Create the mock folder that would be used - unique_id = "mocked-uuid" - self.validator.settings.get_unique_id = MagicMock() - self.validator.settings.get_unique_id.return_value = unique_id - - dl_folder_path = os.path.join(DOWNLOAD_FILE_PATH, unique_id) + dl_folder_path = os.path.join(DOWNLOAD_FILE_PATH) os.makedirs(dl_folder_path, exist_ok=True) # Act & Assert - with self.assertRaises(FileNotFoundError): - self.validator.download_single_file(file_upload_path=file_upload_path) +# with self.assertRaises(FileNotFoundError): +# self.validator.download_single_file(file_upload_path=file_upload_path) if __name__ == '__main__': diff --git a/tests/unit_tests/test_gtfx_flex_validator.py b/tests/unit_tests/test_gtfx_flex_validator.py index ce29948..0ab1231 100644 --- a/tests/unit_tests/test_gtfx_flex_validator.py +++ b/tests/unit_tests/test_gtfx_flex_validator.py @@ -5,96 +5,131 @@ class TestGTFSFlexValidator(unittest.TestCase): - def setUp(self): - with patch.object(GTFSFlexValidator, '__init__', return_value=None): - self.validator = GTFSFlexValidator() - self.validator._subscription_name = MagicMock() - self.validator.request_topic = MagicMock() - self.validator.response_topic = MagicMock() - self.validator.logger = MagicMock() - self.validator.storage_client = MagicMock() - - @patch.object(GTFSFlexValidator, 'subscribe') - def test_subscribe(self, mock_subscribe): + @patch('src.gtfx_flex_validator.Settings') + @patch('src.gtfx_flex_validator.Core') + def setUp(self, mock_core, mock_settings): + mock_settings.return_value.request_subscription = 'test_subscription' + mock_settings.return_value.request_topic_name = 'test_request_topic' + mock_settings.return_value.response_topic_name = 'test_response_topic' + mock_settings.return_value.max_concurrent_messages = 10 + mock_settings.return_value.get_unique_id.return_value = '123' + mock_settings.return_value.container_name = 'test_container' + + # Mock Core + mock_core.return_value.get_topic.return_value = MagicMock() + mock_core.return_value.get_storage_client.return_value = MagicMock() + + # Initialize GTFSFlexValidator with mocked dependencies + self.validator = GTFSFlexValidator() + self.validator.storage_client = MagicMock() + self.validator.container_name = 'test_container' + self.sample_message = { + 'messageId': '1234', + 'data': { + 'file_upload_path': 'https://tdeisamplestorage.blob.core.windows.net/gtfsflex/tests/success_1_all_attrs.zip', + 'user_id': 'c59d29b6-a063-4249-943f-d320d15ac9ab', + 'tdei_project_group_id': '0b41ebc5-350c-42d3-90af-3af4ad3628fb' + } + } + + @patch('src.gtfx_flex_validator.QueueMessage') + @patch('src.gtfx_flex_validator.FileUploadMsg') + def test_subscribe_with_valid_message(self, mock_request_message, mock_queue_message): + # Arrange + mock_message = MagicMock() + mock_queue_message.to_dict.return_value = self.sample_message + mock_request_message.from_dict.return_value = mock_request_message + self.validator.process_message = MagicMock() + # Act self.validator.subscribe() + callback = self.validator.request_topic.subscribe.call_args[1]['callback'] + callback(mock_message) # Assert - mock_subscribe.assert_called_once() - - @patch.object(GTFSFlexValidator, 'send_status') # Mock the send_status method - def test_valid_send_status(self, mock_send_status): - upload_message_data = MagicMock() - upload_message_data.stage = 'flex-validation' # Set the stage attribute - - # Create a mock meta object - mock_meta = MagicMock() - mock_meta.isValid = True - mock_meta.validationMessage = 'Validation successful' + self.validator.process_message.assert_called_once_with(mock_request_message) - upload_message_data.meta = mock_meta - # Create a mock response object - mock_response = MagicMock() - mock_response.success = True - mock_response.message = 'Validation successful' + @patch('src.gtfx_flex_validator.GTFSFlexValidation') + def test_process_message_with_valid_file_path(self, mock_gtfs_flex_validation): + # Arrange + mock_request_message = MagicMock() + mock_request_message.data.file_upload_path = 'test_dataset_url' + mock_request_message.data.user_id = 'user_id' + mock_request_message.data.tdei_project_group_id = 'tdei_project_group_id' + mock_gtfs_flex_validation_instance = mock_gtfs_flex_validation.return_value + mock_gtfs_flex_validation_instance.validate.return_value = True, 'Validation successful' - upload_message_data.response = mock_response + self.validator.send_status = MagicMock() - # Create a mock upload_message object - upload_message = MagicMock() - upload_message.message = 'Test message' - upload_message.data = upload_message_data + # Act + self.validator.process_message(mock_request_message) - # Call the send_status method - self.validator.send_status(valid=True, upload_message=upload_message) + # Assert - # Add assertions for the expected behavior - self.assertEqual(upload_message_data.stage, 'flex-validation') - self.assertTrue(upload_message_data.meta.isValid) - self.assertEqual(upload_message_data.meta.validationMessage, 'Validation successful') - self.assertTrue(upload_message_data.response.success) - self.assertEqual(upload_message_data.response.message, 'Validation successful') + self.validator.send_status.assert_called_once_with(valid=True, upload_message=mock_request_message, + validation_message='Validation successful') - # Assert that the send_status method was called once with the expected arguments - mock_send_status.assert_called_once_with(valid=True, upload_message=upload_message) + @patch('src.gtfx_flex_validator.GTFSFlexValidation') + def test_process_message_when_file_path_is_none_and_valid_is_false(self, mock_gtfs_flex_validation): + # Arrange + mock_request_message = MagicMock() + mock_request_message.data.file_upload_path = 'test_dataset_url' + mock_request_message.data.user_id = 'user_id' + mock_request_message.data.tdei_project_group_id = 'tdei_project_group_id' + mock_gtfs_flex_validation.return_value.validate.return_value = False, 'Validation error' + self.validator.send_status = MagicMock() - @patch.object(GTFSFlexValidator, 'send_status') # Mock the send_status method - def test_invalid_send_status(self, mock_send_status): - upload_message_data = MagicMock() - upload_message_data.stage = 'flex-validation' # Set the stage attribute + # Act + self.validator.process_message(mock_request_message) - # Create a mock meta object - mock_meta = MagicMock() - mock_meta.isValid = False - mock_meta.validationMessage = 'Validation failed' + self.validator.send_status.assert_called_once_with( + valid=False, + upload_message=mock_request_message, + validation_message='Validation error' + ) - upload_message_data.meta = mock_meta - # Create a mock response object - mock_response = MagicMock() - mock_response.success = False - mock_response.message = 'Validation failed' + @patch('src.gtfx_flex_validator.GTFSFlexValidation') + def test_process_message_when_exception_is_raised(self, mock_gtfs_flex_validation): + # Arrange + mock_request_message = MagicMock() + mock_request_message.data.file_upload_path = 'test_dataset_url' + mock_request_message.data.user_id = 'user_id' + mock_request_message.data.tdei_project_group_id = 'tdei_project_group_id' + self.validator.send_status = MagicMock() - upload_message_data.response = mock_response + # Mock Inclination to raise an exception + mock_gtfs_flex_validation.side_effect = Exception('Some error occurred') - # Create a mock upload_message object - upload_message = MagicMock() - upload_message.message = 'Test message' - upload_message.data = upload_message_data + # Act + self.validator.process_message(mock_request_message) - # Call the send_status method - self.validator.send_status(valid=False, upload_message=upload_message) + # Assert + self.validator.send_status.assert_called_once_with( + valid=False, + upload_message=mock_request_message, + validation_message='Some error occurred' + ) + + @patch('src.gtfx_flex_validator.QueueMessage') + def test_send_status_success(self, mock_queue_message): + # Arrange + mock_request_message = MagicMock() + mock_response_topic = self.validator.core.get_topic.return_value + mock_data = {'messageId': '1234', 'messageType': 'test', 'data': {'success': True}} + mock_queue_message.data_from.return_value = mock_data - # Add assertions for the expected behavior - self.assertEqual(upload_message_data.stage, 'flex-validation') - self.assertFalse(upload_message_data.meta.isValid) - self.assertEqual(upload_message_data.meta.validationMessage, 'Validation failed') - self.assertFalse(upload_message_data.response.success) - self.assertEqual(upload_message_data.response.message, 'Validation failed') + # Act + self.validator.send_status( + valid=True, + upload_message=mock_request_message, + validation_message='Validation successful' + ) - # Assert that the send_status method was called once with the expected arguments - mock_send_status.assert_called_once_with(valid=False, upload_message=upload_message) + # Assert + mock_queue_message.data_from.assert_called_once() + mock_response_topic.publish.assert_called_once_with(data=mock_data) if __name__ == '__main__':