Merge pull request #77 from TaskarCenterAtUW/stage

Stage to Main

Author: sujata-m

.github/workflows/unit_tests.yaml

@@ -4,7 +4,7 @@ on:
     branches-ignore:
       - '**'
   pull_request:
-    branches: [master, dev, stage]
+    branches: [main, dev, stage]
 
 jobs:
   UnitTest:
@@ -27,14 +27,48 @@ jobs:
           pip install -r requirements.txt
           pip install numpy==1.26.4
 
+      - name: Determine output folder
+        id: set_output_folder
+        run: |
+          if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then
+            branch_name=$GITHUB_BASE_REF
+          else
+            branch_name=$GITHUB_REF_NAME
+          fi
+          
+          if [[ $branch_name == "main" ]]; then
+            echo "output_folder=prod" >> $GITHUB_ENV
+          elif [[ $branch_name == "stage" ]]; then
+            echo "output_folder=stage" >> $GITHUB_ENV
+          elif [[ $branch_name == "dev" ]]; then
+            echo "output_folder=dev" >> $GITHUB_ENV
+          else
+            echo "Unknown branch: $branch_name"
+            exit 1
+          fi
+
       - name: Run tests with coverage
         run: |
-          coverage run --source=src -m unittest discover -s tests/unit_tests/
-          coverage xml  
+          timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
+          mkdir -p test_results
+          log_file="test_results/${timestamp}_report.log"
+          echo -e "\nTest Cases Report Report\n" >> $log_file
+          # Run the tests and append output to the log file
+          python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1
+          echo -e "\nCoverage Report\n" >> $log_file
+          coverage report >> $log_file
 
       - name: Check coverage
         run: |
           coverage report --fail-under=85
 
-      #- name: Run Coverage Report
-      #  run: coverage run --source=src -m unittest discover -s tests/unit_tests
+      - name: Upload report to Azure
+        uses: LanceMcCarthy/Action-AzureBlobUpload@v2
+        with:
+          source_folder: 'test_results'
+          destination_folder: '${{ env.output_folder }}'
+          connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
+          container_name: 'flex-validation-service'
+          clean_destination_folder: false
+          delete_if_exists: false
+

requirements.txt

@@ -2,5 +2,5 @@ fastapi~=0.111.1
 pydantic==1.10.16
 html_testRunner==1.2.1
 uvicorn==0.20.0
-python-ms-core==0.0.22
-gtfs-canonical-validator==0.0.5
+python-ms-core==0.0.23
+gtfs-canonical-validator==0.0.6

tests/unit_tests/test_gtfs_flex_validation.py

@@ -18,6 +18,7 @@
 FAIL3_FILE_NAME = 'flex-bad-foreignkey.zip'
 FAIL4_FILE_NAME = 'flex-bad-filename.zip'
 
+
 class TestBadFile4(unittest.TestCase):
 
     @patch.object(GTFSFlexValidation, 'download_single_file')
@@ -198,7 +199,6 @@ def test(self):
         self.assertTrue(is_valid)
 
 
-
 class TestSuccessWithMacOSFile(unittest.TestCase):
     @patch.object(GTFSFlexValidation, 'download_single_file')
     def setUp(self, mock_download_single_file):
@@ -219,7 +219,7 @@ def setUp(self, mock_download_single_file):
             mock_download_single_file.return_value = file_path
 
     def tearDown(self):
-        pass #GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix))
+        pass  # GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix))
 
     def test_validate_with_valid_file(self):
         # Arrange
@@ -254,7 +254,7 @@ def setUp(self, mock_download_single_file):
             self.validator = GTFSFlexValidation(file_path=file_path, storage_client=MagicMock())
             self.validator.file_path = file_path
             self.validator.file_relative_path = SUCCESS_FILE_NAME
-            self.validator.container_name = None            
+            self.validator.container_name = None
             self.validator.prefix = Settings().get_unique_id()
             mock_download_single_file.return_value = os.path.join(dl_folder_path, SUCCESS_FILE_NAME)
 
@@ -360,7 +360,7 @@ def setUp(self, mock_download_single_file):
             mock_download_single_file.return_value = file_path
 
     def tearDown(self):
-        pass #GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix))
+        pass  # GTFSFlexValidation.clean_up(os.path.join(DOWNLOAD_FILE_PATH, self.validator.prefix))
 
     def test_validate_with_invalid_file(self):
         # Arrange
@@ -410,15 +410,223 @@ def test_download_single_file_exception(self):
         self.validator.storage_client.get_file_from_url = MagicMock()
         file = MagicMock()
         file.file_path = 'text_file.txt'
-        file.get_stream = MagicMock(side_effect=FileNotFoundError("Mocked FileNotFoundError"))
+        file.get_stream = MagicMock(side_effect=FileNotFoundError('Mocked FileNotFoundError'))
         self.validator.storage_client.get_file_from_url.return_value = file
 
         dl_folder_path = os.path.join(DOWNLOAD_FILE_PATH)
         os.makedirs(dl_folder_path, exist_ok=True)
 
+    def test_clean_up_non_existent_path(self):
+        # Arrange
+        non_existent_path = f'{DOWNLOAD_FILE_PATH}/non_existent_file_or_folder'
+
+        # Act
+        try:
+            GTFSFlexValidation.clean_up(non_existent_path)
+            success = True
+        except Exception:
+            success = False
+
+        # Assert
+        self.assertTrue(success)
+
+    def test_download_single_file_file_not_found_exception(self):
+        # Arrange
+        self.validator.storage_client = MagicMock()
+        self.validator.storage_client.get_file_from_url = MagicMock()
+        self.validator.storage_client.get_file_from_url.side_effect = FileNotFoundError('File not found')
+        self.validator.prefix = Settings().get_unique_id()
+
         # Act & Assert
-#        with self.assertRaises(FileNotFoundError):
-#            self.validator.download_single_file(file_upload_path=file_upload_path)
+        with self.assertRaises(FileNotFoundError):
+            self.validator.download_single_file(file_upload_path='mock_nonexistent_file')
+
+    @patch('src.gtfs_flex_validation.CanonicalValidator')
+    def test_is_gtfs_flex_valid_with_errors(self, mock_canonical_validator):
+        # Arrange
+        mock_result = MagicMock()
+        mock_result.status = False
+        mock_result.error = [
+            {'code': 'INVALID_FIELD', 'sampleNotices': [{'fieldName': 'invalid_field', 'filename': 'flex_data'}]}
+        ]
+        mock_canonical_validator.return_value.validate.return_value = mock_result
+
+        expected_downloaded_file_path = f'{SAVED_FILE_PATH}/fail_schema_1.zip'
+        self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path)
+
+        # Act
+        is_valid, validation_message = self.validator.is_gtfs_flex_valid()
+
+        # Assert
+        self.assertFalse(is_valid)
+        self.assertIn('INVALID_FIELD', validation_message)
+
+    @patch('src.gtfs_flex_validation.CanonicalValidator')
+    def test_is_gtfs_flex_valid_with_pathways_items(self, mock_canonical_validator):
+        # Arrange
+        mock_result = MagicMock()
+        mock_result.status = False
+        mock_result.error = [
+            {'code': 'INVALID_FIELD',
+             'sampleNotices': [{'fieldName': 'start_pickup_dropoff_window', 'filename': 'stop_times.txt'},
+                               {'fieldName': 'end_pickup_dropoff_window', 'filename': 'stop_times.txt'}]}
+        ]
+        mock_canonical_validator.return_value.validate.return_value = mock_result
+
+        expected_downloaded_file_path = f'{SAVED_FILE_PATH}/fail_schema_1.zip'
+        self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path)
+
+        # Act
+        is_valid, validation_message = self.validator.is_gtfs_flex_valid()
+
+        # Assert
+        self.assertFalse(is_valid)
+        self.assertIn('INVALID_FIELD', validation_message)
+
+    @patch('src.gtfs_flex_validation.CanonicalValidator')
+    def test_is_gtfs_flex_valid_with_pathways_parent_items(self, mock_canonical_validator):
+        # Arrange
+        mock_result = MagicMock()
+        mock_result.status = False
+        mock_result.error = [
+            {'code': 'INVALID_FIELD',
+             'sampleNotices': [{'fieldName': 'start_pickup_dropoff_window', 'childFilename': 'locations.geojson'},
+                               {'fieldName': 'end_pickup_dropoff_window', 'childFilename': 'booking_rules.txt'}]}
+        ]
+        mock_canonical_validator.return_value.validate.return_value = mock_result
+
+        expected_downloaded_file_path = f'{SAVED_FILE_PATH}/fail_schema_1.zip'
+        self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path)
+
+        # Act
+        is_valid, validation_message = self.validator.is_gtfs_flex_valid()
+
+        # Assert
+        self.assertFalse(is_valid)
+        self.assertIn('INVALID_FIELD', validation_message)
+
+    @patch('src.gtfs_flex_validation.CanonicalValidator')
+    def test_is_gtfs_flex_valid_with_no_errors(self, mock_canonical_validator):
+        # Arrange
+        mock_result = MagicMock()
+        mock_result.status = True
+        mock_result.error = []
+        mock_canonical_validator.return_value.validate.return_value = mock_result
+
+        expected_downloaded_file_path = f'{SAVED_FILE_PATH}/{SUCCESS2_FILE_NAME}'
+        self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path)
+
+        # Act
+        is_valid, validation_message = self.validator.is_gtfs_flex_valid()
+
+        # Assert
+        self.assertTrue(is_valid)
+
+    def test_is_gtfs_flex_valid_with_invalid_file_extension(self):
+        self.validator.file_relative_path = f'{SAVED_FILE_PATH}/fail_schema_1.txt'
+
+        # Act
+        is_valid, validation_message = self.validator.is_gtfs_flex_valid()
+
+        # Assert
+        self.assertFalse(is_valid)
+
+    @patch.object(GTFSFlexValidation, 'is_gtfs_flex_valid')
+    def test_validate_facade(self, mock_is_gtfs_flex_valid):
+        # Arrange
+        mock_is_gtfs_flex_valid.return_value = (True, 'Valid file')
+
+        # Act
+        is_valid, validation_message = self.validator.validate()
+
+        # Assert
+        mock_is_gtfs_flex_valid.assert_called_once()
+        self.assertTrue(is_valid)
+        self.assertEqual(validation_message, 'Valid file')
+
+    @patch('os.makedirs')
+    @patch('builtins.open', new_callable=MagicMock)
+    def test_download_single_file_creates_directory(self, mock_open, mock_makedirs):
+        # Arrange
+        mock_file = MagicMock()
+        mock_file.file_path = 'file.zip'
+        mock_file.get_stream = MagicMock(return_value=b'file_content')
+
+        self.validator.storage_client = MagicMock()
+        self.validator.storage_client.get_file_from_url = MagicMock(return_value=mock_file)
+
+        self.validator.prefix = "test-prefix"  # Mock prefix
+        expected_dl_folder_path = f"{DOWNLOAD_FILE_PATH}/{self.validator.prefix}"
+        expected_file_path = f"{expected_dl_folder_path}/file.zip"
+
+        # Act
+        downloaded_file_path = self.validator.download_single_file(file_upload_path='mock_file.zip')
+
+        # Assert
+        mock_makedirs.assert_called_once_with(expected_dl_folder_path, exist_ok=True)
+        mock_open.assert_called_once_with(expected_file_path, 'wb')
+        self.assertEqual(downloaded_file_path, expected_file_path)
+        mock_file.get_stream.assert_called_once()
+
+
+class TestGTFSFlexValidationInit(unittest.TestCase):
+    @patch('src.gtfs_flex_validation.Settings')
+    def test_initialization(self, mock_settings):
+        # Arrange
+        mock_storage_client = MagicMock()
+        mock_container = MagicMock()
+        mock_storage_client.get_container.return_value = mock_container
+
+        mock_settings_instance = mock_settings.return_value
+        mock_settings_instance.storage_container_name = "test_container"
+        mock_settings_instance.get_unique_id.return_value = "unique_prefix"
+
+        file_path = "path/to/file.zip"
+
+        # Act
+        validation_instance = GTFSFlexValidation(
+            file_path=file_path,
+            storage_client=mock_storage_client,
+            prefix=None
+        )
+
+        # Assert
+        self.assertEqual(validation_instance.file_path, file_path)
+        self.assertEqual(validation_instance.file_relative_path, "file.zip")
+        self.assertEqual(validation_instance.container_name, "test_container")
+        self.assertEqual(validation_instance.prefix, "unique_prefix")
+        self.assertEqual(validation_instance.client, mock_container)
+
+        mock_storage_client.get_container.assert_called_once_with(container_name="test_container")
+        mock_settings_instance.get_unique_id.assert_called_once()
+
+    @patch('src.gtfs_flex_validation.Settings')
+    def test_initialization_with_prefix(self, mock_settings):
+        # Arrange
+        mock_storage_client = MagicMock()
+        mock_container = MagicMock()
+        mock_storage_client.get_container.return_value = mock_container
+
+        mock_settings_instance = mock_settings.return_value
+        mock_settings_instance.storage_container_name = "test_container"
+
+        file_path = "path/to/file.zip"
+        custom_prefix = "custom_prefix"
+
+        # Act
+        validation_instance = GTFSFlexValidation(
+            file_path=file_path,
+            storage_client=mock_storage_client,
+            prefix=custom_prefix
+        )
+
+        # Assert
+        self.assertEqual(validation_instance.prefix, custom_prefix)
+        self.assertEqual(validation_instance.container_name, "test_container")
+        self.assertEqual(validation_instance.file_relative_path, "file.zip")
+
+        mock_storage_client.get_container.assert_called_once_with(container_name="test_container")
+        mock_settings_instance.get_unique_id.assert_not_called()
 
 
 if __name__ == '__main__':