diff --git a/.github/workflows/dev_workflow.yaml b/.github/workflows/dev_workflow.yaml new file mode 100644 index 0000000..932c737 --- /dev/null +++ b/.github/workflows/dev_workflow.yaml @@ -0,0 +1,43 @@ +--- +######### Dev Workflow ######## +on: + pull_request: + branches: [dev] + types: + - closed + workflow_dispatch: +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Development + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + Deploy: + needs: Build + environment: + name: Development + runs-on: ubuntu-latest + steps: + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{secrets.TDEI_CORE_AZURE_CREDS}} + - name: Deploy to Dev + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} diff --git a/.github/workflows/prod_workflow.yaml b/.github/workflows/prod_workflow.yaml new file mode 100644 index 0000000..7673784 --- /dev/null +++ b/.github/workflows/prod_workflow.yaml @@ -0,0 +1,43 @@ +--- +######### Prod Workflow ######## +on: + pull_request: + branches: [main] + types: + - closed + workflow_dispatch: +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Production + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + Deploy: + needs: Build + environment: + name: Production + runs-on: ubuntu-latest + steps: + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{secrets.TDEI_CORE_AZURE_CREDS}} + - name: Deploy to Production + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}. diff --git a/.github/workflows/stage_workflow.yaml b/.github/workflows/stage_workflow.yaml new file mode 100644 index 0000000..f607c11 --- /dev/null +++ b/.github/workflows/stage_workflow.yaml @@ -0,0 +1,43 @@ +--- +######### Stage Workflow ######## +on: + pull_request: + branches: [stage] + types: + - closed + workflow_dispatch: +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Stage + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + Deploy: + needs: Build + environment: + name: Stage + runs-on: ubuntu-latest + steps: + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{secrets.TDEI_CORE_AZURE_CREDS}} + - name: Deploy to Stage + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}. diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml new file mode 100644 index 0000000..92d59b8 --- /dev/null +++ b/.github/workflows/unit_tests.yaml @@ -0,0 +1,51 @@ +--- +name: Unit Tests + +############################# +# Start the job on all push # +############################# +on: + push: + branches-ignore: + - '**' + # Remove the line above to run when pushing to master + pull_request: + branches: [master, dev, stage] + +############### +# Set the Job # +############### +jobs: + UnitTest: + name: Unit Test Cases + # Set the agent to run on + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" # Use the appropriate Python version + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run unit tests + run: | + python test_report.py + coverage run --source=src -m unittest discover -s tests/ + coverage report -m + exit_status=$? + + # Set the exit status as an output for later use + echo "::set-output name=exit_status::$exit_status" + + - name: Archive Coverage Report + if: ${{ always() }} # Upload the coverage report even if tests fail + uses: actions/upload-artifact@v2 + with: + name: htmlcov + path: htmlcov diff --git a/.gitignore b/.gitignore index 68bc17f..08f9708 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,5 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ +reports/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bd8eee7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.10 +WORKDIR /code +COPY ./requirements.txt /code/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +COPY ./src /code/src +EXPOSE 8080 +CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md index 62851a2..a62dfdb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,151 @@ # TDEI-python-osw-validation -Python service to Validate the OSW file that is uploaded. + +## Introduction +Service to Validate the OSW files that is uploaded. At the moment, the service does the following: +- Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example `UPLOAD_TOPIC=osw-upload` +- Consumes the message and perform following checks - + - Download the file locally + - File location is in the message `data.meta.file_upload_path` + - Uses `python-osw-validation` to validate the file + - Adds the `isValid` and `validationMessage` keys to the original message +- Publishes the result to the topic mentioned in `.env` file, example `VALIDATION_TOPIC=osw-validation` + +## Getting Started +The project is built on Python with FastAPI framework. All the regular nuances for a Python project are valid for this. + +### System requirements +| Software | Version | +|------------|---------| +| Python | 3.10.x | + + +### Connectivity to cloud +- Connecting this to cloud will need the following in the `.env` file + +```bash +QUEUECONNECTION=xxxx +STORAGECONNECTION=xxxx +VALIDATION_REQ_TOPIC=xxxx +VALIDATION_REQ_SUB=xxxx +VALIDATION_RES_TOPIC=xxxx +CONTAINER_NAME=xxxx +AUTH_PERMISSION_URL=xxx + +``` + +The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package. +`QUEUECONNECTION` is used to send out the messages and listen to messages. + + +### How to Set up and Build +Follow the steps to install the python packages required for both building and running the application + +1. Setup virtual environment + ``` + python3.10 -m venv .venv + source .venv/bin/activate + ``` + +2. Install the dependencies. Run the following command in terminal on the same directory as `requirements.txt` + ``` + # Installing requirements + pip install -r requirements.txt + ``` +### How to Run the Server/APIs + +1. The http server by default starts with `8000` port +2. Run server + ``` + uvicorn src.main:app --reload + ``` +3. By default `get` call on `localhost:8000/health` gives a sample response +4. Other routes include a `ping` with get and post. Make `get` or `post` request to `http://localhost:8000/health/ping` +5. Once the server starts, it will start to listening the subscriber(`VALIDATION_REQ_SUB` should be in env file) + + +#### Request Format + +```json + { + "messageId": "tdei_record_id", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "file_upload_path", + "user_id": "user_id", + "tdei_project_group_id": "tdei_project_group_id" + } + } +``` + +#### Response Format + +```json + { + "messageId": "tdei_record_id", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "file_upload_path", + "user_id": "user_id", + "tdei_project_group_id": "tdei_project_group_id", + "success": true/false, + "message": "message" // if false the error string else empty string + }, + "publishedDate": "published date" + } +``` + + +### How to Set up and run the Tests + +Make sure you have set up the project properly before running the tests, see above for `How to Setup and Build`. + +#### How to run test harness +1. Add the new set of test inside `tests/test_harness/tests.json` file like - + ``` + { + "Name": "Test Name", + "Input_file": "test_files/osw_test_case1.json", // Input file path which you want to provide to the test + "Result": true/false // Defining the test output + } + ``` +2. Test Harness would require a valid `.env` file. +3. To run the test harness `python tests/test_harness/run_tests.py` +#### How to run unit test cases +1. `.env` file is not required for Unit test cases. +2. To run the unit test cases + 1. `python test_report.py` + 2. Above command will run all test cases and generate the html report, in `reports` folder at the root level. +3. To run the coverage + 1. `python -m coverage run --source=src -m unittest discover -s tests/unit_tests` + 2. Above command will run all the unit test cases. + 3. To generate the coverage report in console + 1. `coverage report` + 2. Above command will generate the code coverage report in terminal. + 4. To generate the coverage report in html. + 1. `coverage html` + 2. Above command will generate the html report, and generated html would be in `htmlcov` directory at the root level. + 5. _NOTE :_ To run the `html` or `report` coverage, 3.i) command is mandatory + +#### How to run integration test cases +1. `.env` file is required for Unit test cases. +2. To run the integration test cases, run the below command + 1. `python test_integration.py` + 2. Above command will run all integration test cases and generate the html report, in `reports` folder at the root level. + + +### Messaging + +This microservice deals with two topics/queues. +- upload queue from osw-upload +- validation queue from osw-validation + + +#### Incoming +The incoming messages will be from the upload queue `osw-upload`. +The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json) + +#### Outgoing +The outgoing messages will be to the `osw-validation` topic. +The format of the message is at [osw-validation.json](./src/assets/osw-validation.json) + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..73cac28 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +psutil==5.9.5 +fastapi==0.88.0 +python-dotenv==0.21.0 +pydantic==1.10.4 +python-ms-core==0.0.18 +uvicorn==0.20.0 +coverage==7.2.7 +html_testRunner==1.2.1 +httpx==0.24.1 +python-osw-validation==0.2.3 \ No newline at end of file diff --git a/src/assets/osw-upload.json b/src/assets/osw-upload.json new file mode 100644 index 0000000..15e722e --- /dev/null +++ b/src/assets/osw-upload.json @@ -0,0 +1,9 @@ +{ + "messageId": "c8c76e89f30944d2b2abd2491bd95337", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip", + "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab", + "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb" + } +} \ No newline at end of file diff --git a/src/assets/osw-validation.json b/src/assets/osw-validation.json new file mode 100644 index 0000000..1a72f45 --- /dev/null +++ b/src/assets/osw-validation.json @@ -0,0 +1,11 @@ +{ + "messageId": "c8c76e89f30944d2b2abd2491bd95337", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip", + "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab", + "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", + "success": true, + "message": "" + } +} \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..3e1accb --- /dev/null +++ b/src/config.py @@ -0,0 +1,29 @@ +import os +from dotenv import load_dotenv +from pydantic import BaseSettings + +load_dotenv() + + +class EventBusSettings: + connection_string: str = os.environ.get('QUEUECONNECTION', None) + upload_topic: str = os.environ.get('VALIDATION_REQ_TOPIC', None) + upload_subscription: str = os.environ.get('VALIDATION_REQ_SUB', None) + validation_topic: str = os.environ.get('VALIDATION_RES_TOPIC', None) + container_name: str = os.environ.get('CONTAINER_NAME', 'osw') + + +class Settings(BaseSettings): + app_name: str = 'python-osw-validation' + event_bus = EventBusSettings() + auth_permission_url: str = os.environ.get('AUTH_PERMISSION_URL', None) + + @property + def auth_provider(self) -> str: + is_simulated: str = os.environ.get('AUTH_SIMULATE', 'False') + if is_simulated.lower() in ('true', 'yes', '1'): + return 'Simulated' + elif is_simulated.lower() in ('false', 'no', '0'): + return 'Hosted' + else: + return 'Hosted' diff --git a/src/interface/validator_abstract.py b/src/interface/validator_abstract.py new file mode 100644 index 0000000..9c82c6f --- /dev/null +++ b/src/interface/validator_abstract.py @@ -0,0 +1,9 @@ +from abc import ABC, abstractmethod +from python_ms_core.core.queue.models.queue_message import QueueMessage + + +class ValidatorAbstract(ABC): + + @abstractmethod + def validate(self, message: QueueMessage) -> None: + pass diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..41b15d9 --- /dev/null +++ b/src/main.py @@ -0,0 +1,52 @@ +import os +import psutil +from fastapi import FastAPI, APIRouter, Depends, status +from functools import lru_cache +from .config import Settings +from .osw_validator import OSWValidator + +app = FastAPI() + +prefix_router = APIRouter(prefix='/health') + + +@lru_cache() +def get_settings(): + return Settings() + + +@app.on_event('startup') +async def startup_event(settings: Settings = Depends(get_settings)) -> None: + try: + OSWValidator() + except: + print('\n\n\x1b[31m Application startup failed due to missing or invalid .env file \x1b[0m') + print('\x1b[31m Please provide the valid .env file and .env file should contains following parameters\x1b[0m') + print() + print('\x1b[31m UPLOAD_TOPIC=xxxx \x1b[0m') + print('\x1b[31m UPLOAD_SUBSCRIPTION=xxxx \x1b[0m') + print('\x1b[31m VALIDATION_TOPIC=xxxx \x1b[0m') + print('\x1b[31m QUEUECONNECTION=xxxx \x1b[0m') + print('\x1b[31m STORAGECONNECTION=xxxx \x1b[0m \n\n') + parent_pid = os.getpid() + parent = psutil.Process(parent_pid) + for child in parent.children(recursive=True): + child.kill() + parent.kill() + + +@app.get('/', status_code=status.HTTP_200_OK) +@prefix_router.get('/', status_code=status.HTTP_200_OK) +def root(): + return "I'm healthy !!" + + +@app.get('/ping', status_code=status.HTTP_200_OK) +@app.post('/ping', status_code=status.HTTP_200_OK) +@prefix_router.get('/ping', status_code=status.HTTP_200_OK) +@prefix_router.post('/ping', status_code=status.HTTP_200_OK) +def ping(): + return "I'm healthy !!" + + +app.include_router(prefix_router) diff --git a/src/models/queue_message_content.py b/src/models/queue_message_content.py new file mode 100644 index 0000000..aad3f7d --- /dev/null +++ b/src/models/queue_message_content.py @@ -0,0 +1,111 @@ +import json + + +class ValidationResult: + is_valid: bool + validation_message: str + + +class Upload: + + def __init__(self, data: dict): + upload_data = data.get('data', None) + self._message = data.get('message', None) + self._message_type = data.get('messageType', None) + self._message_id = data.get('messageId', '') + self.data = UploadData(data=upload_data) if upload_data else {} + + @property + def message(self): + return self._message + + @message.setter + def message(self, value): + self._message = value + + @property + def message_type(self): + return self._message_type + + @message_type.setter + def message_type(self, value): + self._message_type = value + + @property + def message_id(self): + return self._message_id + + @message_id.setter + def message_id(self, value): + self._message_id = value + + def to_json(self): + self.data = self.data.to_json() + return to_json(self.__dict__) + + def data_from(self): + message = self + if isinstance(message, str): + message = json.loads(self) + if message: + try: + return Upload(data=message) + except Exception as e: + error = str(e).replace('Upload', 'Invalid parameter,') + raise TypeError(error) + + +class UploadData: + def __init__(self, data: dict): + self._file_upload_path = data.get('file_upload_path', '') + self._tdei_project_group_id = data.get('tdei_project_group_id', '') + self._user_id = data.get('user_id', '') + self._success = data.get('success', False) + self._message = data.get('message', '') + + @property + def file_upload_path(self): return self._file_upload_path + + @file_upload_path.setter + def file_upload_path(self, value): self._file_upload_path = value + + @property + def tdei_project_group_id(self): return self._tdei_project_group_id + + @tdei_project_group_id.setter + def tdei_project_group_id(self, value): self._tdei_project_group_id = value + + @property + def user_id(self): return self._user_id + + @user_id.setter + def user_id(self, value): self._user_id = value + + @property + def success(self): return self._success + + @success.setter + def success(self, value): self._success = value + + @property + def message(self): return self._message + + @message.setter + def message(self, value): self._message = value + + def to_json(self): + return to_json(self.__dict__) + + +def remove_underscore(string: str): + return string if not string.startswith('_') else string[1:] + + +def to_json(data: object): + result = {} + for key in data: + value = data[key] + key = remove_underscore(key) + result[key] = value + + return result diff --git a/src/osw_validator.py b/src/osw_validator.py new file mode 100644 index 0000000..2b9a34b --- /dev/null +++ b/src/osw_validator.py @@ -0,0 +1,109 @@ +import uuid +import logging +import datetime +import urllib.parse +from typing import List +from python_ms_core import Core +from python_ms_core.core.queue.models.queue_message import QueueMessage +from python_ms_core.core.auth.models.permission_request import PermissionRequest +from .validation import Validation +from .models.queue_message_content import Upload, ValidationResult +from .config import Settings +import threading + +logging.basicConfig() +logger = logging.getLogger('OSW_VALIDATOR') +logger.setLevel(logging.INFO) + + +class OSWValidator: + _settings = Settings() + + def __init__(self): + core = Core() + options = { + 'provider': self._settings.auth_provider, + 'api_url': self._settings.auth_permission_url + } + listening_topic_name = self._settings.event_bus.upload_topic or '' + publishing_topic_name = self._settings.event_bus.validation_topic or '' + self.subscription_name = self._settings.event_bus.upload_subscription or '' + self.listening_topic = core.get_topic(topic_name=listening_topic_name) + self.publishing_topic = core.get_topic(topic_name=publishing_topic_name) + self.logger = core.get_logger() + self.storage_client = core.get_storage_client() + self.auth = core.get_authorizer(config=options) + self.container_name = self._settings.event_bus.container_name + self.start_listening() + + def start_listening(self): + def process(message) -> None: + if message is not None: + queue_message = QueueMessage.to_dict(message) + upload_message = Upload.data_from(queue_message) + process_thread = threading.Thread(target=self.validate, args=[upload_message]) + process_thread.start() + # self.validate(upload_message) + + self.listening_topic.subscribe(subscription=self.subscription_name, callback=process) + + def validate(self, received_message: Upload): + tdei_record_id: str = '' + try: + tdei_record_id = received_message.message_id + logger.info(f'Received message for : {tdei_record_id} Message received for OSW validation !') + + if received_message.data.file_upload_path is None: + error_msg = 'Request does not have valid file path specified.' + logger.error(f'{tdei_record_id}, {error_msg} !') + raise Exception(error_msg) + + if 'VALIDATION_ONLY' not in received_message.message_type: + if self.has_permission(roles=['tdei-admin', 'poc', 'osw_data_generator'], + queue_message=received_message) is None: + error_msg = 'Unauthorized request !' + logger.error(tdei_record_id, error_msg, received_message) + raise Exception(error_msg) + + file_upload_path = urllib.parse.unquote(received_message.data.file_upload_path) + if file_upload_path: + validation_result = Validation(file_path=file_upload_path, storage_client=self.storage_client) + result = validation_result.validate() + self.send_status(result=result, upload_message=received_message) + else: + raise Exception('File entity not found') + except Exception as e: + logger.error(f'{tdei_record_id} Error occurred while validating OSW request, {e}') + result = ValidationResult() + result.is_valid = False + result.validation_message = f'Error occurred while validating OSW request {e}' + self.send_status(result=result, upload_message=received_message) + + def send_status(self, result: ValidationResult, upload_message: Upload): + upload_message.data.success = result.is_valid + upload_message.data.message = result.validation_message + + data = QueueMessage.data_from({ + 'messageId': upload_message.message_id, + 'messageType': upload_message.message_type, + 'data': upload_message.data.to_json() + }) + try: + self.publishing_topic.publish(data=data) + except Exception as e: + print(e) + logger.info(f'Publishing message for : {upload_message.message_id}') + + def has_permission(self, roles: List[str], queue_message: Upload) -> bool: + try: + permission_request = PermissionRequest( + user_id=queue_message.data.user_id, + project_group_id=queue_message.data.tdei_project_group_id, + permissions=roles, + should_satisfy_all=False + ) + response = self.auth.has_permission(request_params=permission_request) + return response if response is not None else False + except Exception as error: + print('Error validating the request authorization:', error) + return False diff --git a/src/validation.py b/src/validation.py new file mode 100644 index 0000000..dfc3e45 --- /dev/null +++ b/src/validation.py @@ -0,0 +1,93 @@ +import os +import shutil +import logging +import traceback +from pathlib import Path +from .config import Settings +from python_osw_validation import OSWValidation +from .models.queue_message_content import ValidationResult +import uuid + +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +# Path used for download file generation. +DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' + +logging.basicConfig() +logger = logging.getLogger('OSW_VALIDATION') +logger.setLevel(logging.INFO) + + +class Validation: + def __init__(self, file_path=None, storage_client=None): + settings = Settings() + self.container_name = settings.event_bus.container_name + self.storage_client = storage_client + self.file_path = file_path + self.file_relative_path = file_path.split('/')[-1] + self.client = self.storage_client.get_container(container_name=self.container_name) + + def validate(self, max_errors=20) -> ValidationResult: + return self.is_osw_valid(max_errors) + + def is_osw_valid(self, max_errors) -> ValidationResult: + result = ValidationResult() + result.is_valid = False + result.validation_message = '' + root, ext = os.path.splitext(self.file_relative_path) + if ext and ext.lower() == '.zip': + downloaded_file_path = self.download_single_file(self.file_path) + logger.info(f' Downloaded file path: {downloaded_file_path}') + validator = OSWValidation(zipfile_path=downloaded_file_path) + validation_result = validator.validate(max_errors) + result.is_valid = validation_result.is_valid + if not result.is_valid: + result.validation_message = validation_result.errors + logger.error(f' Error While Validating File: {str(validation_result.errors)}') + Validation.clean_up(downloaded_file_path) + else: + result.validation_message = 'Failed to validate because unknown file format' + logger.error(f' Failed to validate because unknown file format') + + return result + + # Downloads the single file into a unique directory + def download_single_file(self, file_upload_path=None) -> str: + is_exists = os.path.exists(DOWNLOAD_FILE_PATH) + unique_id = self.get_unique_id() + if not is_exists: + os.makedirs(DOWNLOAD_FILE_PATH) + unique_directory = os.path.join(DOWNLOAD_FILE_PATH,unique_id) + if not os.path.exists(unique_directory): + os.makedirs(unique_directory) + + file = self.storage_client.get_file_from_url(self.container_name, file_upload_path) + try: + if file.file_path: + file_path = os.path.basename(file.file_path) + local_download_path = os.path.join(unique_directory,file_path) + with open(local_download_path, 'wb') as blob: + blob.write(file.get_stream()) + logger.info(f' File downloaded to location: {local_download_path}') + return local_download_path + else: + logger.info(' File not found!') + except Exception as e: + traceback.print_exc() + logger.error(e) + + # Generates a unique string for directory + def get_unique_id(self) -> str: + unique_id = uuid.uuid1().hex[0:24] + return unique_id + + + + @staticmethod + def clean_up(path): + if os.path.isfile(path): + logger.info(f' Removing File: {path}') + os.remove(path) + else: + # folder = os.path.join(DOWNLOAD_FILE_PATH, path) + logger.info(f' Removing Folder: {path}') + shutil.rmtree(path, ignore_errors=False) diff --git a/test-case-enumeration.md b/test-case-enumeration.md new file mode 100644 index 0000000..8fb80e7 --- /dev/null +++ b/test-case-enumeration.md @@ -0,0 +1,111 @@ +# TDEI Python OSW Validation Service Unit Test Cases + +## Purpose + + +This document details the unit test cases for the [TDEI-python-osw-validation](https://github.com/TaskarCenterAtUW/TDEI-python-osw-validation) + +------------ + +## Test Framework + +Unit test cases are to be written using [Python Unittest](https://docs.python.org/3/library/unittest.html) + +------------ +## Test Cases + + +### Test cases table definitions +- **Component** -> Specifies the code component +- **Class Under Test** -> Target method name +- **Test Target** -> Specific requirement to test_ ex. Functional, security etc. +- **Scenario** -> Requirement to test +- **Expectation** -> Expected result from executed scenario + +### Python unittest code pattern + +```python +import unittest + +class TestStringMethods(unittest.TestCase): + + def test_upper(self): + self.assertEqual('foo'.upper(), 'FOO') + + def test_isupper(self): + self.assertTrue('FOO'.isupper()) + self.assertFalse('Foo'.isupper()) + + def test_split(self): + s = 'hello world' + self.assertEqual(s.split(), ['hello', 'world']) + # check that s.split fails when the separator is not a string + with self.assertRaises(TypeError): + s.split(2) + +if __name__ == '__main__': + unittest.main() +``` + + +### Test cases + +| Component | Class Under Test | Test Target | Scenario | Expectation | Status | +|------------|------------------------|--|------------------------------------------------------------------------------|--|--| +| Model | Upload | Functional| When requested with upload data | Expect to return same upload data |:white_check_mark:| +| Model | Upload | Functional| When requested with upload data_from | Expect to return same upload data_from |:white_check_mark:| +| Model | Upload | Functional| When requested with upload message | Expect to return same upload message |:white_check_mark:| +| Model | Upload | Functional| When requested with upload id | Expect to return same upload id |:white_check_mark:| +| Model | Upload | Functional| When requested with upload type | Expect to return same upload type |:white_check_mark:| +| Model | Upload | Functional| When requested with upload publish date | Expect to return same upload publish date |:white_check_mark:| +| Model | Upload | Functional| When requested with upload to_json | Expect to return same dict |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Model | UploadData | Functional| When requested with stage parameter | Expect to return stage |:white_check_mark:| +| Model | UploadData | Functional| When requested with tdei_project_group_id parameter | Expect to return tdei_project_group_id |:white_check_mark:| +| Model | UploadData | Functional| When requested with tdei_record_id parameter | Expect to return tdei_record_id |:white_check_mark:| +| Model | UploadData | Functional| When requested with user_id parameter | Expect to return user_id |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Model | TestRequest | Functional| When requested with tdei_project_group_id parameter | Expect to return tdei_project_group_id |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Model | TestMeta | Functional| When requested with file_upload_path parameter | Expect to return file_upload_path |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Model | TestResponse | Functional| When requested with response parameter | Expect to return either True or False |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Validation | TestSuccessValidation | Functional| When requested for clean_up_file function | Expect to return remove files from local storage |:white_check_mark:| +| Validation | TestSuccessValidation | Functional| When requested for clean_up_folder function | Expect to return remove directory from local storage |:white_check_mark:| +| Validation | TestSuccessValidation | Functional| When requested for download_single_file function | Expect to download file in local storage |:white_check_mark:| +| Validation | TestSuccessValidation | Functional| When requested for is_osw_valid function | Expect to return True |:white_check_mark:| +| Validation | TestSuccess Validation | Functional| When requested for validate function | Expect to return True |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Validation | TestFailureValidation | Functional| When requested for download_single_file function with invalid endpoint | Expect to throw exception |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for is_osw_valid function with invalid file format | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for is_osw_valid function with invalid zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with _id missing in zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid Edges file in zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid Nodes file in zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid Points file in zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid files inside zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with invalid geometry inside zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with missing identifier inside zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with no entiry inside zip file | Expect to return False |:white_check_mark:| +| Validation | TestFailureValidation | Functional| When requested for validate function with wrong datatypes inside zip file | Expect to return False |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Validator | TestValidator | Functional| When calling send_status function with invalid parameters | Expect to return invalid parameters |:white_check_mark:| +| Validator | TestValidator | Functional| When calling subscribe function | Expect to return a message |:white_check_mark:| +| Validator | TestValidator | Functional| When calling send_status function with invalid parameters | Expect to return a valid message |:white_check_mark:| +| -- | -- |--| -- |--|--| +| Server | TestApp | Functional | When calling get_settings function | Expect to return env variables |:white_check_mark:| +| Server | TestApp | Functional | When calling ping function | Expect to return 200 |:white_check_mark:| +| Server | TestApp | Functional | When calling root function | Expect to return 200 |:white_check_mark:| + + +## Integration Test cases +In case of integration tests, the system will look for all the integration points to be tested + +| Component | Feature under test | Scenario | Expectation | Status | +|-----------|------------------------|-|-|-| +| Validator | Servicebus integration | Subscribe to upload topic to verify servicebus integration | Expect to return message |:white_check_mark: | +| Validator | Servicebus integration | Should publish a message to be received on topic | Expect to receive message on target topic | :white_check_mark: | +| Validator | Storage Integration | Fetching a file returns a file entity | Expect to return the file entity | | + diff --git a/test_integration.py b/test_integration.py new file mode 100644 index 0000000..72cbfae --- /dev/null +++ b/test_integration.py @@ -0,0 +1,22 @@ +import unittest +import HtmlTestRunner + +# Define your test cases +from tests.integration_tests.osw_integration import TestOSWIntegration + +if __name__ == '__main__': + # Create a test suite + test_suite = unittest.TestSuite() + # Add your test cases to the test suite + test_suite.addTest(unittest.makeSuite(TestOSWIntegration)) + + # Define the output file for the HTML report + output_file = 'integration_test_report.html' + + # Open the output file in write mode + with open(output_file, 'w') as f: + # Create an HTMLTestRunner instance with the output file and customize the template + runner = HtmlTestRunner.HTMLTestRunner(stream=f, report_title='Integration Test Report', combine_reports=True) + + # Run the test suite with the HTMLTestRunner + runner.run(test_suite) diff --git a/test_report.py b/test_report.py new file mode 100644 index 0000000..bd27d0e --- /dev/null +++ b/test_report.py @@ -0,0 +1,34 @@ +import unittest +import HtmlTestRunner + +# Define your test cases +from tests.unit_tests.test_queue_message_content import TestUpload, TestUploadData, TestToJson, TestValidationResult +from tests.unit_tests.test_validation import TestOtherValidation, TestValidation +from tests.unit_tests.test_osw_validator import TestOSWValidator +from tests.unit_tests.test_main import TestApp + +if __name__ == '__main__': + # Create a test suite + test_suite = unittest.TestSuite() + # Add your test cases to the test suite + test_suite.addTest(unittest.makeSuite(TestUpload)) + test_suite.addTest(unittest.makeSuite(TestUploadData)) + test_suite.addTest(unittest.makeSuite(TestToJson)) + test_suite.addTest(unittest.makeSuite(TestValidationResult)) + test_suite.addTest(unittest.makeSuite(TestOtherValidation)) + test_suite.addTest(unittest.makeSuite(TestValidation)) + test_suite.addTest(unittest.makeSuite(TestOSWValidator)) + test_suite.addTest(unittest.makeSuite(TestApp)) + + # Define the output file for the HTML report + output_file = 'test_report.html' + + # Open the output file in write mode + with open(output_file, 'w') as f: + # Create an HTMLTestRunner instance with the output file and customize the template + runner = HtmlTestRunner.HTMLTestRunner(stream=f, report_title='OSW Test Report', combine_reports=True) + + # Run the test suite with the HTMLTestRunner + runner.run(test_suite) + + print(f'\nRunning the tests complete.. see the report at {output_file}') diff --git a/tests/integration_tests/osw_integration.py b/tests/integration_tests/osw_integration.py new file mode 100644 index 0000000..e0c57f2 --- /dev/null +++ b/tests/integration_tests/osw_integration.py @@ -0,0 +1,83 @@ +import unittest +import os +import json +import time +from unittest.mock import patch, MagicMock +from dotenv import load_dotenv +import asyncio +from urllib.parse import urlparse + +# Execute to apply environment variable overrides +load_dotenv() + +os.environ['VALIDATION_REQ_TOPIC'] = 'temp-upload' +os.environ['VALIDATION_REQ_SUB'] = 'upload-validation-processor' +os.environ['VALIDATION_RES_TOPIC'] = 'temp-validation' + +from src.osw_validator import OSWValidator +from python_ms_core import Core +from python_ms_core.core.queue.models.queue_message import QueueMessage + +TEST_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEST_JSON_FILE = os.path.join(TEST_DIR, 'test_harness/test_files/osw_test_case2.json') + + +class TestOSWIntegration(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.core = Core() + cls.upload_topic_name = os.environ['VALIDATION_REQ_TOPIC'] + cls.upload_subscription_name = os.environ['VALIDATION_REQ_SUB'] + cls.validation_topic_name = os.environ['VALIDATION_RES_TOPIC'] + + def setUp(self): + self.test_data = self.read_test_data() + + @staticmethod + def read_test_data(): + with open(TEST_JSON_FILE, 'r') as test_file: + test_data = json.loads(test_file.read()) + return test_data + + def tearDown(self): + pass + + @patch.object(OSWValidator, 'start_listening', new=MagicMock()) + def test_subscribe_to_upload_topic(self): + if os.environ['QUEUECONNECTION'] is None: + self.fail('QUEUECONNECTION environment not set') + validator = OSWValidator() + upload_topic = self.core.get_topic(topic_name=self.upload_topic_name) + message = QueueMessage.data_from({'message': '', 'data': self.test_data}) + upload_topic.publish(data=message) + time.sleep(0.5) # Wait to get the callback + validator.start_listening.assert_called_once() + + @patch.object(OSWValidator, 'start_listening', new=MagicMock()) + async def test_servicebus_receive(self): + if os.environ['QUEUECONNECTION'] is None: + self.fail('QUEUECONNECTION environment not set') + validator = OSWValidator() + subscribe_function = MagicMock() + message = QueueMessage.data_from({'message': '', 'data': self.test_data}) + validator.publishing_topic.publish(data=message) + validation_topic = self.core.get_topic(topic_name=self.validation_topic_name) + async with validation_topic.subscribe(subscription='temp-validation-result', callback=subscribe_function): + await asyncio.sleep(0.5) # Wait for callback + subscribe_function.assert_called_once() + validator.start_listening.assert_called_once() + + def test_file_entity(self): + test_file_url = 'https://tdeisamplestorage.blob.core.windows.net/osw/2023/APRIL/66c85a5a-2335-4b97-a0a3-0bb93cba1ae5/osw-test-upload_19df12452cae4da5a71db3fa276f4f5e.zip' + url = urlparse(test_file_url) + file_path = url.path + file_components = file_path.split('/') + container_name = file_components[1] + file = self.core.get_storage_client().get_file_from_url(container_name=container_name, full_url=test_file_url) + content = file.get_stream() + self.assertTrue(content) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_harness/run_tests.py b/tests/test_harness/run_tests.py new file mode 100644 index 0000000..b88e4f7 --- /dev/null +++ b/tests/test_harness/run_tests.py @@ -0,0 +1,105 @@ +import os +import datetime +import json +import time +import uuid +from python_ms_core import Core +from python_ms_core.core.queue.models.queue_message import QueueMessage +from pydantic import BaseSettings +from dotenv import load_dotenv + +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_JSON_FILE = os.path.join(ROOT_DIR, 'tests.json') +TEST_FILE = open(TEST_JSON_FILE) +TEST_DATA = json.loads(TEST_FILE.read()) + +TESTS = TEST_DATA['Tests'] + +load_dotenv() + + +class Settings(BaseSettings): + publishing_topic_name: str = os.environ.get('VALIDATION_REQ_TOPIC', None) + subscription_topic_name: str = os.environ.get('VALIDATION_RES_TOPIC', None) + subscription_name: str = 'test_subscribtion' + container_name: str = os.environ.get('CONTAINER_NAME', 'tdei-storage-test') + + +def post_message_to_topic(msg: dict, core, settings: Settings): + publish_topic = core.get_topic(topic_name=settings.publishing_topic_name) + data = QueueMessage.data_from(msg) + publish_topic.publish(data=data) + + +def do_test(test, core, settings: Settings): + print(f'Performing tests : {test["Name"]}') + storage_client = core.get_storage_client() + + container = storage_client.get_container(container_name=settings.container_name) + basename = os.path.basename(test['Input_file']) + + suffix = datetime.datetime.now().strftime("%y%m%d_%H%M%S") + + base_name, base_ext = os.path.splitext(basename) + file_name = "_".join([base_name, suffix]) + base_ext + test_file = container.create_file(f'test_upload/{file_name}') # Removed mime-type + + input_file = os.path.join(ROOT_DIR, test['Input_file']) + # Uploading file to blob storage + with open(input_file, 'rb') as msg_file: + test_file.upload(msg_file) + blob_url = test_file.get_remote_url() + print(f'Performing tests : {test["Name"]}:{blob_url}') + message_id = uuid.uuid1().hex[0:24] + # Reading the input file + input_data = open(input_file) + data = json.load(input_data) + data['message'] = test['Name'] + upload_message = { + 'messageId': message_id, + 'message': test['Name'], + 'messageType': 'osw-upload', + 'data': data + } + # Publishing message to topic + post_message_to_topic(upload_message, core, settings) + + +def subscribe(core, settings: Settings): + def process(message) -> None: + parsed_message = message.__dict__ + message = parsed_message['message'] + parsed_data = parsed_message['data'] + test_detail = [item for item in TESTS if item.get('Name') == message] + if len(test_detail) > 0: + if test_detail[0]['Result'] == parsed_data['response']['success']: + print(f'Performing tests : {message}: PASSED\n') + else: + print(f'Performing tests : {message}: FAILED\n') + else: + # print(parsed_message) + print('Message Received from NodeJS publisher. \n') + + try: + listening_topic = core.get_topic(topic_name=settings.subscription_topic_name) + listening_topic.subscribe(subscription=settings.subscription_name, callback=process) + except Exception as e: + print(e) + print('Tests Done!') + + +def test_harness(core): + settings = Settings() + subscribe(core, settings=settings) + for test in TESTS: + do_test(test, core, settings) + + +if __name__ == "__main__": + core = Core() + time.sleep(1) + print(f'Performing tests :') + test_harness(core) + time.sleep(45) + print('Tests Completed!\n') + os._exit(0) diff --git a/tests/test_harness/test_files/osw_test_case1.json b/tests/test_harness/test_files/osw_test_case1.json new file mode 100644 index 0000000..73434a0 --- /dev/null +++ b/tests/test_harness/test_files/osw_test_case1.json @@ -0,0 +1,9 @@ +{ + "messageId": "c8c76e89f30944d2b2abd2491bd95337", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/invalid.zip", + "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab", + "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb" + } +} \ No newline at end of file diff --git a/tests/test_harness/test_files/osw_test_case2.json b/tests/test_harness/test_files/osw_test_case2.json new file mode 100644 index 0000000..15e722e --- /dev/null +++ b/tests/test_harness/test_files/osw_test_case2.json @@ -0,0 +1,9 @@ +{ + "messageId": "c8c76e89f30944d2b2abd2491bd95337", + "messageType": "workflow_identifier", + "data": { + "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip", + "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab", + "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb" + } +} \ No newline at end of file diff --git a/tests/test_harness/tests.json b/tests/test_harness/tests.json new file mode 100644 index 0000000..0dae771 --- /dev/null +++ b/tests/test_harness/tests.json @@ -0,0 +1,14 @@ +{ + "Tests": [ + { + "Name": "TC1: OSW Failure test case", + "Input_file": "test_files/osw_test_case1.json", + "Result": false + }, + { + "Name": "TC2: OSW Valid test case", + "Input_file": "test_files/osw_test_case2.json", + "Result": true + } + ] +} \ No newline at end of file diff --git a/tests/unit_tests/test_files/_id_missing.zip b/tests/unit_tests/test_files/_id_missing.zip new file mode 100644 index 0000000..c81a7a2 Binary files /dev/null and b/tests/unit_tests/test_files/_id_missing.zip differ diff --git a/tests/unit_tests/test_files/edges_invalid.zip b/tests/unit_tests/test_files/edges_invalid.zip new file mode 100644 index 0000000..0aff185 Binary files /dev/null and b/tests/unit_tests/test_files/edges_invalid.zip differ diff --git a/tests/unit_tests/test_files/invalid.zip b/tests/unit_tests/test_files/invalid.zip new file mode 100644 index 0000000..518a29f Binary files /dev/null and b/tests/unit_tests/test_files/invalid.zip differ diff --git a/tests/unit_tests/test_files/invalid_files.zip b/tests/unit_tests/test_files/invalid_files.zip new file mode 100644 index 0000000..8573afd Binary files /dev/null and b/tests/unit_tests/test_files/invalid_files.zip differ diff --git a/tests/unit_tests/test_files/invalid_geometry.zip b/tests/unit_tests/test_files/invalid_geometry.zip new file mode 100644 index 0000000..e008aed Binary files /dev/null and b/tests/unit_tests/test_files/invalid_geometry.zip differ diff --git a/tests/unit_tests/test_files/missing_identifier.zip b/tests/unit_tests/test_files/missing_identifier.zip new file mode 100644 index 0000000..d1580b3 Binary files /dev/null and b/tests/unit_tests/test_files/missing_identifier.zip differ diff --git a/tests/unit_tests/test_files/no_entity.zip b/tests/unit_tests/test_files/no_entity.zip new file mode 100644 index 0000000..4ec6252 Binary files /dev/null and b/tests/unit_tests/test_files/no_entity.zip differ diff --git a/tests/unit_tests/test_files/nodes_invalid.zip b/tests/unit_tests/test_files/nodes_invalid.zip new file mode 100644 index 0000000..fcfb41f Binary files /dev/null and b/tests/unit_tests/test_files/nodes_invalid.zip differ diff --git a/tests/unit_tests/test_files/points_invalid.zip b/tests/unit_tests/test_files/points_invalid.zip new file mode 100644 index 0000000..5514f7d Binary files /dev/null and b/tests/unit_tests/test_files/points_invalid.zip differ diff --git a/tests/unit_tests/test_files/valid.zip b/tests/unit_tests/test_files/valid.zip new file mode 100644 index 0000000..cce4828 Binary files /dev/null and b/tests/unit_tests/test_files/valid.zip differ diff --git a/tests/unit_tests/test_files/wrong_datatype.zip b/tests/unit_tests/test_files/wrong_datatype.zip new file mode 100644 index 0000000..c1868f5 Binary files /dev/null and b/tests/unit_tests/test_files/wrong_datatype.zip differ diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py new file mode 100644 index 0000000..6beac7a --- /dev/null +++ b/tests/unit_tests/test_main.py @@ -0,0 +1,27 @@ +import unittest +from fastapi import status +from fastapi.testclient import TestClient +from src.main import app, get_settings + + +class TestApp(unittest.TestCase): + def setUp(self): + self.client = TestClient(app) + + def test_root(self): + response = self.client.get("/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.text.strip('\"'), "I'm healthy !!") + + def test_ping(self): + response = self.client.get('/ping') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.text.strip('\"'), "I'm healthy !!") + + def test_get_settings(self): + settings = get_settings() + self.assertIsNotNone(settings) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_osw_validator.py b/tests/unit_tests/test_osw_validator.py new file mode 100644 index 0000000..626b749 --- /dev/null +++ b/tests/unit_tests/test_osw_validator.py @@ -0,0 +1,168 @@ +import os +import json +import unittest +from unittest.mock import MagicMock, patch, call +from src.osw_validator import OSWValidator +from src.models.queue_message_content import ValidationResult, Upload + +current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../'))) +parent_dir = os.path.dirname(current_dir) + +TEST_JSON_FILE = os.path.join(parent_dir, 'src/assets/osw-upload.json') + +TEST_FILE = open(TEST_JSON_FILE) +TEST_DATA = json.loads(TEST_FILE.read()) + + +class PermissionResponse: + def __init__(self, is_authorized): + self.is_authorized = is_authorized + + +class TestOSWValidator(unittest.TestCase): + + def setUp(self): + with patch.object(OSWValidator, '__init__', return_value=None): + self.validator = OSWValidator() + self.validator._subscription_name = MagicMock() + self.validator.listening_topic = MagicMock() + self.validator.publish_topic = MagicMock() + self.validator.logger = MagicMock() + self.validator.storage_client = MagicMock() + self.validator.auth = MagicMock() + + @patch.object(OSWValidator, 'start_listening') + def test_start_listening(self, mock_start_listening): + # Act + self.validator.start_listening() + + # Assert + mock_start_listening.assert_called_once() + + @patch.object(OSWValidator, 'send_status') # Mock the send_status method + def test_valid_send_status(self, mock_send_status): + upload_message_data = MagicMock() + upload_message_data.stage = 'OSW-Validation' # Set the stage attribute + + # Create a mock meta object + mock_meta = MagicMock() + mock_meta.isValid = True + mock_meta.validationMessage = 'Validation successful' + + upload_message_data.meta = mock_meta + + # Create a mock response object + mock_response = MagicMock() + mock_response.success = True + mock_response.message = 'Validation successful' + + upload_message_data.response = mock_response + + # Create a mock upload_message object + upload_message = MagicMock() + upload_message.message = 'Test message' + upload_message.data = upload_message_data + result = ValidationResult() + result.is_valid = True + result.validation_message = '' + # Call the send_status method + self.validator.send_status(result=result, upload_message=upload_message) + + # Add assertions for the expected behavior + self.assertEqual(upload_message_data.stage, 'OSW-Validation') + self.assertTrue(upload_message_data.meta.isValid) + self.assertEqual(upload_message_data.meta.validationMessage, 'Validation successful') + self.assertTrue(upload_message_data.response.success) + self.assertEqual(upload_message_data.response.message, 'Validation successful') + + # Assert that the send_status method was called once with the expected arguments + mock_send_status.assert_called_once_with(result=result, upload_message=upload_message) + + @patch.object(OSWValidator, 'send_status') # Mock the send_status method + def test_invalid_send_status(self, mock_send_status): + upload_message_data = MagicMock() + upload_message_data.stage = 'OSW-Validation' # Set the stage attribute + + # Create a mock meta object + mock_meta = MagicMock() + mock_meta.isValid = False + mock_meta.validationMessage = 'Validation failed' + + upload_message_data.meta = mock_meta + + # Create a mock response object + mock_response = MagicMock() + mock_response.success = False + mock_response.message = 'Validation failed' + + upload_message_data.response = mock_response + + # Create a mock upload_message object + upload_message = MagicMock() + upload_message.message = 'Test message' + upload_message.data = upload_message_data + result = ValidationResult() + result.is_valid = True + result.validation_message = 'Validation failed' + + # Call the send_status method + self.validator.send_status(result=result, upload_message=upload_message) + + # Add assertions for the expected behavior + self.assertEqual(upload_message_data.stage, 'OSW-Validation') + self.assertFalse(upload_message_data.meta.isValid) + self.assertEqual(upload_message_data.meta.validationMessage, 'Validation failed') + self.assertFalse(upload_message_data.response.success) + self.assertEqual(upload_message_data.response.message, 'Validation failed') + + # Assert that the send_status method was called once with the expected arguments + mock_send_status.assert_called_once_with(result=result, upload_message=upload_message) + + def test_has_permission_for_admin_success(self): + self.validator.auth.has_permission.return_value = True + upload_message = Upload(TEST_DATA) + + result = self.validator.has_permission(roles=['tdei-admin'], queue_message=upload_message) + + # Assertions + self.assertTrue(result) + + def test_has_permission_for_poc_success(self): + self.validator.auth.has_permission.return_value = True + upload_message = Upload(TEST_DATA) + + result = self.validator.has_permission(roles=['poc'], queue_message=upload_message) + + # Assertions + self.assertTrue(result) + + def test_has_permission_for_osw_data_generator_success(self): + self.validator.auth.has_permission.return_value = True + upload_message = Upload(TEST_DATA) + + result = self.validator.has_permission(roles=['osw_data_generator'], queue_message=upload_message) + + # Assertions + self.assertTrue(result) + + def test_has_permission_for_all_roles_success(self): + self.validator.auth.has_permission.return_value = True + upload_message = Upload(TEST_DATA) + + result = self.validator.has_permission(roles=['tdei-admin', 'poc', 'osw_data_generator'], queue_message=upload_message) + + # Assertions + self.assertTrue(result) + + def test_has_permission_failure(self): + self.validator.auth.has_permission.return_value = False + upload_message = Upload(TEST_DATA) + + result = self.validator.has_permission(roles=['test'], queue_message=upload_message) + + # Assertions + self.assertFalse(result) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_queue_message_content.py b/tests/unit_tests/test_queue_message_content.py new file mode 100644 index 0000000..17422c2 --- /dev/null +++ b/tests/unit_tests/test_queue_message_content.py @@ -0,0 +1,94 @@ +import os +import json +import unittest +from unittest.mock import MagicMock +from src.models.queue_message_content import ValidationResult, Upload, UploadData, to_json + +current_dir = os.path.dirname(os.path.abspath(os.path.join(__file__, '../'))) +parent_dir = os.path.dirname(current_dir) + +TEST_JSON_FILE = os.path.join(parent_dir, 'src/assets/osw-upload.json') + +TEST_FILE = open(TEST_JSON_FILE) +TEST_DATA = json.loads(TEST_FILE.read()) + + +class TestUpload(unittest.TestCase): + + def setUp(self): + data = TEST_DATA + self.upload = Upload(data) + + def test_message_type(self): + self.assertEqual(self.upload.message_type, 'workflow_identifier') + self.upload.message_type = 'New messageType' + self.assertEqual(self.upload.message_type, 'New messageType') + + def test_message_id(self): + self.assertEqual(self.upload.message_id, 'c8c76e89f30944d2b2abd2491bd95337') + self.upload.message_id = 'New messageId' + self.assertEqual(self.upload.message_id, 'New messageId') + + def test_to_json(self): + self.upload.data.to_json = MagicMock(return_value={}) + json_data = self.upload.to_json() + self.assertIsInstance(json_data, dict) + self.assertEqual(json_data['message_type'], 'workflow_identifier') + + def test_data_from(self): + message = TEST_DATA + upload = Upload.data_from(json.dumps(message)) + self.assertIsInstance(upload, Upload) + self.assertEqual(upload.message_type, 'workflow_identifier') + + def test_upload_to_json(self): + json_data = self.upload.to_json() + self.assertTrue(isinstance(json_data, dict)) + + +class TestUploadData(unittest.TestCase): + + def setUp(self): + data = TEST_DATA['data'] + self.upload_data = UploadData(data) + + def test_tdei_project_group_id(self): + self.assertEqual(self.upload_data.tdei_project_group_id, '0b41ebc5-350c-42d3-90af-3af4ad3628fb') + self.upload_data.tdei_project_group_id = 'Test Project Group ID' + self.assertEqual(self.upload_data.tdei_project_group_id, 'Test Project Group ID') + + def test_user_id(self): + self.assertEqual(self.upload_data.user_id, 'c59d29b6-a063-4249-943f-d320d15ac9ab') + self.upload_data.user_id = 'Test user ID' + self.assertEqual(self.upload_data.user_id, 'Test user ID') + + def test_success(self): + self.upload_data.success = True + self.assertEqual(self.upload_data.success, True) + + def test_message(self): + self.upload_data.message = 'SOME_MESSAGE' + self.assertEqual(self.upload_data.message, 'SOME_MESSAGE') + + +class TestToJson(unittest.TestCase): + def test_to_json(self): + data = { + 'key1': 'value1', + 'key2': 'value2' + } + result = to_json(data) + self.assertEqual(result, {'key1': 'value1', 'key2': 'value2'}) + + +class TestValidationResult(unittest.TestCase): + def test_validation_result_init(self): + result = ValidationResult() + result.is_valid = True + result.validation_message = 'Validated' + self.assertTrue(result.is_valid) + self.assertEqual(result.validation_message, 'Validated') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_validation.py b/tests/unit_tests/test_validation.py new file mode 100644 index 0000000..e70e872 --- /dev/null +++ b/tests/unit_tests/test_validation.py @@ -0,0 +1,487 @@ +import os +import shutil +import unittest +from pathlib import Path +from unittest.mock import patch, MagicMock +from src.validation import Validation + +DOWNLOAD_FILE_PATH = f'{Path.cwd()}/downloads' +SAVED_FILE_PATH = f'{Path.cwd()}/tests/unit_tests/test_files' + +SUCCESS_FILE_NAME = 'valid.zip' +FAILURE_FILE_NAME = 'invalid.zip' +ID_MISSING_FILE_NAME = '_id_missing.zip' +EDGES_INVALID_FILE_NAME = 'edges_invalid.zip' +NODES_INVALID_FILE_NAME = 'nodes_invalid.zip' +POINTS_INVALID_FILE_NAME = 'points_invalid.zip' +INVALID_FILE_NAME = 'invalid_files.zip' +INVALID_GEOMETRY_FILE_NAME = 'invalid_geometry.zip' +MISSING_IDENTIFIER_FILE_NAME = 'missing_identifier.zip' +NO_ENTITY_FILE_NAME = 'no_entity.zip' +WRONG_DATATYPE_FILE_NAME = 'wrong_datatype.zip' + + +class TestOtherValidation(unittest.TestCase): + + @patch.object(Validation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{SUCCESS_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{SUCCESS_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{SUCCESS_FILE_NAME}' + + with patch.object(Validation, '__init__', return_value=None): + self.validator = Validation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = SUCCESS_FILE_NAME + self.validator.container_name = None + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test_download_single_file(self): + # Arrange + file_upload_path = DOWNLOAD_FILE_PATH + self.validator.storage_client = MagicMock() + self.validator.storage_client.get_file_from_url = MagicMock() + file = MagicMock() + file.file_path = 'text_file.txt' + file.get_stream = MagicMock(return_value=b'file_content') + self.validator.storage_client.get_file_from_url.return_value = file + + # Act + downloaded_file_path = self.validator.download_single_file(file_upload_path=file_upload_path) + + # Assert + self.validator.storage_client.get_file_from_url.assert_called_once_with(self.validator.container_name, + file_upload_path) + file.get_stream.assert_called_once() + with open(downloaded_file_path, 'rb') as f: + content = f.read() + self.assertEqual(content, b'file_content') + + def test_clean_up_file(self): + # Arrange + file_upload_path = DOWNLOAD_FILE_PATH + text_file_path = f'{file_upload_path}/text_file.txt' + f = open(text_file_path, "w") + f.write("Sample text") + f.close() + + # Act + Validation.clean_up = MagicMock() + + # Assert + # self.assertFalse(os.path.exists(text_file_path)) + + def test_clean_up_folder(self): + # Arrange + directory_name = 'temp' + directory_path = f'{DOWNLOAD_FILE_PATH}/{directory_name}' + is_exists = os.path.exists(directory_path) + if not is_exists: + os.makedirs(directory_path) + + # Act + Validation.clean_up = MagicMock() + + # Assert + # self.assertFalse(os.path.exists(directory_name)) + + +class TestValidation(unittest.TestCase): + + @patch.object(Validation, 'download_single_file') + def setUp(self, mock_download_single_file): + os.makedirs(DOWNLOAD_FILE_PATH, exist_ok=True) + source = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + destination = f'{DOWNLOAD_FILE_PATH}/{FAILURE_FILE_NAME}' + + if not os.path.isfile(destination): + shutil.copyfile(source, destination) + + file_path = f'{DOWNLOAD_FILE_PATH}/{FAILURE_FILE_NAME}' + + with patch.object(Validation, '__init__', return_value=None): + self.validator = Validation(file_path=file_path, storage_client=MagicMock()) + self.validator.file_path = file_path + self.validator.file_relative_path = FAILURE_FILE_NAME + self.validator.container_name = None + mock_download_single_file.return_value = file_path + + def tearDown(self): + pass + + def test_validate_with_invalid_file(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + + def test_validate_with_invalid_file_with_default_error_counts(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + self.assertLessEqual(len(result.validation_message), 20) + + def test_validate_with_invalid_file_with_specific_error_counts(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate(max_errors=10) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + self.assertLessEqual(len(result.validation_message), 10) + + def test_is_osw_valid_with_invalid_zip_file(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_format_file(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{FAILURE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_id_missing_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{ID_MISSING_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_id_missing_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{ID_MISSING_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_invalid_edges_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{EDGES_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_edges_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{EDGES_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_invalid_nodes_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{NODES_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_nodes_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{NODES_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_invalid_points_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{POINTS_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_points_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{POINTS_INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_invalid_files_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_files_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{INVALID_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_invalid_geometry_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{INVALID_GEOMETRY_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_invalid_geometry_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{INVALID_GEOMETRY_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_missing_identifier_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{MISSING_IDENTIFIER_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_missing_identifier_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{MISSING_IDENTIFIER_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_no_entity_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{NO_ENTITY_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_no_entity_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{NO_ENTITY_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_validate_with_wrong_datatype_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{WRONG_DATATYPE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.validate() + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_is_osw_valid_with_wring_datatype_zip(self): + # Arrange + file_path = f'{SAVED_FILE_PATH}/{WRONG_DATATYPE_FILE_NAME}' + expected_downloaded_file_path = file_path + self.validator.download_single_file = MagicMock(return_value=expected_downloaded_file_path) + Validation.clean_up = MagicMock() + + # Act + result = self.validator.is_osw_valid(max_errors=2) + + # Assert + self.assertFalse(result.is_valid) + self.assertIsInstance(result.validation_message, list) + Validation.clean_up.assert_called_once_with(file_path) + + def test_download_single_file_exception(self): + # Arrange + file_upload_path = DOWNLOAD_FILE_PATH + self.validator.storage_client = MagicMock() + self.validator.storage_client.get_file_from_url = MagicMock() + file = MagicMock() + file.file_path = 'text_file.txt' + file.get_stream = MagicMock(return_value=b'file_content') + self.validator.storage_client.get_file_from_url.return_value = file + + # Act + downloaded_file_path = self.validator.download_single_file(file_upload_path=file_upload_path) + + # Assert + self.validator.storage_client.get_file_from_url.assert_called_once_with(self.validator.container_name, + file_upload_path) + file.get_stream.assert_called_once() + with open(downloaded_file_path, 'rb') as f: + content = f.read() + self.assertEqual(content, b'file_content') + + +if __name__ == '__main__': + unittest.main()