Added unit test cases

.gitignore

@@ -157,4 +157,5 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+reports/

README.md

@@ -1,2 +1,114 @@
 # TDEI-python-osw-validation
-Python service to Validate the OSW file that is uploaded.
+
+## Introduction 
+Service to Validate the OSW files that is uploaded. At the moment, the service does the following:
+- Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example  `UPLOAD_TOPIC=osw-upload` 
+- Consumes the message and perform following checks - 
+  - Download the file locally 
+  - File location is in the message `data.meta.file_upload_path`
+  - Uses `python-osw-validation` to validate the file
+  - Adds the `isValid` and `validationMessage` keys to the original message 
+- Publishes the result to the topic mentioned in `.env` file, example `VALIDATION_TOPIC=osw-validation`
+
+## Getting Started
+The project is built on Python with FastAPI framework. All the regular nuances for a Python project are valid for this.
+
+### System requirements
+| Software   | Version |
+|------------|---------|
+| Python     | 3.10.x  |
+
+
+### Connectivity to cloud
+- Connecting this to cloud will need the following in the `.env` file
+
+```bash
+UPLOAD_TOPIC=xxxx
+UPLOAD_SUBSCRIPTION=xxxx
+VALIDATION_TOPIC=xxxx
+QUEUECONNECTION=xxxx
+STORAGECONNECTION=xxxx
+```
+
+The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package.
+`QUEUECONNECTION` is used to send out the messages and listen to messages.
+
+### How to Setup and Build
+Follow the steps to install the python packages required for both building and running the application
+
+1. Setup virtual environment
+    ```
+    python3.10 -m venv .venv
+    source .venv/bin/activate
+    ```
+
+2. Install the dependencies. Run the following command in terminal on the same directory as `requirements.txt`
+    ```
+    # Installing requirements
+    pip install -r requirements.txt
+    # Installing python-osw-validation package
+    pip install -i https://test.pypi.org/simple/ python-osw-validation==0.0.3
+    ```
+### How to Run the Server/APIs   
+
+1. The http server by default starts with `8000` port
+2. Run server
+    ```
+    uvicorn src.main:app --reload
+    ```
+3. By default `get` call on `localhost:8000/health` gives a sample response
+4. Other routes include a `ping` with get and post. Make `get` or `post` request to `http://localhost:8000/health/ping`
+5. Once the server starts, it will start to listening the subscriber(`UPLOAD_SUBSCRIPTION` should be in env file)
+
+### How to Setup and run the Tests
+
+Make sure you have set up the project properly before running the tests, see above for `How to Setup and Build`.
+
+#### How to run test harness
+1. Add the new set of test inside `tests/test_harness/tests.json` file like -
+    ```
+    {
+     "Name": "Test Name",
+     "Input_file": "test_files/osw_test_case1.json", // Input file path which you want to provide to the test
+     "Result": true/false // Defining the test output 
+     }
+    ```
+2. Test Harness would require a valid `.env` file.
+3. To run the test harness `python tests/test_harness/run_tests.py` 
+#### How to run unit test cases
+1. `.env` file is not required for Unit test cases.
+2. To run the unit test cases
+   1. `python test_report.py`
+   2. Above command will run all test cases and generate the html report, in `reports` folder at the root level.
+3. To run the coverage
+   1. `coverage run --source=src -m unittest discover -s tests/unit_tests`
+   2. Above command will run all the unit test cases.
+   3. To generate the coverage report in console
+      1. `coverage report`
+      2. Above command will generate the code coverage report in terminal. 
+   4. To generate the coverage report in html.
+      1. `coverage html`
+      2. Above command will generate the html report, and generated html would be in `htmlcov` directory at the root level.
+   5. _NOTE :_ To run the `html` or `report` coverage, 3.i) command is mandatory
+
+#### How to run integration test cases
+1. `.env` file is required for Unit test cases.
+2. To run the integration test cases, run the below command
+   1. `python test_integration.py`
+   2. Above command will run all integration test cases and generate the html report, in `reports` folder at the root level.
+
+
+### Messaging
+
+This microservice deals with two topics/queues. 
+- upload queue from osw-upload
+- validation queue from osw-validation
+
+
+#### Incoming
+The incoming messages will be from the upload queue `osw-upload`.
+The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json)
+
+#### Outgoing
+The outgoing messages will be to the `osw-validation` topic.
+The format of the message is at [osw-validation.json](./src/assets/osw-validation.json)

requirements.txt

@@ -3,4 +3,10 @@ fastapi==0.88.0
 python-dotenv==0.21.0
 pydantic==1.10.4
 python-ms-core==0.0.16
-uvicorn==0.20.0
+uvicorn==0.20.0
+coverage==7.2.7
+html_testRunner==1.2.1
+httpx==0.24.1
+
+jsonschema==4.19.1
+zipfile36==0.1.3

src/assets/osw-upload.json

@@ -0,0 +1,77 @@
+{
+  "messageType": "osw-upload",
+  "publishedDate": "2023-02-08T08:33:36.267213Z",
+  "data": {
+    "stage": "OSW-Upload",
+    "request": {
+      "tdei_org_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
+      "collected_by": "6D3E5B8C-FB16-4B6A-9436-72FD24756CC9",
+      "collection_date": "2022-11-22T09:43:07.978Z",
+      "collection_method": "manual",
+      "valid_from": "2022-11-22T09:43:07.978Z",
+      "valid_to": "2022-11-22T09:43:07.978Z",
+      "data_source": "local",
+      "polygon": {
+        "coordinates": [
+          [
+            [
+              77.5873101438346,
+              12.975857172137523
+            ],
+            [
+              77.58650666918095,
+              12.97272532145334
+            ],
+            [
+              77.5869485802404,
+              12.970219812510848
+            ],
+            [
+              77.58739049129997,
+              12.967792576658127
+            ],
+            [
+              77.5886760507459,
+              12.967753427498707
+            ],
+            [
+              77.58911796180547,
+              12.969123644412946
+            ],
+            [
+              77.59016247885529,
+              12.972921063276544
+            ],
+            [
+              77.58984108899386,
+              12.973429991296015
+            ],
+            [
+              77.5904035212514,
+              12.974995917131267
+            ],
+            [
+              77.58863587701325,
+              12.975074213164191
+            ],
+            [
+              77.5873101438346,
+              12.975857172137523
+            ]
+          ]
+        ]
+      },
+      "osw_schema_version": "1.0.0"
+    },
+    "meta": {
+      "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip"
+    },
+    "response": {
+      "success": true,
+      "message": "File uploaded for the Organization : 0b41ebc5-350c-42d3-90af-3af4ad3628fb with tdei record id : c8c76e89f30944d2b2abd2491bd95337"
+    },
+    "tdei_record_id": "c8c76e89f30944d2b2abd2491bd95337",
+    "tdei_org_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
+    "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab"
+  }
+}

src/assets/osw-validation.json

@@ -0,0 +1,78 @@
+{
+    "messageType": "osw-validation",
+    "publishedDate": "2023-02-08T08:33:36.267213Z",
+    "data": {
+        "stage": "OSW-Validation",
+        "request": {
+            "tdei_org_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
+            "collected_by": "6D3E5B8C-FB16-4B6A-9436-72FD24756CC9",
+            "collection_date": "2022-11-22T09:43:07.978Z",
+            "collection_method": "manual",
+            "valid_from": "2022-11-22T09:43:07.978Z",
+            "valid_to": "2022-11-22T09:43:07.978Z",
+            "data_source": "local",
+            "polygon": {
+                "coordinates": [
+                    [
+                        [
+                            77.5873101438346,
+                            12.975857172137523
+                        ],
+                        [
+                            77.58650666918095,
+                            12.97272532145334
+                        ],
+                        [
+                            77.5869485802404,
+                            12.970219812510848
+                        ],
+                        [
+                            77.58739049129997,
+                            12.967792576658127
+                        ],
+                        [
+                            77.5886760507459,
+                            12.967753427498707
+                        ],
+                        [
+                            77.58911796180547,
+                            12.969123644412946
+                        ],
+                        [
+                            77.59016247885529,
+                            12.972921063276544
+                        ],
+                        [
+                            77.58984108899386,
+                            12.973429991296015
+                        ],
+                        [
+                            77.5904035212514,
+                            12.974995917131267
+                        ],
+                        [
+                            77.58863587701325,
+                            12.975074213164191
+                        ],
+                        [
+                            77.5873101438346,
+                            12.975857172137523
+                        ]
+                    ]
+                ]
+            },
+            "osw_schema_version": "1.0.0"
+        },
+        "meta": {
+            "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/2023%2FFEBRUARY%2F0b41ebc5-350c-42d3-90af-3af4ad3628fb%2Fvalid_c8c76e89f30944d2b2abd2491bd95337.zip",
+            "isValid": true
+        },
+        "response": {
+            "success": true,
+            "message": "Validation successful"
+        },
+        "tdei_record_id": "c8c76e89f30944d2b2abd2491bd95337",
+        "tdei_org_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
+        "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab"
+    }
+}

src/main.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI, APIRouter, Depends, status
 from functools import lru_cache
 from .config import Settings
-from .osw_validator import OswValidator
+from .osw_validator import OSWValidator
 
 app = FastAPI()
 
@@ -18,7 +18,7 @@ def get_settings():
 @app.on_event('startup')
 async def startup_event(settings: Settings = Depends(get_settings)) -> None:
     try:
-        OswValidator().start_listening()
+        OSWValidator()
     except:
         print('\n\n\x1b[31m Application startup failed due to missing or invalid .env file \x1b[0m')
         print('\x1b[31m Please provide the valid .env file and .env file should contains following parameters\x1b[0m')

src/models/queue_message_content.py

@@ -60,7 +60,7 @@ def data_from(self):
             try:
                 return Upload(data=message)
             except Exception as e:
-                error = str(e).replace('GTFSFlexUpload', 'Invalid parameter,')
+                error = str(e).replace('Upload', 'Invalid parameter,')
                 raise TypeError(error)
 
 
@@ -121,7 +121,6 @@ def __init__(self, data: dict):
         self._valid_to = data.get('valid_to', '')
         self._data_source = data.get('data_source', '')
         self._polygon = data.get('polygon', {})
-        self._pathways_schema_version = data.get('pathways_schema_version', '')
 
     @property
     def tdei_org_id(self): return self._tdei_org_id
@@ -177,11 +176,6 @@ def polygon(self): return self._polygon
     @polygon.setter
     def polygon(self, value): self._polygon = value
 
-    @property
-    def pathways_schema_version(self): return self._pathways_schema_version
-
-    @pathways_schema_version.setter
-    def pathways_schema_version(self, value): self._pathways_schema_version = value
 
 
 class Meta: