Stage to Prod upmerge

.github/workflows/dev_workflow.yaml

@@ -0,0 +1,43 @@
+---
+######### Dev Workflow  ########
+on:
+  pull_request:
+    branches: [dev]
+    types:
+      - closed
+  workflow_dispatch:
+permissions:
+    id-token: write
+    contents: read
+
+jobs:
+  Build:
+    environment: Development
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.REGISTRY_DOMAIN }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+      - name: Publish image to Azure Registry
+        run: |
+          docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
+          docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
+  Deploy:
+    needs: Build
+    environment:
+      name: Development
+    runs-on: ubuntu-latest
+    steps:
+    - name: Login to Azure
+      uses: azure/login@v2.0.0
+      with:
+        creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
+    - name: Deploy to Dev
+      uses: azure/webapps-deploy@v2
+      with:
+        app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
+        images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}

.github/workflows/prod_workflow.yaml

@@ -0,0 +1,43 @@
+---
+######### Prod Workflow  ########
+on:
+  pull_request:
+    branches: [main]
+    types:
+      - closed
+  workflow_dispatch:
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  Build:
+    environment: Production
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.REGISTRY_DOMAIN }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+      - name: Publish image to Azure Registry
+        run: |
+          docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
+          docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
+  Deploy:
+    needs: Build
+    environment:
+      name: Production
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Azure
+        uses: azure/login@v2.0.0
+        with:
+          creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
+      - name: Deploy to Production
+        uses: azure/webapps-deploy@v2
+        with:
+          app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
+          images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}.

.github/workflows/stage_workflow.yaml

@@ -0,0 +1,43 @@
+---
+######### Stage Workflow  ########
+on:
+  pull_request:
+    branches: [stage]
+    types:
+      - closed
+  workflow_dispatch:
+permissions:
+    id-token: write
+    contents: read
+
+jobs:
+  Build:
+    environment: Stage
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/docker-login@v1
+        with:
+          login-server: ${{ secrets.REGISTRY_DOMAIN }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+      - name: Publish image to Azure Registry
+        run: |
+          docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} .
+          docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags
+  Deploy:
+    needs: Build
+    environment:
+      name: Stage
+    runs-on: ubuntu-latest
+    steps:
+    - name: Login to Azure
+      uses: azure/login@v2.0.0
+      with:
+        creds: ${{secrets.TDEI_CORE_AZURE_CREDS}}
+    - name: Deploy to Stage
+      uses: azure/webapps-deploy@v2
+      with:
+        app-name: ${{ secrets.AZURE_WEBAPP_NAME }}
+        images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}.

.github/workflows/unit_tests.yaml

@@ -0,0 +1,51 @@
+---
+name: Unit Tests
+
+#############################
+# Start the job on all push #
+#############################
+on:
+  push:
+    branches-ignore:
+      - '**'
+    # Remove the line above to run when pushing to master
+  pull_request:
+    branches: [master, dev, stage]
+
+###############
+# Set the Job #
+###############
+jobs:
+  UnitTest:
+    name: Unit Test Cases
+    # Set the agent to run on
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10" # Use the appropriate  Python version
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Run unit tests
+        run: |
+          python test_report.py
+          coverage run --source=src -m unittest discover -s tests/
+          coverage report -m
+          exit_status=$?
+
+          # Set the exit status as an output for later use
+          echo "::set-output name=exit_status::$exit_status"
+
+      - name: Archive Coverage Report
+        if: ${{ always() }}  # Upload the coverage report even if tests fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: htmlcov
+          path: htmlcov

.gitignore

@@ -157,4 +157,5 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+reports/

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY ./src /code/src
+EXPOSE 8080
+CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "8080"]

README.md

@@ -1,2 +1,151 @@
 # TDEI-python-osw-validation
-Python service to Validate the OSW file that is uploaded.
+
+## Introduction 
+Service to Validate the OSW files that is uploaded. At the moment, the service does the following:
+- Listens to the topic which is mentioned in `.env` file for any new message (that is triggered when a file is uploaded), example  `UPLOAD_TOPIC=osw-upload` 
+- Consumes the message and perform following checks - 
+  - Download the file locally 
+  - File location is in the message `data.meta.file_upload_path`
+  - Uses `python-osw-validation` to validate the file
+  - Adds the `isValid` and `validationMessage` keys to the original message 
+- Publishes the result to the topic mentioned in `.env` file, example `VALIDATION_TOPIC=osw-validation`
+
+## Getting Started
+The project is built on Python with FastAPI framework. All the regular nuances for a Python project are valid for this.
+
+### System requirements
+| Software   | Version |
+|------------|---------|
+| Python     | 3.10.x  |
+
+
+### Connectivity to cloud
+- Connecting this to cloud will need the following in the `.env` file
+
+```bash
+QUEUECONNECTION=xxxx
+STORAGECONNECTION=xxxx
+VALIDATION_REQ_TOPIC=xxxx
+VALIDATION_REQ_SUB=xxxx
+VALIDATION_RES_TOPIC=xxxx
+CONTAINER_NAME=xxxx
+AUTH_PERMISSION_URL=xxx
+
+```
+
+The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package.
+`QUEUECONNECTION` is used to send out the messages and listen to messages.
+
+
+### How to Set up and Build
+Follow the steps to install the python packages required for both building and running the application
+
+1. Setup virtual environment
+    ```
+    python3.10 -m venv .venv
+    source .venv/bin/activate
+    ```
+
+2. Install the dependencies. Run the following command in terminal on the same directory as `requirements.txt`
+    ```
+    # Installing requirements
+    pip install -r requirements.txt
+    ```
+### How to Run the Server/APIs   
+
+1. The http server by default starts with `8000` port
+2. Run server
+    ```
+    uvicorn src.main:app --reload
+    ```
+3. By default `get` call on `localhost:8000/health` gives a sample response
+4. Other routes include a `ping` with get and post. Make `get` or `post` request to `http://localhost:8000/health/ping`
+5. Once the server starts, it will start to listening the subscriber(`VALIDATION_REQ_SUB` should be in env file)
+
+
+#### Request Format
+
+```json
+  {
+    "messageId": "tdei_record_id",
+    "messageType": "workflow_identifier",
+    "data": {
+      "file_upload_path": "file_upload_path",
+      "user_id": "user_id",
+      "tdei_project_group_id": "tdei_project_group_id"
+    } 
+  }
+```
+
+#### Response Format
+
+```json
+  {
+    "messageId": "tdei_record_id",
+    "messageType": "workflow_identifier",
+    "data": {
+      "file_upload_path": "file_upload_path",
+      "user_id": "user_id",
+      "tdei_project_group_id": "tdei_project_group_id",
+      "success": true/false,
+      "message": "message" // if false the error string else empty string
+    },
+    "publishedDate": "published date"
+  }
+```
+
+
+### How to Set up and run the Tests
+
+Make sure you have set up the project properly before running the tests, see above for `How to Setup and Build`.
+
+#### How to run test harness
+1. Add the new set of test inside `tests/test_harness/tests.json` file like -
+    ```
+    {
+     "Name": "Test Name",
+     "Input_file": "test_files/osw_test_case1.json", // Input file path which you want to provide to the test
+     "Result": true/false // Defining the test output 
+     }
+    ```
+2. Test Harness would require a valid `.env` file.
+3. To run the test harness `python tests/test_harness/run_tests.py` 
+#### How to run unit test cases
+1. `.env` file is not required for Unit test cases.
+2. To run the unit test cases
+   1. `python test_report.py`
+   2. Above command will run all test cases and generate the html report, in `reports` folder at the root level.
+3. To run the coverage
+   1. `python -m coverage run --source=src -m unittest discover -s tests/unit_tests`
+   2. Above command will run all the unit test cases.
+   3. To generate the coverage report in console
+      1. `coverage report`
+      2. Above command will generate the code coverage report in terminal. 
+   4. To generate the coverage report in html.
+      1. `coverage html`
+      2. Above command will generate the html report, and generated html would be in `htmlcov` directory at the root level.
+   5. _NOTE :_ To run the `html` or `report` coverage, 3.i) command is mandatory
+
+#### How to run integration test cases
+1. `.env` file is required for Unit test cases.
+2. To run the integration test cases, run the below command
+   1. `python test_integration.py`
+   2. Above command will run all integration test cases and generate the html report, in `reports` folder at the root level.
+
+
+### Messaging
+
+This microservice deals with two topics/queues. 
+- upload queue from osw-upload
+- validation queue from osw-validation
+
+
+#### Incoming
+The incoming messages will be from the upload queue `osw-upload`.
+The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json)
+
+#### Outgoing
+The outgoing messages will be to the `osw-validation` topic.
+The format of the message is at [osw-validation.json](./src/assets/osw-validation.json)
+
+

requirements.txt

@@ -0,0 +1,10 @@
+psutil==5.9.5
+fastapi==0.88.0
+python-dotenv==0.21.0
+pydantic==1.10.4
+python-ms-core==0.0.18
+uvicorn==0.20.0
+coverage==7.2.7
+html_testRunner==1.2.1
+httpx==0.24.1
+python-osw-validation==0.2.3

src/assets/osw-upload.json

@@ -0,0 +1,9 @@
+{
+  "messageId": "c8c76e89f30944d2b2abd2491bd95337",
+  "messageType": "workflow_identifier",
+  "data": {
+    "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip",
+    "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab",
+    "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb"
+  }
+}

src/assets/osw-validation.json

@@ -0,0 +1,11 @@
+{
+  "messageId": "c8c76e89f30944d2b2abd2491bd95337",
+  "messageType": "workflow_identifier",
+  "data": {
+    "file_upload_path": "https://tdeisamplestorage.blob.core.windows.net/osw/test_upload/valid.zip",
+    "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab",
+    "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb",
+    "success": true,
+    "message": ""
+  }
+}