From ccc06bd18b9be60f7945b5dac2a4cf81d39caeab Mon Sep 17 00:00:00 2001 From: karthikkadajji Date: Tue, 17 Oct 2023 12:42:20 +0200 Subject: [PATCH 1/6] Task-454 add workflow --- .github/workflows/dev_workflow.yaml | 36 +++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/dev_workflow.yaml diff --git a/.github/workflows/dev_workflow.yaml b/.github/workflows/dev_workflow.yaml new file mode 100644 index 0000000..32c444e --- /dev/null +++ b/.github/workflows/dev_workflow.yaml @@ -0,0 +1,36 @@ +--- +######### Dev Workflow ######## +on: + pull_request: + branches: [dev] + types: + - closed + workflow_dispatch: + +jobs: + Build: + environment: Development + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'master' && 'prod' || github.ref_name }}${{ github.ref_name != 'master' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + Deploy: + needs: Build + environment: + name: Development + runs-on: ubuntu-latest + steps: + - name: Deploy to Dev + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} diff --git a/README.md b/README.md index 3c10960..d219c7d 100644 --- a/README.md +++ b/README.md @@ -112,3 +112,4 @@ The format is mentioned in [osw-upload.json](./src/assets/osw-upload.json) #### Outgoing The outgoing messages will be to the `osw-validation` topic. The format of the message is at [osw-validation.json](./src/assets/osw-validation.json) + From 20695d865737e8bf7e1ef5c57f101451420314d0 Mon Sep 17 00:00:00 2001 From: karthikkadajji Date: Tue, 17 Oct 2023 13:05:48 +0200 Subject: [PATCH 2/6] Task-454 modify dockerfile --- dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 dockerfile diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..66e722f --- /dev/null +++ b/dockerfile @@ -0,0 +1,8 @@ +FROM python:3.10 +WORKDIR /code +COPY ./requirements.txt /code/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +RUN python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple python-osw-validation==0.0.3 +COPY ./src /code/src +EXPOSE 8080 +CMD ["uvicorn", "src.main:app", "--reload", "--host", "0.0.0.0", "--port", "8080"] From 96882ddf5f23d471efbddf2e8549b281ef975835 Mon Sep 17 00:00:00 2001 From: karthikkadajji Date: Tue, 17 Oct 2023 13:22:13 +0200 Subject: [PATCH 3/6] Task-454 modify workflow to trigger the build and deploy for test --- .github/workflows/dev_workflow.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dev_workflow.yaml b/.github/workflows/dev_workflow.yaml index 32c444e..0382471 100644 --- a/.github/workflows/dev_workflow.yaml +++ b/.github/workflows/dev_workflow.yaml @@ -5,6 +5,9 @@ on: branches: [dev] types: - closed + push: + branches: + - 'Task-454' workflow_dispatch: jobs: From a5679510777bce8d33a1547b474a4cfaac78f000 Mon Sep 17 00:00:00 2001 From: Sujata Date: Tue, 17 Oct 2023 19:03:25 +0530 Subject: [PATCH 4/6] Changed docker file name --- dockerfile => Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dockerfile => Dockerfile (100%) diff --git a/dockerfile b/Dockerfile similarity index 100% rename from dockerfile rename to Dockerfile From d3f49bfb0fb30d0d3efbfefa1ec2c1bad354ff5a Mon Sep 17 00:00:00 2001 From: karthikkadajji Date: Wed, 18 Oct 2023 09:36:56 +0200 Subject: [PATCH 5/6] Task-454 modify workflow to trigger --- .github/workflows/dev_workflow.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/dev_workflow.yaml b/.github/workflows/dev_workflow.yaml index 0382471..32c444e 100644 --- a/.github/workflows/dev_workflow.yaml +++ b/.github/workflows/dev_workflow.yaml @@ -5,9 +5,6 @@ on: branches: [dev] types: - closed - push: - branches: - - 'Task-454' workflow_dispatch: jobs: From b39ed5e044e0c85c73cc7646a7dbaf9d4013b13f Mon Sep 17 00:00:00 2001 From: Sujata Date: Wed, 18 Oct 2023 17:19:24 +0530 Subject: [PATCH 6/6] Fixed following things - - Added osw_schema_version and publication_date to request schema - Removed valid_from, valid_to and tdei_station_id keys from request schema - Added few more unit test cases - Modified existing unit test cases --- src/assets/osw-upload.json | 3 +- src/models/queue_message_content.py | 34 ++++++-------- src/validation.py | 2 +- test_report.py | 2 +- ...ntant.py => test_queue_message_content.py} | 46 ++++++++++++++++++- 5 files changed, 60 insertions(+), 27 deletions(-) rename tests/unit_tests/{test_queue_message_contant.py => test_queue_message_content.py} (71%) diff --git a/src/assets/osw-upload.json b/src/assets/osw-upload.json index 5b33d88..b52861a 100644 --- a/src/assets/osw-upload.json +++ b/src/assets/osw-upload.json @@ -8,8 +8,7 @@ "collected_by": "6D3E5B8C-FB16-4B6A-9436-72FD24756CC9", "collection_date": "2022-11-22T09:43:07.978Z", "collection_method": "manual", - "valid_from": "2022-11-22T09:43:07.978Z", - "valid_to": "2022-11-22T09:43:07.978Z", + "publication_date": "2022-11-22T09:43:07.978Z", "data_source": "local", "polygon": { "coordinates": [ diff --git a/src/models/queue_message_content.py b/src/models/queue_message_content.py index 8815b44..0c14a10 100644 --- a/src/models/queue_message_content.py +++ b/src/models/queue_message_content.py @@ -113,14 +113,13 @@ def to_json(self): class Request: def __init__(self, data: dict): self._tdei_org_id = data.get('tdei_org_id', '') - self._tdei_station_id = data.get('tdei_station_id', '') self._collected_by = data.get('collected_by', '') self._collection_date = data.get('collection_date', '') self._collection_method = data.get('collection_method', '') - self._valid_from = data.get('valid_from', '') - self._valid_to = data.get('valid_to', '') + self._publication_date = data.get('publication_date', '') self._data_source = data.get('data_source', '') self._polygon = data.get('polygon', {}) + self._osw_schema_version = data.get('osw_schema_version', '') @property def tdei_org_id(self): return self._tdei_org_id @@ -128,12 +127,6 @@ def tdei_org_id(self): return self._tdei_org_id @tdei_org_id.setter def tdei_org_id(self, value): self._tdei_org_id = value - @property - def tdei_station_id(self): return self._tdei_station_id - - @tdei_station_id.setter - def tdei_station_id(self, value): self._tdei_station_id = value - @property def collected_by(self): return self._collected_by @@ -152,18 +145,6 @@ def collection_method(self): return self._collection_method @collection_method.setter def collection_method(self, value): self._collection_method = value - @property - def valid_from(self): return self._valid_from - - @valid_from.setter - def valid_from(self, value): self._valid_from = value - - @property - def valid_to(self): return self._valid_to - - @valid_to.setter - def valid_to(self, value): self._valid_to = value - @property def data_source(self): return self._data_source @@ -176,6 +157,17 @@ def polygon(self): return self._polygon @polygon.setter def polygon(self, value): self._polygon = value + @property + def publication_date(self): return self._publication_date + + @publication_date.setter + def publication_date(self, value): self._publication_date = value + + @property + def osw_schema_version(self): return self._osw_schema_version + + @osw_schema_version.setter + def osw_schema_version(self, value): self._osw_schema_version = value class Meta: diff --git a/src/validation.py b/src/validation.py index eeb5062..0b716a3 100644 --- a/src/validation.py +++ b/src/validation.py @@ -41,7 +41,7 @@ def is_osw_valid(self) -> ValidationResult: result.is_valid = validation_result.is_valid if not result.is_valid: result.validation_message = validation_result.errors - # logger.error(f' Error While Validating File: {str(validation_result.errors)}') + logger.error(f' Error While Validating File: {str(validation_result.errors)}') Validation.clean_up(downloaded_file_path) else: result.validation_message = 'Failed to validate because unknown file format' diff --git a/test_report.py b/test_report.py index 7901976..f93085d 100644 --- a/test_report.py +++ b/test_report.py @@ -2,7 +2,7 @@ import HtmlTestRunner # Define your test cases -from tests.unit_tests.test_queue_message_contant import TestUpload, TestUploadData, TestRequest, \ +from tests.unit_tests.test_queue_message_content import TestUpload, TestUploadData, TestRequest, \ TestMeta, TestResponse, TestToJson, TestValidationResult from tests.unit_tests.test_validation import TestSuccessValidation, TestFailureValidation from tests.unit_tests.test_osw_validator import TestOSWValidator diff --git a/tests/unit_tests/test_queue_message_contant.py b/tests/unit_tests/test_queue_message_content.py similarity index 71% rename from tests/unit_tests/test_queue_message_contant.py rename to tests/unit_tests/test_queue_message_content.py index 9ee53b7..91aec25 100644 --- a/tests/unit_tests/test_queue_message_contant.py +++ b/tests/unit_tests/test_queue_message_content.py @@ -92,7 +92,7 @@ def test_user_id(self): class TestRequest(unittest.TestCase): def setUp(self): - data = TEST_DATA['data'] + data = TEST_DATA['data']['request'] self.request = Request(data) def test_tdei_org_id(self): @@ -100,7 +100,43 @@ def test_tdei_org_id(self): self.request.tdei_org_id = 'Test org ID' self.assertEqual(self.request.tdei_org_id, 'Test org ID') - # Add more test cases for other properties of Request + def test_collected_by(self): + self.assertEqual(self.request.collected_by, '6D3E5B8C-FB16-4B6A-9436-72FD24756CC9') + self.request.collected_by = 'Test Collected By' + self.assertEqual(self.request.collected_by, 'Test Collected By') + + def test_collection_date(self): + self.assertEqual(self.request.collection_date, '2022-11-22T09:43:07.978Z') + self.request.collection_date = 'Test Collected Date' + self.assertEqual(self.request.collection_date, 'Test Collected Date') + + def test_collection_method(self): + self.assertEqual(self.request.collection_method, 'manual') + self.request.collection_method = 'Test Collected Method' + self.assertEqual(self.request.collection_method, 'Test Collected Method') + + def test_publication_date(self): + self.assertEqual(self.request.publication_date, '2022-11-22T09:43:07.978Z') + self.request.publication_date = 'Test Publication Date' + self.assertEqual(self.request.publication_date, 'Test Publication Date') + + def test_data_source(self): + self.assertEqual(self.request.data_source, 'local') + self.request.data_source = 'Test Data Source' + self.assertEqual(self.request.data_source, 'Test Data Source') + + def test_polygon(self): + self.request.polygon = { + 'test': 'Polygon' + } + self.assertEqual(self.request.polygon, { + 'test': 'Polygon' + }) + + def test_osw_schema_version(self): + self.assertEqual(self.request.osw_schema_version, '1.0.0') + self.request.osw_schema_version = '1.1.0' + self.assertEqual(self.request.osw_schema_version, '1.1.0') class TestMeta(unittest.TestCase): @@ -127,6 +163,12 @@ def test_success(self): self.response.success = False self.assertEqual(self.response.success, False) + def test_message(self): + self.assertEqual(self.response.message, + 'File uploaded for the Organization : 0b41ebc5-350c-42d3-90af-3af4ad3628fb with tdei record id : c8c76e89f30944d2b2abd2491bd95337') + self.response.message = 'Successful' + self.assertEqual(self.response.message, 'Successful') + class TestToJson(unittest.TestCase): def test_to_json(self):