merged with branch 63-MSME-application-endpoints

.dockerignore

@@ -1,5 +1,4 @@
-# Dotfiles
-**/.*
+
 
 # Docker
 **/Dockerfile*
@@ -18,9 +17,9 @@ README.rst
 # Configuration
 pyproject.toml
 setup.cfg
-*.env
 *.db
 
+
 # Testing
 **/tests
 pytest.ini

.envtest

@@ -1,18 +1,21 @@
-COGNITO_CLIENT_ID=********\*\*********
-AWS_ACCESS_KEY=********\*\*********
-AWS_CLIENT_SECRET=********\*\*********
-AWS_REGION=********\*\*********
-COGNITO_POOL_ID=********\*\*********
-EMAIL_SENDER_ADDRESS=********\*\*********
-DATABASE_URL=********\*\*********
-SECOP_APP_TOKEN=********\*\*********
-COLOMBIA_SECOP_APP_TOKEN=********\*\*********
-FRONTEND_URL=********\*\*********
-HASH_KEY=********\*\*********
-FRONT_PUBLIC_IMAGES_ES = ********\*\*********
-FRONT_PUBLIC_IMAGES_EN = ********\*\*********
-TEMPORAL_BUCKET = ********\*\*********
-FACEBOOK_LINK = ********\*\*********
-TWITTER_LINK = ********\*\*********
-LINK_LINK = ********\*\*********
+AWS_ACCESS_KEY = XXXXXX
+AWS_CLIENT_SECRET = XXXXXX
+AWS_REGION = XXXXXX
+COGNITO_POOL_ID = XXXXXX
+COGNITO_CLIENT_ID = XXXXXX
+COGNITO_CLIENT_SECRE = XXXXXX
+EMAIL_SENDER_ADDRESS = XXXXXX
+FRONTEND_URL = XXXXXX
+SENTRY_DNS = XXXXXX
+DATABASE_URL = XXXXXX
 
+COLOMBIA_SECOP_APP_TOKEN = XXXXXX
+SECOP_PAGINATION_LIMIT = XXXXXX
+HASH_KEY = XXXXXX
+TEST_MAIL_RECEIVER = XXXXXX
+
+
+IMAGES_LANG_SUBPATH = XXXXXX
+FACEBOOK_LINK = XXXXXX
+TWITTER_LINK = XXXXXX
+LINK_LINK = XXXXXX

.github/workflows/ci.yml

@@ -8,15 +8,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: pip
           cache-dependency-path: '**/requirements*.txt'
       - run: pip install -r requirements_dev.txt
       - name: Run checks and tests
         env:
           DATABASE_URL: postgresql://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres
-        run: |
-          pytest -W error
+        run: pytest -W error --cov app
       - env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: coveralls --service=github

.github/workflows/lint.yml

@@ -2,6 +2,7 @@ name: Lint
 on: [push, pull_request]
 env:
   BASEDIR: https://raw.githubusercontent.com/open-contracting/standard-maintenance-scripts/main
+  STANDARD_MAINTENANCE_SCRIPTS_IGNORE: python-jose
 jobs:
   build:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
@@ -10,12 +11,17 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: pip
           cache-dependency-path: '**/requirements*.txt'
+      - id: changed-files
+        uses: tj-actions/changed-files@v36
+      - uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: pip-compile --files ${{ steps.changed-files.outputs.all_changed_files }}
       - shell: bash
         run: curl -s -S --retry 3 $BASEDIR/tests/install.sh | bash -
       - shell: bash
         run: curl -s -S --retry 3 $BASEDIR/tests/script.sh | bash -
       - run: pip install -r requirements_dev.txt
-      - run: pytest /tmp/test_requirements.py
+      - run: pytest /tmp/test_requirements.py

.pre-commit-config.yaml

@@ -1,8 +1,9 @@
 ci:
   autoupdate_schedule: quarterly
+  skip: [pip-compile]
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8

Dockerfile

@@ -1,17 +1,26 @@
-# 
-FROM python:3.9
+FROM python:3.11.3-alpine3.18
 
 # 
 WORKDIR /code
 
-# 
+#
+
 COPY ./requirements.txt /code/requirements.txt
-RUN pip install --ignore-installed uvicorn==0.22.0
-# 
-RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN \
+ apk add --no-cache python3 postgresql-libs && \
+ apk add --no-cache --virtual .build-deps gcc python3-dev musl-dev postgresql-dev libffi-dev openssl-dev && \
+ pip install --ignore-installed uvicorn==0.22.0 && \
+ pip install --no-cache-dir --upgrade -r /code/requirements.txt && \
+ apk --purge del .build-deps
 
 # 
+
+COPY ./.env /code/.env
+COPY ./alembic.ini /code/alembic.ini
+COPY ./migrations /code/migrations
 COPY ./app /code/app
 
+# Run Alembic migrations
+RUN alembic upgrade head
 # 
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -51,24 +51,26 @@ uvicorn app.main:app --reload
 
 you can use .envtest as an example, it has the following keys:
 
-COGNITO_CLIENT_ID -> your client id inside cognito
-COGNITO_CLIENT_SECRET -> your client secret from cognito client app
-AWS_ACCESS_KEY -> AWS key from the account that owns the users pool
-AWS_CLIENT_SECRET -> AWS secret from the account that owns the users pool
-AWS_REGION -> conigo and SES pool region
-COGNITO_POOL_ID -> cognito pool id
-EMAIL_SENDER_ADDRESS -> authorized sender in cognito
-FRONTEND_URL -> frontend url, use http://127.0.0.1:3000/ for dev
-SENTRY_DNS -> the DNS for sentry
-COLOMBIA_SECOP_APP_TOKEN -> Token from fetching SECOP data
-SECOP_PAGINATION_LIMIT -> Page size for the fected SECOP data
-HASH_KEY -> hash key for hashing personal data before submitting it to Database
-FRONT_PUBLIC_IMAGES_ES = -> Directory on the frontend server containing the Spanish version of the text in the images.
-FRONT_PUBLIC_IMAGES_EN = ->Directory on the frontend server containing the English version of the text in the images.
-TEMPORAL_BUCKET = -> Temporal S3 bucket used in develop to serve images
-FACEBOOK_LINK = -> Link to OCP Facebook account
-TWITTER_LINK = ->Link to OCP Twitter account
-LINK_LINK = -> Link to (pending to ask)
+- COGNITO_CLIENT_ID -> your client id inside cognito
+- COGNITO_CLIENT_SECRET -> your client secret from cognito client app
+- AWS_ACCESS_KEY -> AWS key from the account that owns the users pool
+- AWS_CLIENT_SECRET -> AWS secret from the account that owns the users pool
+- AWS_REGION -> cognito and SES pool region
+- COGNITO_POOL_ID -> cognito pool id
+- EMAIL_SENDER_ADDRESS -> authorized sender in cognito
+- FRONTEND_URL -> frontend url, use http://localhost:3000/ for dev
+- SENTRY_DNS -> the DNS for sentry
+- COLOMBIA_SECOP_APP_TOKEN -> token to set header to fetch SECOP data
+- SECOP_PAGINATION_LIMIT -> page size to fetch SECOP data
+- SECOP_DEFAULT_DAYS_FROM_ULTIMA_ACTUALIZACION -> days used to compare field ultima_actualizacion the first time a fetching to SECOP data is made (or no awards in database)
+- HASH_KEY -> key for hashing identifiers for privacy concerns
+- APPLICATION_EXPIRATION_DAYS -> days to expire link after application creation
+- IMAGES_BASE_URL -> url where the images are served
+- IMAGES_LANG_SUBPATH -> static sub-path to IMAGES_BASE_URL containing the localized versions of the text in the images buttons.
+- FACEBOOK_LINK -> link to OCP Facebook account
+- TWITTER_LINK -> link to OCP Twitter account
+- LINK_LINK -> link to (Pending to define)
+- TEST_MAIL_RECEIVER -> email used to send invitations when fetching new awards (Will be removed)
 
 You should configure the pre-commit for the repo one time
 
@@ -130,9 +132,11 @@ You need to have a postgresql service running. You can either install postgres f
 
 Once you have the service you need to create a database
 
-then you can construct the env variable like this
+then you can set the env variable like this
 
+```
 DATABASE_URL=postgresql://{username}:{password}@{host_adress:port}/{db_name}
+```
 
 in order to apply migrations in tables use
 
@@ -163,13 +167,20 @@ pytest tests -W error
 
 ## Run background jobs
 
-To run test locally
+To run the list of commands available use
 
 ```
-python3 -m background_processes.fetch_awards
+python -m app.commands --help
 ```
-to fech previous awards you can run the command like this:
 
+The command to fetch new awards is
+
+```
+python -m app.commands fetch-awards
+```
+
+It will send invitations to the email configure in the env variable _TEST_MAIL_RECEIVER_. Alternative could receive a custom email destination with **--email-invitation** argument
+
+```
+python -m app.commands fetch-awards --email-invitation test@example.com
 ```
-python3 -m background_processes.fetch_awards True
-```

app/background_processes/application_utils.py

@@ -0,0 +1,75 @@
+from datetime import datetime, timedelta
+
+from sqlalchemy.orm.session import Session
+
+from app.db.session import app_settings
+from app.schema.core import Application, Message, MessageType
+
+from . import background_utils
+
+
+def insert_application(application: Application, session: Session):
+    obj_db = Application(**application)
+    obj_db.created_at = datetime.utcnow()
+
+    session.add(obj_db)
+    session.flush()
+
+    return obj_db
+
+
+def insert_message(application: Application, session: Session):
+    obj_db = Message(application=application, type=MessageType.BORROWER_INVITACION)
+    obj_db.created_at = datetime.utcnow()
+
+    session.add(obj_db)
+    session.flush()
+
+    return obj_db
+
+
+def get_existing_application(award_borrower_identifier: str, session: Session):
+    application = (
+        session.query(Application)
+        .filter(Application.award_borrower_identifier == award_borrower_identifier)
+        .first()
+    )
+
+    return application
+
+
+def create_application(
+    award_id, borrower_id, email, legal_identifier, source_contract_id, session: Session
+) -> Application:
+    award_borrower_identifier: str = background_utils.get_secret_hash(
+        legal_identifier + source_contract_id
+    )
+
+    # if application already exists
+    application = get_existing_application(award_borrower_identifier, session)
+    if application:
+        error_data = {
+            "legal_identifier": legal_identifier,
+            "source_contract_id": source_contract_id,
+            "application_id": application.id,
+        }
+
+        background_utils.raise_sentry_error(
+            f"Skipping Award - Application ID {application.id} already exists on for award {source_contract_id}",
+            error_data,
+        )
+
+    new_uuid: str = background_utils.generate_uuid(award_borrower_identifier)
+    application = {
+        "award_id": award_id,
+        "borrower_id": borrower_id,
+        "primary_email": email,
+        "award_borrower_identifier": award_borrower_identifier,
+        "uuid": new_uuid,
+        "expired_at": datetime.utcnow()
+        + timedelta(days=app_settings.application_expiration_days),
+    }
+
+    application = insert_application(application, session)
+
+    return application

app/background_processes/awards_utils.py

@@ -0,0 +1,82 @@
+from contextlib import contextmanager
+from datetime import datetime
+
+from sqlalchemy import desc
+from sqlalchemy.orm.session import Session
+
+from app.db.session import get_db
+from app.schema.core import Award
+
+from . import background_utils
+from . import colombia_data_access as data_access
+
+
+def get_existing_award(source_contract_id: str, session: Session):
+    award = (
+        session.query(Award)
+        .filter(Award.source_contract_id == source_contract_id)
+        .first()
+    )
+
+    return award
+
+
+def get_last_updated_award_date():
+    with contextmanager(get_db)() as session:
+        award = (
+            session.query(Award).order_by(desc(Award.source_last_updated_at)).first()
+        )
+
+        if not award:
+            return None
+
+        return award.source_last_updated_at
+
+
+def insert_award(award: Award, session: Session):
+    obj_db = Award(**award)
+    obj_db.created_at = datetime.utcnow()
+    obj_db.missing_data = background_utils.get_missing_data_keys(award)
+
+    session.add(obj_db)
+    session.flush()
+    return obj_db
+
+
+def create_new_award(
+    source_contract_id: str,
+    entry: dict,
+    borrower_id: int = None,
+    previous: bool = False,
+) -> dict:
+    return data_access.create_new_award(
+        source_contract_id, entry, borrower_id, previous
+    )
+
+
+def get_new_contracts(index: int, last_updated_award_date):
+    return data_access.get_new_contracts(index, last_updated_award_date)
+
+
+def get_previous_contracts(documento_proveedor):
+    return data_access.get_previous_contracts(documento_proveedor)
+
+
+def get_source_contract_id(entry):
+    return data_access.get_source_contract_id(entry)
+
+
+def create_award(entry, session: Session, borrower_id=None, previous=False) -> Award:
+    source_contract_id = get_source_contract_id(entry)
+
+    # if award already exists
+    if get_existing_award(source_contract_id, session):
+        background_utils.raise_sentry_error(
+            f"Skipping Award [previous {previous}] - Already exists on database", entry
+        )
+
+    new_award = create_new_award(source_contract_id, entry, borrower_id, previous)
+
+    award = insert_award(new_award, session)
+
+    return award