diff --git a/.env b/.env index 8eed936..6a9a3cc 100644 --- a/.env +++ b/.env @@ -15,3 +15,14 @@ SENTRY_DNS= # Log level to use, DEBUG by default in dev LOG_LEVEL=DEBUG + +POSTGRES_HOST=postgres +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +# expose postgres on localhost for dev +# POSTGRES_EXPOSE=127.0.0.1:5432 + +# The top-level domain used for Open Food Facts, +# it's either `net` (staging) or `org` (production) +OFF_TLD=net \ No newline at end of file diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index f1405a0..1ef1e6b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,4 +11,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 + with: + python-version: 3.11 - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 736e6a8..fdaa88c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,39 +1,17 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + # Note for all linters: do not forget to update pyproject.toml when updating version. + - repo: https://github.com/ambv/black + rev: 23.11.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: debug-statements - - id: double-quote-string-fixer - - id: name-tests-test - - id: requirements-txt-fixer -- repo: https://github.com/asottile/setup-cfg-fmt - rev: v2.0.0 + - id: black + language_version: python3.11 + + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 hooks: - - id: setup-cfg-fmt -- repo: https://github.com/asottile/reorder_python_imports - rev: v3.8.2 + - id: flake8 + + - repo: https://github.com/timothycrosley/isort + rev: 5.12.0 hooks: - - id: reorder-python-imports - args: [--py37-plus, --add-import, 'from __future__ import annotations'] -- repo: https://github.com/asottile/add-trailing-comma - rev: v2.2.3 - hooks: - - id: add-trailing-comma - args: [--py36-plus] -- repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.6.0 - hooks: - - id: autopep8 -- repo: https://github.com/PyCQA/flake8 - rev: 5.0.2 - hooks: - - id: flake8 - args: [--ignore=E501] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 - hooks: - - id: mypy - additional_dependencies: [types-all] + - id: isort \ No newline at end of file diff --git a/app/api.py b/app/api.py index e149bff..dc293cb 100644 --- a/app/api.py +++ b/app/api.py @@ -1,16 +1,25 @@ +import hashlib +from datetime import datetime +from enum import StrEnum, auto from pathlib import Path +from typing import Any -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse, PlainTextResponse from fastapi.templating import Jinja2Templates -from openfoodfacts.utils import get_logger +from openfoodfacts import Flavor +from openfoodfacts.images import generate_image_url +from openfoodfacts.utils import URLBuilder, get_logger +from peewee import DoesNotExist +from playhouse.shortcuts import model_to_dict +from pydantic import BaseModel, Field, model_validator from app.config import settings +from app.models import FlagModel, TicketModel, db from app.utils import init_sentry logger = get_logger(level=settings.log_level.to_int()) - app = FastAPI( title="nutripatrol", contact={ @@ -38,3 +47,284 @@ def main_page(request: Request): @app.get("/robots.txt", response_class=PlainTextResponse) def robots_txt(): return """User-agent: *\nDisallow: /""" + + +def _get_device_id(request: Request): + """Get the device ID from the request, or generate one if not provided.""" + device_id = request.query_params.get("device_id") + if device_id is None: + device_id = hashlib.sha1(str(request.client.host).encode()).hexdigest() + return device_id + + +class TicketStatus(StrEnum): + open = auto() + closed = auto() + + +class IssueType(StrEnum): + """Type of the flag/ticket.""" + + # Issue about any of the product fields (image excluded), or about the + # product as a whole + product = auto() + # Issue about a product image + image = auto() + # Issue about search results + search = auto() + + +class TicketCreate(BaseModel): + barcode: str | None = Field( + None, + description="Barcode of the product, if the ticket is about a product or a product image. " + "In case of a search issue, this field is null.", + ) + type: IssueType = Field(..., description="Type of the issue") + url: str = Field(..., description="URL of the product or of the flagged image") + status: TicketStatus = Field(TicketStatus.open, description="Status of the ticket") + image_id: str | None = Field( + None, + description="ID of the flagged image, if the ticket type is `image`", + examples=["1", "front_fr"], + ) + flavor: Flavor = Field( + ..., description="Flavor (project) associated with the ticket" + ) + created_at: datetime = Field( + default_factory=datetime.utcnow, description="Creation datetime of the ticket" + ) + + +class Ticket(TicketCreate): + id: int = Field(..., description="ID of the ticket") + + +class SourceType(StrEnum): + mobile = auto() + web = auto() + robotoff = auto() + + +class FlagCreate(BaseModel): + barcode: str | None = Field( + None, + description="Barcode of the product, if the flag is about a product or a product image. " + "In case of a search issue, this field is null.", + ) + type: IssueType = Field(..., description="Type of the issue") + url: str = Field(..., description="URL of the product or of the flagged image") + user_id: str = Field(..., description="Open Food Facts User ID of the flagger") + source: SourceType = Field( + ..., + description="Source of the flag. It can be a user from the mobile app, " + "the web or a flag generated automatically by robotoff.", + ) + confidence: float | None = Field( + None, + ge=0, + le=1, + description="Confidence score of the model that generated the flag, " + "this field should only be provided by Robotoff.", + ) + image_id: str | None = Field( + None, + min_length=1, + description="ID of the flagged image", + examples=["1", "front_fr"], + ) + flavor: Flavor = Field( + ..., description="Flavor (project) associated with the ticket" + ) + reason: str | None = Field( + None, + min_length=1, + description="Reason for flagging provided by the user. The field is optional.", + ) + comment: str | None = Field( + None, + description="Comment provided by the user during flagging. This is a free text field.", + ) + created_at: datetime = Field( + default_factory=datetime.utcnow, description="Creation datetime of the flag" + ) + + @model_validator(mode="after") + def image_id_is_provided_when_type_is_image(self) -> "FlagCreate": + """Validate that `image_id` is provided when flag type is `image`.""" + if self.type is IssueType.image and self.image_id is None: + raise ValueError("`image_id` must be provided when flag type is `image`") + return self + + @model_validator(mode="after") + def barcode_should_not_be_provided_for_search_type(self) -> "FlagCreate": + """Validate that `barcode` is not provided when flag type is + `search`.""" + if self.type is IssueType.search and self.barcode is not None: + raise ValueError( + "`barcode` must not be provided when flag type is `search`" + ) + return self + + @model_validator(mode="before") + @classmethod + def generate_url(cls, data: Any) -> Any: + """Generate a URL for the flag based on the flag type and flavor.""" + if not isinstance(data, dict): + # Let Pydantic handle the validation + return data + flag_type = data.get("type") + flavor = data.get("flavor") + barcode = data.get("barcode") + image_id = data.get("image_id") + + if not flag_type or flavor not in [f.value for f in Flavor]: + # Let Pydantic handle the validation + return data + + flavor_enum = Flavor[flavor] + environment = settings.off_tld + # Set-up a default URL in case validation fails + + if flag_type == "product": + base_url = URLBuilder.world(flavor_enum, environment) + data["url"] = f"{base_url}/product/{barcode}" + elif flag_type == "image": + if image_id: + data["url"] = generate_image_url( + barcode, image_id, flavor_enum, environment + ) + else: + # Set-up a dummy URL in case image_id is not provided + # Pydantic otherwise raises an error + data["url"] = "http://localhost" + + return data + + +class Flag(FlagCreate): + id: int = Field(..., description="ID of the flag") + ticket_id: int = Field(..., description="ID of the ticket associated with the flag") + device_id: str = Field(..., description="Device ID of the flagger") + + +# Create a flag (one to one relationship) +@app.post("/flags") +def create_flag(flag: FlagCreate, request: Request): + with db: + # Check if the flag already exists + if ( + FlagModel.get_or_none( + FlagModel.barcode == flag.barcode, + FlagModel.url == flag.url, + FlagModel.type == flag.type, + FlagModel.flavor == flag.flavor, + FlagModel.user_id == flag.user_id, + ) + is not None + ): + raise HTTPException( + status_code=409, + detail="Flag already exists", + ) + + # Search for existing ticket + # With the same barcode, url, type and flavor + ticket = TicketModel.get_or_none( + TicketModel.barcode == flag.barcode, + TicketModel.url == flag.url, + TicketModel.type == flag.type, + TicketModel.flavor == flag.flavor, + ) + # If no ticket found, create a new one + if ticket is None: + ticket = _create_ticket( + TicketCreate( + barcode=flag.barcode, + url=flag.url, + type=flag.type, + flavor=flag.flavor, + image_id=flag.image_id, + ) + ) + elif ticket.status == TicketStatus.closed: + # Reopen the ticket if it was closed + ticket.status = TicketStatus.open + ticket.save() + + device_id = _get_device_id(request) + return model_to_dict( + FlagModel.create(ticket=ticket, device_id=device_id, **flag.model_dump()) + ) + + +# Get all flags (one to many relationship) +@app.get("/flags") +def get_flags(): + with db: + return {"flags": list(FlagModel.select().dicts().iterator())} + + +# Get flag by ID (one to one relationship) +@app.get("/flags/{flag_id}") +def get_flag(flag_id: int): + with db: + try: + return FlagModel.get_by_id(flag_id) + except DoesNotExist: + raise HTTPException(status_code=404, detail="Not found") + + +def _create_ticket(ticket: TicketCreate): + return TicketModel.create(**ticket.model_dump()) + + +# Create a ticket (one to one relationship) +@app.post("/tickets") +def create_ticket(ticket: TicketCreate) -> Ticket: + with db: + return _create_ticket(ticket) + + +# Get all tickets (one to many relationship) +@app.get("/tickets") +def get_tickets(): + with db: + return {"tickets": list(TicketModel.select().dicts().iterator())} + + +# Get ticket by id (one to one relationship) +@app.get("/tickets/{ticket_id}") +def get_ticket(ticket_id: int): + with db: + try: + return model_to_dict(TicketModel.get_by_id(ticket_id)) + except DoesNotExist: + raise HTTPException(status_code=404, detail="Not found") + + +# Get all flags for a ticket by id (one to many relationship) +@app.get("/tickets/{ticket_id}/flags") +def get_flags_by_ticket(ticket_id: int): + with db: + return { + "flags": list( + FlagModel.select() + .where(FlagModel.ticket_id == ticket_id) + .dicts() + .iterator() + ) + } + + +# Update ticket status by id with enum : open, closed (soft delete) +@app.put("/tickets/{ticket_id}/status") +def update_ticket_status(ticket_id: int, status: TicketStatus): + with db: + try: + ticket = TicketModel.get_by_id(ticket_id) + ticket.status = status + ticket.save() + return model_to_dict(ticket) + except DoesNotExist: + raise HTTPException(status_code=404, detail="Not found") diff --git a/app/config.py b/app/config.py index 7051e60..40a22ce 100644 --- a/app/config.py +++ b/app/config.py @@ -1,5 +1,6 @@ from enum import StrEnum +from openfoodfacts import Environment from pydantic_settings import BaseSettings @@ -29,6 +30,12 @@ def to_int(self): class Settings(BaseSettings): sentry_dns: str | None = None log_level: LoggingLevel = LoggingLevel.INFO + postgres_host: str = "localhost" + postgres_db: str = "postgres" + postgres_user: str = "postgres" + postgres_password: str = "postgres" + postgres_port: int = 5432 + off_tld: Environment = Environment.net settings = Settings() diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..e4426db --- /dev/null +++ b/app/models.py @@ -0,0 +1,63 @@ +from peewee import ( + CharField, + DateTimeField, + FloatField, + ForeignKeyField, + Model, + PostgresqlDatabase, +) + +from .config import settings + +db = PostgresqlDatabase( + settings.postgres_db, + user=settings.postgres_user, + password=settings.postgres_password, + host=settings.postgres_host, + port=settings.postgres_port, +) + + +class TicketModel(Model): + barcode = CharField(null=True) + type = CharField() + url = CharField() + status = CharField() + image_id = CharField(null=True) + flavor = CharField() + created_at = DateTimeField() + + class Meta: + database = db + table_name = "tickets" + + +class ModeratorActionModel(Model): + action_type = CharField() + user_id = CharField() + ticket = ForeignKeyField(TicketModel, backref="moderator_actions") + created_at = DateTimeField() + + class Meta: + database = db + table_name = "moderator_actions" + + +class FlagModel(Model): + ticket = ForeignKeyField(TicketModel, backref="flags") + barcode = CharField(null=True) + type = CharField() + url = CharField() + user_id = CharField() + device_id = CharField() + source = CharField() + confidence = FloatField(null=True) + image_id = CharField(null=True) + flavor = CharField() + reason = CharField(null=True) + comment = CharField(max_length=500, null=True) + created_at = DateTimeField() + + class Meta: + database = db + table_name = "flags" diff --git a/docker-compose.yml b/docker-compose.yml index 8393e86..8846ce9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,11 @@ x-api-common: &api-common environment: - SENTRY_DNS - LOG_LEVEL + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - POSTGRES_HOST + - OFF_TLD networks: - default @@ -15,5 +20,20 @@ services: ports: - "${API_PORT}:8000" + postgres: + restart: $RESTART_POLICY + image: postgres:16.0-alpine + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + volumes: + - postgres-data:/var/lib/postgresql/data + command: postgres -c shared_buffers=1024MB -c work_mem=64MB + mem_limit: 4g + shm_size: 1g + ports: + - "${POSTGRES_EXPOSE:-127.0.0.1:5432}:5432" + volumes: - rediscache: + postgres-data: \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..56a5ff5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.isort] # From https://black.readthedocs.io/en/stable/compatible_configs.html#isort +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 49ba13e..1aff57f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,6 @@ openfoodfacts==0.1.10 requests==2.31.0 pydantic-settings==2.0.3 sentry-sdk[fastapi]==1.31.0 -jinja2==3.1.2 \ No newline at end of file +jinja2==3.1.2 +peewee==3.17.0 +psycopg2-binary==2.9.9 \ No newline at end of file