Merge pull request #22 from vectornguyen76/develop

Develop backend service

.github/workflows/development_pipeline.yml

@@ -33,7 +33,7 @@ jobs:
         run: pip install -r ./image-search-engine/requirements.txt
 
       - name: Run isort
-        run: isort --check-only ./image-search-engine/.
+        run: isort --check-only --profile=black ./image-search-engine/.
 
       - name: Run black
         run: black --check ./image-search-engine/.
@@ -119,6 +119,62 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+  build-push-backend:
+    runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('./backend/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-text-search
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install Dependencies
+        run: pip install -r ./backend/requirements.txt
+
+      - name: Run isort
+        run: isort --check-only --profile=black ./backend/.
+
+      - name: Run black
+        run: black --check ./backend/.
+
+      - name: Run flake8
+        run: flake8 --ignore=E501,W503,F401 ./backend
+
+      # - name: Run Pylint
+      #   run: pylint ./backend/*.py
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        id: docker_hub_auth
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: ./backend
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/backend:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
   build-push-frontend:
     runs-on: ubuntu-latest
     env:

.gitignore

@@ -4,8 +4,8 @@ index.faiss
 test*
 # *.ipynb
 NOTE.md
-data-elastic-search
-qdrant-db
+elastic_search_data
+qdrant_db_data
 architectures
 *.csv
 data.csv

README.md

@@ -1,40 +1,13 @@
-# Search Engine Shopee
+# Search Engine
 
-[![Development](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/development_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/development_pipeline.yml)
-[![Staging](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/staging_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/staging_pipeline.yml)
-[![Production](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/production_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine-shopee/actions/workflows/production_pipeline.yml)
+[![Development](https://github.com/vectornguyen76/search-engine/actions/workflows/development_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine/actions/workflows/development_pipeline.yml)
+[![Staging](https://github.com/vectornguyen76/search-engine/actions/workflows/staging_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine/actions/workflows/staging_pipeline.yml)
+[![Production](https://github.com/vectornguyen76/search-engine/actions/workflows/production_pipeline.yml/badge.svg)](https://github.com/vectornguyen76/search-engine/actions/workflows/production_pipeline.yml)
 
-### Set up development enviroments
+## Architecture
 
-1. **Frontend development enviroment**
-   - Run backend, database services
-     ```
-     docker compose --profile dev.frontend up -d
-     ```
-   - Run frontend
-     ```
-     cd frontend
-     ```
-     ```
-     npm run dev
-     ```
-2. **Backend development enviroment**
-   - Run frontend, database services
-     ```
-     docker compose --profile dev.backend up -d
-     ```
-3. **Production development enviroment**
-   - Run all services: backend, frontend, database,...
-     ```
-     docker compose --profile prod up -d
-     ```
-
-### Appy pre-commit
-
-   <p align="center">
-   <img src="./assets/pre-commit.jpg" alt="Workflow Pre-commit" />
-   <br>
-   <em>Workflow Pre-commit</em>
-   </p>
-
-- https://pre-commit.com/
+  <p align="center">
+  <img src="./assets/architectures.png" alt="Architecture" />
+  <br>
+  <em>System Architecture</em>
+  </p>

backend/.env.example

@@ -0,0 +1,13 @@
+JWT_ALG=HS256
+JWT_EXP=21000
+JWT_SECRET=SECRET
+DATABASE_URL=postgresql+asyncpg://db_user:db_password@localhost/db_dev
+
+SITE_DOMAIN=127.0.0.1
+SECURE_COOKIES=false
+
+CORS_HEADERS=["*"]
+CORS_ORIGINS=["http://localhost:3000"]
+
+TEXT_SEARCH_URL=http://localhost:8000
+IMAGE_SEARCH_URL=http://localhost:7000

backend/.env.local

@@ -0,0 +1,22 @@
+JWT_ALG=HS256
+JWT_EXP=21000
+JWT_SECRET=SECRET
+DATABASE_URL=postgresql+asyncpg://db_user:db_password@db_service/db_dev
+
+SITE_DOMAIN=127.0.0.1
+SECURE_COOKIES=false
+
+ENVIRONMENT=LOCAL
+
+CORS_HEADERS=["*"]
+CORS_ORIGINS=["http://localhost:3000"]
+
+TEXT_SEARCH_URL=http://text-search-engine:8000
+IMAGE_SEARCH_URL=http://image-search-engine:7000
+
+POSTGRES_USER=db_user
+POSTGRES_PASSWORD=db_password
+POSTGRES_DB=db_dev
+POSTGRES_HOST=db_service
+POSTGRES_PORT=5432
+PGPASSWORD=db_password

backend/.gitignore

@@ -94,7 +94,7 @@ celerybeat.pid
 *.sage.py
 
 # Environment variable
-# .env
+.env
 # .env*
 
 # Environments

backend/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.10.9-slim-buster
+
+RUN apt-get update && \
+    apt-get install -y gcc libpq-dev && \
+    apt clean && \
+    rm -rf /var/cache/apt/* && \
+    apt-get install -y postgresql-client
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=utf-8
+
+COPY requirements.txt /tmp
+
+RUN pip install -U pip && \
+    pip install --no-cache-dir -r /tmp/requirements.txt
+
+COPY . /app
+ENV PATH "$PATH:/app/scripts"
+
+RUN useradd -m -d /app -s /bin/bash app \
+    && chown -R app:app /app/* && chmod +x /app/scripts/* \
+    && chmod +x /app/entrypoint.sh
+
+WORKDIR /app
+USER app
+
+ENTRYPOINT ["/app/entrypoint.sh"]

backend/README.md

@@ -1 +1,163 @@
-uvicorn app:app --reload
+# Search Engine Backend
+
+## Index
+
+- [Overview](#overview)
+- [Environments](#environments)
+  - [Develop](#develop)
+  - [Local - Docker](#local---docker)
+- [Reference](#reference)
+
+## Overview
+
+- Dockerfile optimized for small size and fast builds with a non-root user
+- Easy local development environment with configured PostgreSQL
+- SQLAlchemy with slightly configured `alembic`
+- Async SQLAlchemy engine
+- Migrations set in an easy-to-sort format (`YYYY-MM-DD_slug`)
+- Pre-installed JWT authorization
+- Short-lived access token
+- Long-lived refresh token stored in http-only cookies
+
+## Environments
+
+### Develop
+
+1. **Create Environment and Install Packages**
+
+   First, create a virtual environment and install the required packages:
+
+   ```shell
+   conda create -n backend python=3.10
+   conda activate backend
+   pip install -r requirements.txt
+   ```
+
+2. **Set Up PostgreSQL on Ubuntu 20.04**
+
+   Install PostgreSQL:
+
+   ```shell
+   sudo apt-get install postgresql-14
+   ```
+
+   Access the PostgreSQL command line:
+
+   ```shell
+   sudo -u postgres psql
+   ```
+
+   Create a user and password for your database:
+
+   ```shell
+   CREATE USER db_user WITH PASSWORD 'db_password';
+   ```
+
+   Create a database named 'db_dev':
+
+   ```shell
+   CREATE DATABASE db_dev;
+   ```
+
+   Grant all privileges to the user for the 'db_dev' database:
+
+   ```shell
+   GRANT ALL PRIVILEGES ON DATABASE db_dev TO db_user;
+   ```
+
+   If you need to reset the database, you can drop it:
+
+   ```shell
+   DROP DATABASE IF EXISTS db_dev;
+   ```
+
+   Connect to the 'db_dev' database:
+
+   ```shell
+   sudo -u postgres psql -d db_dev
+   ```
+
+   List Tables:
+
+   To list all the tables in the current schema (usually "public"), you can use the following SQL command within the psql session:
+
+   ```
+   \dt
+   ```
+
+3. **Migrations**
+
+   1. Initialize Alembic:
+
+      ```shell
+      alembic init alembic
+      ```
+
+   2. Define and generate your first migration:
+
+      Modify the `alembic.ini` configuration file to specify your database connection URL. Then, create an initial migration:
+
+      ```shell
+      alembic revision -m "initial" --autogenerate
+      ```
+
+   3. Apply the initial migration to the database:
+
+      Apply the migration to create the initial database schema:
+
+      ```shell
+      alembic upgrade head
+      ```
+
+   4. Create .env from .env.example
+      - Generate Secret key:
+      ```
+      openssl rand -hex 32
+      ```
+
+4. **Run the Application**
+
+   ```shell
+   uvicorn --reload app:app --port 5000
+   ```
+
+### Local - Docker
+
+1. **Build Docker Image**
+
+   ```shell
+   docker compose build
+   ```
+
+2. **Run Containers**
+
+   ```shell
+   docker compose up -d
+   ```
+
+3. **Migrations**
+
+   - Create an automatic migration from changes in `src/database.py`
+
+     ```shell
+     docker compose exec backend_service makemigrations *migration_name*
+     ```
+
+   - Run migrations
+
+     ```shell
+     docker compose exec backend_service migrate
+     ```
+
+   - Downgrade migrations
+
+     ```shell
+     docker compose exec backend_service downgrade -1  # or -2 or base or hash of the migration
+     ```
+
+## Reference
+
+- [fastapi_production_template](https://github.com/zhanymkanov/fastapi_production_template)
+- [fastapi-best-practices](https://github.com/zhanymkanov/fastapi-best-practices)
+- [fastapi_sqlalchemy_async_orm](https://github.com/nf1s/fastapi_sqlalchemy_async_orm)
+- [Tutorial: FastAPI with SQLAlchemy Async ORM and Alembic](https://ahmed-nafies.medium.com/tutorial-fastapi-with-sqlalchemy-async-orm-and-alembic-2fa68102f82d)