Merge branch 'develop' of https://github.com/vectornguyen76/search-en…

…gine-shopee into develop

.gitignore

@@ -5,14 +5,17 @@ test*
 # *.ipynb
 NOTE.md
 data-elastic-search
-qdrant-vector-database
+qdrant-db
 architectures
 *.csv
 data.csv
 
 image-search-engine/assets/uploaded_images/*
 !image-search-engine/assets/uploaded_images/.gitkeep
 
+# Logs
+image-search-engine/logs/*
+
 # Model
 *.pth
 *.pt

docker-compose-prod.yaml

@@ -1,31 +1,31 @@
 version: "3"
 services:
-  triton-server:
-    container_name: triton-server
-    image: nvcr.io/nvidia/tritonserver:23.09-py3
-    ports:
-      - 9000:8000
-      - 9001:8001
-      - 9002:8002
-    command: tritonserver --model-repository=/models
-    volumes:
-      - ./image-search-engine/model_repository:/models
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
+  # triton-server:
+  #   container_name: triton-server
+  #   image: nvcr.io/nvidia/tritonserver:23.09-py3
+  #   ports:
+  #     - 9000:8000
+  #     - 9001:8001
+  #     - 9002:8002
+  #   command: tritonserver --model-repository=/models
+  #   volumes:
+  #     - ./image-search-engine/model_repository:/models
+  #   deploy:
+  #     resources:
+  #       reservations:
+  #         devices:
+  #           - driver: nvidia
+  #             count: 1
+  #             capabilities: [gpu]
 
-  qdrant-vector-database:
-    container_name: qdrant-vector-database
+  qdrant-db:
+    container_name: qdrant-db
     image: qdrant/qdrant:v1.5.1
     ports:
       - 6333:6333
       - 6334:6334
     volumes:
-      - ./qdrant-vector-database:/qdrant/storage
+      - ./qdrant-db:/qdrant/storage
 
   image-search-engine:
     container_name: image-search-container
@@ -34,17 +34,37 @@ services:
       context: ./image-search-engine
       dockerfile: Dockerfile
     environment:
-      - QDRANT_URL=http://qdrant-vector-database:6334
+      - QDRANT_URL=http://qdrant-db:6334
       - TRITON_SERVER_URL=triton-server:8001
     ports:
       - 7000:7000
     volumes:
       - ./image-search-engine/logs:/app/logs
     depends_on:
-      - qdrant-vector-database
-    profiles:
-      - dev.frontend
-      - prod
+      - qdrant-db
+
+  # image-search-engine-gpu:
+  #   container_name: image-search-container-gpu
+  #   image: vectornguyen76/image-search-engine-gpu
+  #   build:
+  #     context: ./image-search-engine
+  #     dockerfile: Dockerfile.gpu
+  #   environment:
+  #     - QDRANT_URL=http://qdrant-db:6334
+  #     - TRITON_SERVER_URL=triton-server:8001
+  #   ports:
+  #     - 7000:7000
+  #   volumes:
+  #     - ./image-search-engine/logs:/app/logs
+  #   depends_on:
+  #     - qdrant-db
+  #   deploy:
+  #     resources:
+  #       reservations:
+  #         devices:
+  #           - driver: nvidia
+  #             count: 1
+  #             capabilities: [gpu]
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0
@@ -94,31 +114,15 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    profiles:
-      - dev.frontend
-      - prod
-
-  frontend_dev_service:
-    container_name: frontend_dev_container
-    image: vectornguyen76/frontend_dev_image
-    build:
-      context: ./frontend
-      dockerfile: Dockerfile.dev
-    ports:
-      - 3000:3000
-    profiles:
-      - dev.backend
 
   frontend_service:
-    container_name: frontend_prod_container
-    image: vectornguyen76/frontend_prod_image
+    container_name: frontend_container
+    image: vectornguyen76/frontend_image
     build:
       context: ./frontend
       dockerfile: Dockerfile
     ports:
       - 3000:3000
-    profiles:
-      - prod
 
   nginx_service:
     container_name: nginx_container
@@ -132,11 +136,9 @@ services:
       - frontend_service
       - image-search-engine
       - text-search-engine
-    profiles:
-      - prod
 
 volumes:
   data-elastic-search:
     driver: local
-  qdrant-vector-database:
+  qdrant-db:
     driver: local

docker-compose.yaml

@@ -1,71 +1,63 @@
 version: "3"
 services:
-  triton-server:
-    container_name: triton-server
-    image: nvcr.io/nvidia/tritonserver:23.09-py3
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0
+    container_name: elasticsearch
+    environment:
+      - node.name=elasticsearch
+      - cluster.name=es-docker-cluster
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
     ports:
-      - 9000:8000
-      - 9001:8001
-      - 9002:8002
-    command: tritonserver --model-repository=/models
+      - 9200:9200
+    healthcheck:
+      test: curl --fail http://localhost:9200/_cat/health || exit 1
+      interval: 10s
+      timeout: 1s
+      retries: 10
     volumes:
-      - ./image-search-engine/model_repository:/models
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
+      - data-elastic-search:/usr/share/elasticsearch/data
 
-  qdrant-vector-database:
-    container_name: qdrant-vector-database
-    image: qdrant/qdrant:v1.5.1
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.10.0
+    container_name: kibana
     ports:
-      - 6333:6333
-      - 6334:6334
-    volumes:
-      - ./qdrant-vector-database:/qdrant/storage
+      - 5601:5601
+    environment:
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
 
-  image-search-engine:
-    container_name: image-search-container
-    image: vectornguyen76/image-search-engine
+  text-search-engine:
+    container_name: text-search-container
+    image: vectornguyen76/text-search-engine
     build:
-      context: ./image-search-engine
+      context: ./text-search-engine
       dockerfile: Dockerfile
-    environment:
-      - QDRANT_URL=http://qdrant-vector-database:6334
-      - TRITON_SERVER_URL=triton-server:8001
     ports:
-      - 7000:7000
-    volumes:
-      - ./image-search-engine/logs:/app/logs
+      - 8000:8000
+    environment:
+      ELASTICSEARCH_HOST: http://elasticsearch:9200
     depends_on:
-      - qdrant-vector-database
+      elasticsearch:
+        condition: service_healthy
 
-  # image-search-engine-gpu:
-  #   container_name: image-search-container-gpu
-  #   image: vectornguyen76/image-search-engine-gpu
-  #   build:
-  #     context: ./image-search-engine
-  #     dockerfile: Dockerfile.gpu
-  #   environment:
-  #     - QDRANT_URL=http://qdrant-vector-database:6334
-  #     - TRITON_SERVER_URL=triton-server:8001
-  #   ports:
-  #     - 7000:7000
-  #   volumes:
-  #     - ./image-search-engine/logs:/app/logs
-  #   depends_on:
-  #     - qdrant-vector-database
-  #   deploy:
-  #     resources:
-  #       reservations:
-  #         devices:
-  #           - driver: nvidia
-  #             count: 1
-  #             capabilities: [gpu]
+  frontend_service:
+    container_name: frontend_prod_container
+    image: vectornguyen76/frontend_prod_image
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+    ports:
+      - 3000:3000
 
 volumes:
-  qdrant-vector-database:
+  data-elastic-search:
     driver: local

image-search-engine/app.py

@@ -36,7 +36,7 @@ def healthcheck() -> bool:
 
 @app.post("/search-image-faiss", response_model=list[Product])
 async def search_image_faiss(file: UploadFile = File(...)):
-    # start_time = time.time()
+    start_time = time.time()
     try:
         image_path = await save_image_file(file=file)
 
@@ -46,7 +46,7 @@ async def search_image_faiss(file: UploadFile = File(...)):
         # Perform a search using the extracted feature vector
         search_results = faiss_search.search(query_vector=feature, top_k=20)
 
-        # LOGGER.info(f"Faiss search executed in {time.time() - start_time:.4f} seconds.")
+        LOGGER.info(f"Faiss search executed in {time.time() - start_time:.4f} seconds.")
         return search_results
 
     except Exception as e:
@@ -56,7 +56,7 @@ async def search_image_faiss(file: UploadFile = File(...)):
 
 @app.post("/search-image-qdrant", response_model=list[Product])
 async def search_image_qdrant(file: UploadFile = File(...)):
-    # start_time = time.time()
+    start_time = time.time()
     try:
         image_path = await save_image_file(file=file)
 
@@ -68,9 +68,9 @@ async def search_image_qdrant(file: UploadFile = File(...)):
 
         result = [Product.from_point(point) for point in search_results.result]
 
-        # LOGGER.info(
-        #     f"Qdrant search executed in {time.time() - start_time:.4f} seconds."
-        # )
+        LOGGER.info(
+            f"Qdrant search executed in {time.time() - start_time:.4f} seconds."
+        )
         return result
 
     except UnexpectedResponse as e:

image-search-engine/faiss_ingest.py

@@ -1,11 +1,22 @@
 from src.faiss_search.ingest_data import FaissIngest
 from src.utils import LOGGER
 
-# Create an instance of FaissIngest
-faiss_ingest = FaissIngest()
-
-if faiss_ingest.check_index_exists():
-    LOGGER.info("Faiss index already exists!")
-else:
-    # Create and save the Faiss index
-    faiss_ingest.create_index()
+
+def main():
+    """
+    Main function to perform QdrantIngest data ingestion.
+    """
+    # Create an instance of FaissIngest
+    faiss_ingest = FaissIngest()
+
+    if faiss_ingest.check_index_exists():
+        LOGGER.info("Index in Faiss already exists!")
+    else:
+        LOGGER.info("Create Index in Faiss!")
+
+        # Create and save the Faiss index
+        faiss_ingest.create_index()
+
+
+if __name__ == "__main__":
+    main()

image-search-engine/qdrant_ingest.py

@@ -1,18 +1,26 @@
 from src.qdrant_search.ingest_data import QdrantIngest
 from src.utils import LOGGER
 
-# Instantiate the QdrantIngest class
-qdrant_ingest = QdrantIngest()
 
-try:
-    response = qdrant_ingest.check_collection()
-    if response.result.status == 1:
-        LOGGER.info("Collection already exists!")
-except Exception as e:
-    LOGGER.info(f"Error checking collection: {e}")
+def main():
+    """
+    Main function to perform QdrantIngest data ingestion.
+    """
+    qdrant_ingest = QdrantIngest()
 
-    LOGGER.info("Create collection!")
-    response = qdrant_ingest.create_collection()
-    LOGGER.info(response)
+    try:
+        response = qdrant_ingest.check_collection()
+        if response.result.status == 1:
+            LOGGER.info("Collection already exists!")
+    except Exception as e:
+        LOGGER.info(f"Error checking collection: {e}")
 
-    qdrant_ingest.add_points()
+        LOGGER.info("Create collection!")
+        response = qdrant_ingest.create_collection()
+        LOGGER.info(response)
+
+        qdrant_ingest.add_points()
+
+
+if __name__ == "__main__":
+    main()