build and push workflows latest custom images

.github/workflows/build-latest.yml

@@ -0,0 +1,42 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push custom MinIO image
+      uses: docker/build-push-action@v3
+      with:
+        context: ./minio
+        file: ./minio/Dockerfile
+        push: true
+        tags: cdaprod/cda-minio:latest
+        platforms: linux/amd64,linux/arm64
+
+    - name: Build and push custom Weaviate image
+      uses: docker/build-push-action@v3
+      with:
+        context: ./weaviate
+        file: ./weaviate/Dockerfile
+        push: true
+        tags: cdaprod/cda-weaviate:latest
+        platforms: linux/amd64,linux/arm64

.github/workflows/deploy-latest.yml

@@ -0,0 +1,38 @@
+name: Deploy Services
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Deploy MinIO Stack
+        run: |
+          export MINIO_ACCESS_KEY=${{ secrets.MINIO_ACCESS_KEY }}
+          export MINIO_SECRET_KEY=${{ secrets.MINIO_SECRET_KEY }}
+          docker stack deploy -c ./minio/docker-compose.minio.yml minio_stack
+
+      - name: Deploy Weaviate Stack
+        run: |
+          # Set any required environment variables for Weaviate here
+          docker stack deploy -c ./weaviate/docker-compose.weaviate.yml weaviate_stack
+
+      - name: Deploy NGINX Stack
+        run: |
+          # If you have environment variables for NGINX, set them here
+          docker stack deploy -c ./nginx-server/docker-compose.nginx.yml nginx_stack

minio/Dockerfile

@@ -0,0 +1,12 @@
+# Use an argument for the base image
+ARG BASE_IMAGE=minio/minio:latest
+FROM $BASE_IMAGE
+
+# Add custom configuration or scripts if needed
+# COPY your-custom-config /your-config-location
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENV MINIO_ROOT_USER=minioadmin \
+    MINIO_ROOT_PASSWORD=minioadmin
+EXPOSE 9000 9001
+ENTRYPOINT ["/entrypoint.sh"]

minio/docker-compose.minio.yaml

@@ -0,0 +1,24 @@
+version: '3.8'
+
+services:
+  minio:
+    image: cdaprod/cda-minio:latest
+    container_name: cda_minio
+    volumes:
+      - minio_data:/data
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY}
+      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY}
+    command: server /data --console-address ":9001"
+    networks:
+      - app_network
+
+volumes:
+  minio_data:
+
+networks:
+  app_network:
+    driver: overlay

minio/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+# Start MinIO in the background
+minio server /data --console-address ":9001" &
+
+# Wait for MinIO to start
+sleep 5
+
+# Set up alias and create bucket
+mc alias set myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}
+mc mb myminio/weaviate-backups
+mc mb myminio/cda-datasets
+
+# Keep the script running to prevent the container from exiting
+tail -f /dev/null

nginx/docker-compose.nginx.yaml

@@ -0,0 +1,17 @@
+version: '3.8'
+
+services:
+  cda-nginx:
+    image: nginx:latest
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./certs:/etc/nginx/certs:ro # If using HTTPS
+    networks:
+      - app_network
+
+networks:
+  app_network:
+    external: true

nginx/nginx.conf

@@ -0,0 +1,40 @@
+events {}
+
+http {
+    # MIME types and default MIME type
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    # Logging
+    access_log    /var/log/nginx/access.log;
+    error_log     /var/log/nginx/error.log;
+
+    # Configurations for MinIO, Weaviate, and Python application
+    server {
+        listen 80;
+
+        # MinIO
+        location /minio/ {
+            proxy_pass http://minio:9000/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # Weaviate
+        location /weaviate/ {
+            proxy_pass http://weaviate:8080/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # Python Application (web)
+        location /app/ {
+            proxy_pass http://web:8000/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}

weaviate/Dockerfile

@@ -0,0 +1,6 @@
+# Use an argument for the base image
+ARG BASE_IMAGE=semitechnologies/weaviate:latest
+FROM $BASE_IMAGE
+
+# Add custom configuration, schema, or scripts if needed
+# COPY your-custom-config /your-config-location

weaviate/docker-compose.weaviate.yaml

@@ -0,0 +1,34 @@
+version: '3.8'
+
+services:
+  weaviate:
+    container_name: cda_weaviate
+    image: cdaprod/cda-weaviate
+    ports:
+      - "8080:8080"
+      - "50051:50051"
+    environment:
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      QUERY_DEFAULTS_LIMIT: '25'
+      DEFAULT_VECTORIZER_MODULE: 'text2vec-huggingface'
+      ENABLE_MODULES: 'backup-s3, text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai'
+      BACKUP_S3_BUCKET: 'weaviate-backups'
+      BACKUP_S3_ENDPOINT: 'minio:9000'
+      BACKUP_S3_ACCESS_KEY_ID: 'minio'
+      BACKUP_S3_SECRET_ACCESS_KEY: 'minio123'
+      BACKUP_S3_USE_SSL: 'false'
+      MINIO_STORAGE_BUCKET_NAME: 'weaviate-backups'
+    volumes:
+      - weaviate_data:/var/lib/weaviate
+    depends_on:
+      - minio
+    networks:
+      - app_network
+
+volumes:
+  weaviate_data:
+
+networks:
+  app_network:
+    driver: overlay