From a00d7a4cefa505684c73f10b65b729e9f72615fd Mon Sep 17 00:00:00 2001 From: Hien To Date: Mon, 12 Aug 2024 21:42:01 +0700 Subject: [PATCH 1/4] Initiate Dockerfile --- Dockerfile | 15 --------------- docker-compose.yml | 35 ----------------------------------- docker/Dockerfile | 35 +++++++++++++++++++++++++++++++++++ docker/README.md | 0 docker/entrypoint.sh | 10 ++++++++++ 5 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/entrypoint.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0a6fa9e2d..000000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:14 - -WORKDIR /app - -COPY package*.json ./ - -RUN npm install - -COPY . . - -RUN npm run build - -EXPOSE 3000 - -CMD ["npm", "start"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index eb5c75a00..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3' -services: - cortex-js: - build: ./cortex-js - ports: - - "3000:3000" - depends_on: - - db - environment: - - DATABASE_URL=postgres://user:password@db:5432/database - volumes: - - ./cortex-js:/app - - cortex-cpp: - build: ./cortex-cpp - ports: - - "3001:3001" - depends_on: - - cortex-js - environment: - - GPU_ENABLED=true - - GPU_FAMILY=Nvidia - - CPU_INSTRUCTIONS=AVX2 - - db: - image: postgres - environment: - - POSTGRES_USER=user - - POSTGRES_PASSWORD=password - - POSTGRES_DB=database - volumes: - - db-data:/var/lib/postgresql/data - -volumes: - db-data: \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..7c6c3d724 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,35 @@ +# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility +# Run nvidia-smi to check the CUDA version and the corresponding driver version +# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags + +FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base + +# 1. Install dependencies only when needed +FROM base AS devel + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +WORKDIR /app + +FROM devel AS release + +RUN npm install -g cortexso + +EXPOSE 1337 + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh + +RUN chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 000000000..03d687b2e --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Run cortex +cortex -a 0.0.0.0 + +cortex engines llamacpp init +cortex engines tensorrt-llm init + +# Keep the container running by tailing the log file +tail -f /root/cortex/cortex.log From 4fb1630a7f26c6bc759ce024730ec40422ddefa5 Mon Sep 17 00:00:00 2001 From: Hien To Date: Mon, 12 Aug 2024 23:29:26 +0700 Subject: [PATCH 2/4] Add Dockerfile.firewall --- docker/Dockerfile | 4 +-- docker/Dockerfile.firewall | 38 ++++++++++++++++++++++++++++ docker/common/blocked-domains.txt | 5 ++++ docker/common/dnsmasq.conf | 5 ++++ docker/common/entrypoint-firewall.sh | 15 +++++++++++ docker/{ => common}/entrypoint.sh | 1 + 6 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 docker/Dockerfile.firewall create mode 100644 docker/common/blocked-domains.txt create mode 100644 docker/common/dnsmasq.conf create mode 100644 docker/common/entrypoint-firewall.sh rename docker/{ => common}/entrypoint.sh (88%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7c6c3d724..7e948b1f0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,11 +24,9 @@ WORKDIR /app FROM devel AS release -RUN npm install -g cortexso - EXPOSE 1337 -COPY entrypoint.sh /usr/local/bin/entrypoint.sh +COPY ./common/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/docker/Dockerfile.firewall b/docker/Dockerfile.firewall new file mode 100644 index 000000000..9595802c5 --- /dev/null +++ b/docker/Dockerfile.firewall @@ -0,0 +1,38 @@ +# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility +# Run nvidia-smi to check the CUDA version and the corresponding driver version +# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags + +FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base + +# 1. Install dependencies only when needed +FROM base AS devel + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev dnsmasq iptables iproute2 && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +RUN mkdir -p /etc/dnsmasq.d/ + +WORKDIR /app + +FROM devel AS release + +EXPOSE 1337 + +COPY ./common/dnsmasq.conf /etc/dnsmasq.conf +COPY ./common/blocked-domains.txt /etc/dnsmasq.d/blocked-domains.txt + +COPY ./common/entrypoint-firewall.sh /usr/local/bin/entrypoint.sh + +RUN chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/common/blocked-domains.txt b/docker/common/blocked-domains.txt new file mode 100644 index 000000000..59dfd0ed6 --- /dev/null +++ b/docker/common/blocked-domains.txt @@ -0,0 +1,5 @@ +# Block IPv4 of domain openai.com and all subdomains *.openai.com +address=/openai.com/0.0.0.0 + +# Block IPv6 of domain openai.com and all subdomains *.openai.com +address=/openai.com/:: diff --git a/docker/common/dnsmasq.conf b/docker/common/dnsmasq.conf new file mode 100644 index 000000000..080edcfec --- /dev/null +++ b/docker/common/dnsmasq.conf @@ -0,0 +1,5 @@ +server=8.8.8.8 + +no-resolv + +conf-file=/etc/dnsmasq.d/blocked-domains.txt \ No newline at end of file diff --git a/docker/common/entrypoint-firewall.sh b/docker/common/entrypoint-firewall.sh new file mode 100644 index 000000000..0ecd59952 --- /dev/null +++ b/docker/common/entrypoint-firewall.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +echo "nameserver 127.0.0.1" > /etc/resolv.conf + +dnsmasq -k & + +npm install -g cortexso +# Run cortex +cortex -a 0.0.0.0 + +cortex engines llamacpp init +cortex engines tensorrt-llm init + +# Keep the container running by tailing the log file +tail -f /root/cortex/cortex.log diff --git a/docker/entrypoint.sh b/docker/common/entrypoint.sh similarity index 88% rename from docker/entrypoint.sh rename to docker/common/entrypoint.sh index 03d687b2e..7b8db86a2 100644 --- a/docker/entrypoint.sh +++ b/docker/common/entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/sh +npm install -g cortexso # Run cortex cortex -a 0.0.0.0 From 17108369d12629e15670c9a20b94ae82a565b27e Mon Sep 17 00:00:00 2001 From: Hien To Date: Tue, 13 Aug 2024 01:33:24 +0700 Subject: [PATCH 3/4] Add deny by route path --- docker/Dockerfile.firewall | 11 ++++++-- docker/common/entrypoint-firewall.sh | 14 ++++++++-- docker/common/generate_nginx_conf.sh | 21 ++++++++++++++ docker/common/nginx.conf | 41 ++++++++++++++++++++++++++++ docker/common/routes.txt | 1 + 5 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 docker/common/generate_nginx_conf.sh create mode 100644 docker/common/nginx.conf create mode 100644 docker/common/routes.txt diff --git a/docker/Dockerfile.firewall b/docker/Dockerfile.firewall index 9595802c5..d807cfb43 100644 --- a/docker/Dockerfile.firewall +++ b/docker/Dockerfile.firewall @@ -8,7 +8,7 @@ FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base FROM base AS devel # Install g++ 11 -RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev dnsmasq iptables iproute2 && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev dnsmasq nginx iproute2 && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* # Update alternatives for GCC and related tools RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ @@ -26,13 +26,20 @@ WORKDIR /app FROM devel AS release -EXPOSE 1337 +EXPOSE 80 COPY ./common/dnsmasq.conf /etc/dnsmasq.conf COPY ./common/blocked-domains.txt /etc/dnsmasq.d/blocked-domains.txt COPY ./common/entrypoint-firewall.sh /usr/local/bin/entrypoint.sh +COPY ./common/routes.txt /app/routes.txt +COPY ./common/generate_nginx_conf.sh /usr/local/bin/generate_nginx_conf.sh +COPY ./common/nginx.conf /etc/nginx/nginx.conf + RUN chmod +x /usr/local/bin/entrypoint.sh +HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \ + CMD curl -f http://127.0.0.1/api/system || exit 1 + ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/common/entrypoint-firewall.sh b/docker/common/entrypoint-firewall.sh index 0ecd59952..679404d57 100644 --- a/docker/common/entrypoint-firewall.sh +++ b/docker/common/entrypoint-firewall.sh @@ -1,15 +1,23 @@ #!/bin/sh +# Setup DNS resolution with dnsmasq echo "nameserver 127.0.0.1" > /etc/resolv.conf - dnsmasq -k & +# Generate Nginx configuration from routes.txt +/usr/local/bin/generate_nginx_conf.sh + +# Install cortex npm install -g cortexso -# Run cortex -cortex -a 0.0.0.0 + +# Start cortex +cortex -a 127.0.0.1 cortex engines llamacpp init cortex engines tensorrt-llm init +# Start nginx +nginx -g 'daemon off;' & + # Keep the container running by tailing the log file tail -f /root/cortex/cortex.log diff --git a/docker/common/generate_nginx_conf.sh b/docker/common/generate_nginx_conf.sh new file mode 100644 index 000000000..4ad570c46 --- /dev/null +++ b/docker/common/generate_nginx_conf.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +NGINX_CONF="/etc/nginx/conf.d/generated_routes.conf" + +rm -f $NGINX_CONF + +touch $NGINX_CONF + +while IFS= read -r line || [ -n "$line" ] +do + route=$(echo $line | awk '{print $1}') + action=$(echo $line | awk '{print $2}') + + echo "location $route {" >> $NGINX_CONF + if [ "$action" = "allow" ]; then + echo " allow all;" >> $NGINX_CONF + else + echo " deny all;" >> $NGINX_CONF + fi + echo "}" >> $NGINX_CONF +done < /app/routes.txt diff --git a/docker/common/nginx.conf b/docker/common/nginx.conf new file mode 100644 index 000000000..24972d53c --- /dev/null +++ b/docker/common/nginx.conf @@ -0,0 +1,41 @@ +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + server { + listen 80 default_server; + listen [::]:80 default_server; + + server_name _; + + # Include the generated routes configuration (location blocks only) + include /etc/nginx/conf.d/generated_routes.conf; + + # Default location block (catch-all) + location / { + proxy_pass http://127.0.0.1:1337; # Forward to your backend service + allow all; # Default to allow all requests + } + } +} \ No newline at end of file diff --git a/docker/common/routes.txt b/docker/common/routes.txt new file mode 100644 index 000000000..149743ec0 --- /dev/null +++ b/docker/common/routes.txt @@ -0,0 +1 @@ +/v1/models/start deny From 238189ffff25b393d8158a10f1a17b4ce6d39b6b Mon Sep 17 00:00:00 2001 From: Hien To Date: Tue, 13 Aug 2024 01:50:02 +0700 Subject: [PATCH 4/4] Add Readme Dockerfile --- docker/Dockerfile | 3 +++ docker/README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e948b1f0..015a6220b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,4 +30,7 @@ COPY ./common/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh +HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \ + CMD curl -f http://127.0.0.1:1337/api/system || exit 1 + ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md index e69de29bb..bf1ea4263 100644 --- a/docker/README.md +++ b/docker/README.md @@ -0,0 +1,46 @@ +# Docker with cortex + +We offer two methods for deploying the Cortex environment on Docker. + +## Method 1: Use the default Dockerfile with Cortex. + +To use this method, you need to follow these steps: +```bash +git clone https://github.com/janhq/cortex.git +cd cortex/docker +docker build -t cortex:latest . + +# Run the container with GPU support +docker run -it --gpus all -d -p 1337:1337 cortex:latest + +# Run the container with CPU support +docker run -it -d -p 1337:1337 cortex:latest + +# After starting, you can access Swagger at http://localhost:1337/api and the API server at http://localhost:1337. +# Additionally, you can exec into the container and use cortex-cli to perform other operations. +``` + +## Method 2: Use Dockerfile.firewall with the feature to block outbound connections by domain and block inbound connections by API path. + +The use case for this method is when you want to host the Cortex API 100% offline, preventing access to remote models like the OpenAI API. Alternatively, you might want to block inbound connections by restricting clients from calling the API to load models `/v1/models/start`. + +To use this method, you need to follow these steps: + +- Step 1: Edit the contents of the [blocked-domains.txt](./docker/common/blocked-domains.txt) file according to your requirements. Refer to the provided examples in the file. The goal is to block outbound connections to the domains you do not want to allow. +- Step 2: Edit the contents of the [blocked-paths.txt](./docker/common/blocked-paths.txt) file according to your requirements. Refer to the provided examples in the file. The goal is to block inbound connections to the paths you do not want to allow. +- Step 3: Build the image with Dockerfile.firewall following the instructions below: + + ```bash + git clone https://github.com/janhq/cortex.git + cd cortex/docker + docker build -f Dockerfile.firewall -t cortex-with-firewall:latest . + + # Run the container with GPU support + docker run -it --gpus all -d -p 1337:1337 cortex:latest + + # Run the container with CPU support + docker run -it -d -p 1337:1337 cortex:latest + + # After starting, you can access Swagger at http://localhost:1337/api and the API server at http://localhost:1337. + # Additionally, you can exec into the container and use cortex-cli to perform other operations. + ```