Adds async processing, session management and a starter cloud setup (#…

…921)

This PR adds the ability to use Redis as cache, Celery for task
execution, Session management, async tasks for the API and some other
improvements.

Run the following command after setting up the .env file in `./deploy`
folder:
```bash
docker compose up --exit-code-from tests tests result_backend broker celeryworker db --build
```

.env.example

@@ -46,10 +46,20 @@ LANGFLOW_OPEN_BROWSER=
 # Example: LANGFLOW_REMOVE_API_KEYS=false
 LANGFLOW_REMOVE_API_KEYS=
 
+# Whether to use RedisCache or InMemoryCache
+# Values: memory, redis
+# Example: LANGFLOW_CACHE_TYPE=memory
+# If you want to use redis then the following environment variables must be set:
+# LANGFLOW_REDIS_HOST (default: localhost)
+# LANGFLOW_REDIS_PORT (default: 6379)
+# LANGFLOW_REDIS_DB (default: 0)
+# LANGFLOW_REDIS_CACHE_EXPIRE (default: 3600)
+LANGFLOW_CACHE_TYPE=
+
 # Superuser username
 # Example: LANGFLOW_SUPERUSER=admin
 LANGFLOW_SUPERUSER=
 
 # Superuser password
 # Example: LANGFLOW_SUPERUSER_PASSWORD=123456
-LANGFLOW_SUPERUSER_PASSWORD=
+LANGFLOW_SUPERUSER_PASSWORD=

.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: "Async API tests"
+
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - dev
+      - main
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Set up Docker
+        run: docker --version && docker-compose --version
+
+      - name: "Create env file"
+        working-directory: ./deploy
+        run: |
+          echo "${{ secrets.ENV_FILE }}" > .env
+
+      - name: Build and start services
+
+        working-directory: ./deploy
+        run: docker compose up --exit-code-from tests tests result_backend broker celeryworker db --build
+        continue-on-error: true
+
+      - name: Stop services
+        run: docker compose down

.github/workflows/test.yml

@@ -7,7 +7,7 @@ on:
     branches: [dev]
 
 env:
-  POETRY_VERSION: "1.4.0"
+  POETRY_VERSION: "1.5.0"
 
 jobs:
   build:
@@ -16,6 +16,8 @@ jobs:
       matrix:
         python-version:
           - "3.10"
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.gitignore

@@ -254,3 +254,4 @@ langflow.db
 
 /tmp/*
 src/backend/langflow/frontend/
+.docker

Makefile

@@ -61,7 +61,7 @@ frontendc:
 	make run_frontend
 
 install_backend:
-	poetry install
+	poetry install --extras deploy
 
 backend:
 	make install_backend

base.Dockerfile

@@ -0,0 +1,97 @@
+
+
+# syntax=docker/dockerfile:1
+# Keep this syntax directive! It's used to enable Docker BuildKit
+
+# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
+# but I try to keep it updated (see history)
+
+################################
+# PYTHON-BASE
+# Sets up all our shared environment variables
+################################
+FROM python:3.10-slim as python-base
+
+# python
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.5.1 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+
+################################
+# BUILDER-BASE
+# Used to build deps + create our virtual environment
+################################
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # deps for installing poetry
+    curl \
+    # deps for building python deps
+    build-essential
+
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+# The --mount will mount the buildx cache directory to where
+# Poetry and Pip store their cache so that they can re-use it
+RUN --mount=type=cache,target=/root/.cache \
+    curl -sSL https://install.python-poetry.org | python3 -
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY poetry.lock pyproject.toml ./
+COPY ./src/backend/langflow/main.py ./src/backend/langflow/main.py
+# Copy README.md to the build context
+COPY README.md .
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --without dev --extras deploy
+
+
+################################
+# DEVELOPMENT
+# Image used during development / testing
+################################
+FROM python-base as development
+WORKDIR $PYSETUP_PATH
+
+# copy in our built poetry + venv
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# Copy just one file to avoid rebuilding the whole image
+COPY ./src/backend/langflow/__init__.py ./src/backend/langflow/__init__.py
+# quicker install as runtime deps are already installed
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --with=dev --extras deploy
+
+# copy in our app code
+COPY ./src/backend ./src/backend
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --with=dev --extras deploy
+COPY ./tests ./tests=
+

deploy/.env.example

@@ -0,0 +1,57 @@
+DOMAIN=localhost
+STACK_NAME=langflow-stack
+ENVIRONMENT=development
+
+TRAEFIK_PUBLIC_NETWORK=traefik-public
+TRAEFIK_TAG=langflow-traefik
+TRAEFIK_PUBLIC_TAG=traefik-public
+
+# RabbitMQ configuration
+RABBITMQ_DEFAULT_USER=langflow
+RABBITMQ_DEFAULT_PASS=langflow
+
+# Database configuration
+DB_USER=langflow
+DB_PASSWORD=langflow
+DB_HOST=db
+DB_PORT=5432
+DB_NAME=langflow
+
+# Logging configuration
+LOG_LEVEL=debug
+
+# DB configuration
+POSTGRES_USER=langflow
+POSTGRES_PASSWORD=langflow
+POSTGRES_DB=langflow
+POSTGRES_PORT=5432
+
+# Flower configuration
+LANGFLOW_CACHE_TYPE=redis
+LANGFLOW_REDIS_HOST=result_backend
+LANGFLOW_REDIS_PORT=6379
+LANGFLOW_REDIS_DB=0
+LANGFLOW_REDIS_EXPIRE=3600
+LANGFLOW_REDIS_PASSWORD=
+FLOWER_UNAUTHENTICATED_API=True
+BROKER_URL=amqp://langflow:langflow@broker:5672
+RESULT_BACKEND=redis://result_backend:6379/0
+C_FORCE_ROOT="true"
+
+# Frontend configuration
+VITE_PROXY_TARGET=http://backend:7860/api/
+BACKEND_URL=http://backend:7860
+
+# PGAdmin configuration
+PGADMIN_DEFAULT_EMAIL=admin@admin.com
+PGADMIN_DEFAULT_PASSWORD=admin
+
+# OpenAI configuration (for testing purposes)
+OPENAI_API_KEY=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
+
+# Superuser configuration
+LANGFLOW_SUPERUSER=superuser
+LANGFLOW_SUPERUSER_PASSWORD=superuser
+
+# New user configuration
+LANGFLOW_NEW_USER_IS_ACTIVE=False

deploy/.gitignore

@@ -0,0 +1 @@
+pgadmin

deploy/base.Dockerfile

@@ -0,0 +1,92 @@
+
+
+# syntax=docker/dockerfile:1
+# Keep this syntax directive! It's used to enable Docker BuildKit
+
+# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
+# but I try to keep it updated (see history)
+
+################################
+# PYTHON-BASE
+# Sets up all our shared environment variables
+################################
+FROM python:3.10-slim as python-base
+
+# python
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.5.1 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+
+################################
+# BUILDER-BASE
+# Used to build deps + create our virtual environment
+################################
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # deps for installing poetry
+    curl \
+    # deps for building python deps
+    build-essential
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+# The --mount will mount the buildx cache directory to where
+# Poetry and Pip store their cache so that they can re-use it
+RUN --mount=type=cache,target=/root/.cache \
+    curl -sSL https://install.python-poetry.org | python3 -
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY ./poetry.lock ./pyproject.toml ./
+# Copy README.md to the build context
+COPY ./README.md ./
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --without dev --extras deploy
+
+
+################################
+# DEVELOPMENT
+# Image used during development / testing
+################################
+FROM python-base as development
+WORKDIR $PYSETUP_PATH
+
+# copy in our built poetry + venv
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# Copy just one file to avoid rebuilding the whole image
+COPY ./src/backend/langflow/__init__.py ./src/backend/langflow/__init__.py
+# quicker install as runtime deps are already installed
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install --with=dev --extras deploy
+
+# copy in our app code
+COPY ./src/backend ./src/backend
+COPY ./tests ./tests