RAG using CPU instead of GPU

[holchan]

How to enable GPU to process the RAG files?

# Variables
export RAG_EMBEDDING_ENGINE="ollama"
export RAG_EMBEDDING_MODEL="yi:34b-chat-q4_K_M"
export RAG_EMBEDDING_MODEL_DEVICE_TYPE="cuda"
export ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION=True
export DEFAULT_MODELS="yi:34b-chat-q4_K_M"
export USE_CUDA=True
export CUDA_VISIBLE_DEVICES=0,1
export SENTENCE_TRANSFORMERS_USE_CUDA=1

# Iterate over each file in the directory
for file_path in "$directory"/*; do
    # Extract the filename from the file path
    filename=$(basename "$file_path")
    name="${filename%.*}"
    collection_name=$(generate_collection_name)
    
    # Extract the suffix from the filename
    suffix=$(echo "$filename" | rev | cut -d'_' -f1 | rev | cut -d'.' -f1)

    # Create the RAG collection
    response=$(curl -X POST "${OPENWEBUI_API_ENDPOINT}/rag/api/v1/doc" \
        -H "Accept: application/json" \
        -H "Authorization: Bearer ${secret_key}" \
        -F "file=@${file_path}" \
        -F "collection_name=${collection_name}" > /dev/null 2>&1)

    # Create new document
    response=$(curl -X POST "${OPENWEBUI_API_ENDPOINT}/api/v1/documents/create" \
        -H "Accept: application/json" \
        -H "Content-Type: application/json" \
        -H "Authorization: Bearer ${secret_key}" \
        -d '{
              "collection_name": "'"${collection_name}"'",
              "filename": "'"${filename}"'",
              "name": "'"${name}"'",
              "title": "'"${title}"'"
            }' > /dev/null 2>&1)

    # Tag the document
    response=$(curl -X POST "${OPENWEBUI_API_ENDPOINT}/api/v1/documents/doc/tags" \
        -H "Accept: application/json" \
        -H "Content-Type: application/json" \
        -H "Authorization: Bearer ${secret_key}" \
        -d '{
              "name": "'"${name}"'",
              "tags": [{"name": "UE5"}, {"name": "'"${suffix}"'"}]
            }' > /dev/null 2>&1)

    # Store the collection name
    collection_names+=("${collection_name}")
done

[justinh-rahb]

docker run -d \
  -p 3000:8080 \
  --gpus all \
  --add-host=host.docker.internal:host-gateway \
  -v open-webui:/app/backend/data \
  --name open-webui \
  --restart always \
  ghcr.io/open-webui/open-webui:cuda

[matnalopez2]

very useful. thanks!