Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

install torch cpu before install sentence_transformers to avoid some components #221

Merged
merged 8 commits into from
Jun 24, 2024
Merged

install torch cpu before install sentence_transformers to avoid some components #221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5eaf28e
install torch cpu before install `sentence_transformers` to avoid nv …
chensuyue Jun 20, 2024
b940458
add dataprep/redis/langchain test
chensuyue Jun 20, 2024
94777f4
add torch vision for retrievers langchain redis
chensuyue Jun 20, 2024
63a9d15
expand test wait time
chensuyue Jun 20, 2024
c3411e9
remove test
chensuyue Jun 21, 2024
d80036d
support torch cpu install
chensuyue Jun 24, 2024
f90a5d8
support torch cpu install
chensuyue Jun 24, 2024
f45efab
fix bug
chensuyue Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/reuse-get-test-matrix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
if [[ "${path_level_1[@]}" =~ "${service}" ]]; then
run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
else
vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md' | sort -u)
vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md|*.sh' | sort -u)
for vendor in ${vendors}; do
if [[ "${path_level_3[@]}" =~ "${service}/${vendor}" ]]; then
sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u)
Expand Down
3 changes: 3 additions & 0 deletions comps/dataprep/milvus/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/dataprep/milvus/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/dataprep/qdrant/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/dataprep/qdrant/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/dataprep/redis/langchain/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip setuptools && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/dataprep/redis/langchain_ray/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip setuptools && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/dataprep/redis/langchain_ray/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/dataprep/redis/llama_index/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip setuptools && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/dataprep/redis/llama_index/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/embeddings/langchain/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

FROM langchain/langchain:latest

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
libgl1-mesa-glx \
libjemalloc-dev \
Expand All @@ -18,6 +20,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/embeddings/langchain/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/reranks/langchain/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
libgl1-mesa-glx \
libjemalloc-dev \
Expand All @@ -20,6 +22,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/reranks/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/retrievers/langchain/milvus/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ FROM python:3.11-slim

ENV LANG C.UTF-8

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
build-essential \
libgl1-mesa-glx \
Expand All @@ -21,6 +23,7 @@ USER user
COPY comps /home/user/comps

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/milvus/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
3 changes: 3 additions & 0 deletions comps/retrievers/langchain/redis/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

FROM langchain/langchain:latest

ARG ARCH="cpu"

RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
libgl1-mesa-glx \
libjemalloc-dev \
Expand All @@ -20,6 +22,7 @@ RUN chmod +x /home/user/comps/retrievers/langchain/redis/run.sh
USER user

RUN pip install --no-cache-dir --upgrade pip && \
if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/redis/requirements.txt

ENV PYTHONPATH=$PYTHONPATH:/home/user
Expand Down
Loading