From 0122936b2bacbb220e201789cc319036a96b8666 Mon Sep 17 00:00:00 2001 From: NGB Office GPGPU Workstation#01 Date: Fri, 8 Dec 2023 01:04:57 +0800 Subject: [PATCH] upgrade dependen --- Dockerfile | 7 +- apps/webui/pyproject.toml | 8 +- apps/yuren_trainer/pyproject.toml | 6 +- pyproject.toml | 18 +-- requirements-dev.lock | 212 +++++++++++++++--------------- requirements.lock | 204 ++++++++++++++-------------- 6 files changed, 233 insertions(+), 222 deletions(-) diff --git a/Dockerfile b/Dockerfile index b7f2850..9ca155e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ # This Dockerfile is work in progress -FROM nvidia/cuda:11.7.1-runtime-ubuntu20.04 as base +FROM nvidia/cuda:12.3.1-runtime-ubuntu20.04 as base ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt upgrade -y && \ apt install --fix-missing -y git curl dos2unix \ - libcudnn8 libcupt-common cuda-cupti-11-7 + libcudnn8 libcupt-common cuda-cupti-12-3 ENV RYE_HOME="/opt/rye" @@ -21,7 +21,8 @@ COPY . . SHELL [ "bash", "-c" ] RUN ${RYE_HOME}/self/bin/pip install -U pip==23.1 -RUN rye sync +RUN --mount=type=cache,target=~/.cache \ + rm requirements-dev.lock requirements.lock && rye sync --verbose FROM base diff --git a/apps/webui/pyproject.toml b/apps/webui/pyproject.toml index c596559..dfd71db 100644 --- a/apps/webui/pyproject.toml +++ b/apps/webui/pyproject.toml @@ -6,11 +6,11 @@ authors = [ { name = "NGB Office GPGPU Workstation#01", email = "gpgpu-1@corp.pleisto.com" } ] dependencies = [ - "gradio>=3.35.2", + "gradio==3.35.2", "mdtex2html>=1.2.0", - "markdown>=3.4.3", - "llama_index>=0.6.33", - "pygments>=2.15.1", + "markdown>=3.5.1", + "llama_index>=0.9.13", + "pygments>=2.17.2", ] readme = "README.md" requires-python = ">= 3.10" diff --git a/apps/yuren_trainer/pyproject.toml b/apps/yuren_trainer/pyproject.toml index df945f1..8eb2013 100644 --- a/apps/yuren_trainer/pyproject.toml +++ b/apps/yuren_trainer/pyproject.toml @@ -4,9 +4,9 @@ version = "0.1.0" description = "Add a short description here" dependencies = [ "peft @ git+https://github.com/huggingface/peft", - "datasets>=2.13.0", - "wandb>=0.15.4", - "deepspeed>=0.9.5", + "datasets>=2.15.0", + "wandb>=0.16.0", + "deepspeed>=0.12.4", ] readme = "README.md" requires-python = ">= 3.10" diff --git a/pyproject.toml b/pyproject.toml index 7d7cbef..fea446a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,23 @@ [project] name = "yuren" -version = "0.1.0" +version = "0.1.1" description = "Yuren is a open source Instruction-following LLM based on baichuan." authors = [ { name = "Pleisto Inc", email = "engineering@pleisto.com" } ] license = "Apache-2.0" dependencies = [ - "transformers @ git+https://github.com/huggingface/transformers@de9255de27abfcae4a1f816b904915f0b1e23cd9", + "transformers @ git+https://github.com/huggingface/transformers@v4.35.2", + "torch>=2.1.1", "sentencepiece>=0.1.99", - "einops>=0.6.1", + "einops>=0.7.0", "protobuf==3.*", - "bitsandbytes>=0.39.1", - "torch>=2.0.1", - "accelerate>=0.20.3", - "setuptools>=68.0.0", - "scipy>=1.10.1", + "bitsandbytes>=0.41.3", + "accelerate>=0.25.0", + "setuptools>=69.0.2", + "scipy>=1.11.4", + + "openai>=1.3.7" ] readme = "README.md" requires-python = ">= 3.10" diff --git a/requirements-dev.lock b/requirements-dev.lock index fae842e..c500310 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -12,145 +12,149 @@ -e file:. -e file:libs/yuren_core -e file:apps/yuren_trainer -accelerate==0.20.3 -aiofiles==23.1.0 -aiohttp==3.8.4 +accelerate==0.25.0 +aiofiles==23.2.1 +aiohttp==3.9.1 aiosignal==1.3.1 -altair==5.0.1 -anyio==3.7.0 +aiostream==0.5.2 +altair==5.2.0 +annotated-types==0.6.0 +anyio==3.7.1 appdirs==1.4.4 -async-timeout==4.0.2 +async-timeout==4.0.3 attrs==23.1.0 beautifulsoup4==4.12.2 -bitsandbytes==0.39.1 -black==23.3.0 -bs4==0.0.1 -certifi==2023.5.7 -charset-normalizer==3.1.0 -click==8.1.3 -cmake==3.26.4 -contourpy==1.1.0 -cycler==0.11.0 -dataclasses-json==0.5.8 -datasets==2.13.0 -deepspeed==0.9.5 -dill==0.3.6 +bitsandbytes==0.41.3 +black==23.11.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +contourpy==1.2.0 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.4 +deprecated==1.2.14 +dill==0.3.7 +distro==1.8.0 docker-pycreds==0.4.0 -einops==0.6.1 -exceptiongroup==1.1.1 -fastapi==0.98.0 -ffmpy==0.3.0 -filelock==3.12.2 -fonttools==4.40.0 -frozenlist==1.3.3 -fsspec==2023.6.0 -gitdb==4.0.10 -gitpython==3.1.31 +einops==0.7.0 +exceptiongroup==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.13.1 +fonttools==4.46.0 +frozenlist==1.4.0 +fsspec==2023.10.0 +gitdb==4.0.11 +gitpython==3.1.40 gradio==3.35.2 -gradio-client==0.2.7 -greenlet==2.0.2 +gradio-client==0.7.1 +greenlet==3.0.1 h11==0.14.0 hjson==3.1.0 -httpcore==0.17.2 -httpx==0.24.1 -huggingface-hub==0.15.1 -idna==3.4 +httpcore==1.0.2 +httpx==0.25.2 +huggingface-hub==0.19.4 +idna==3.6 isort==5.12.0 jinja2==3.1.2 -jsonschema==4.17.3 -kiwisolver==1.4.4 -langchain==0.0.215 -langchainplus-sdk==0.0.17 -latex2mathml==3.76.0 +joblib==1.3.2 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.2 +kiwisolver==1.4.5 +latex2mathml==3.77.0 linkify-it-py==2.0.2 -lit==16.0.6 -llama-index==0.6.33 -markdown==3.4.3 +llama-index==0.9.13 +markdown==3.5.1 markdown-it-py==2.2.0 markupsafe==2.1.3 -marshmallow==3.19.0 -marshmallow-enum==1.5.1 -matplotlib==3.7.1 +marshmallow==3.20.1 +matplotlib==3.8.2 mdit-py-plugins==0.3.3 mdtex2html==1.2.0 mdurl==0.1.2 mpmath==1.3.0 multidict==6.0.4 -multiprocess==0.70.14 +multiprocess==0.70.15 mypy-extensions==1.0.0 -networkx==3.1 -ninja==1.11.1 -numexpr==2.8.4 -numpy==1.25.0 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -openai==0.27.8 -openapi-schema-pydantic==1.2.4 -orjson==3.9.1 -packaging==23.1 -pandas==2.0.2 -pathspec==0.11.1 -pathtools==0.1.2 +nest-asyncio==1.5.8 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +openai==1.3.7 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.3 +pathspec==0.11.2 peft @ git+https://github.com/huggingface/peft -pillow==9.5.0 -platformdirs==3.8.0 +pillow==10.1.0 +platformdirs==4.1.0 protobuf==3.20.3 -psutil==5.9.5 +psutil==5.9.6 py-cpuinfo==9.0.0 -pyarrow==12.0.1 -pydantic==1.10.9 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic-core==2.14.5 pydub==0.25.1 -pygments==2.15.1 -pyparsing==3.1.0 -pyrsistent==0.19.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 python-dateutil==2.8.2 python-multipart==0.0.6 -pytz==2023.3 -pyyaml==6.0 -regex==2023.6.3 +pytz==2023.3.post1 +pyyaml==6.0.1 +referencing==0.31.1 +regex==2023.10.3 requests==2.31.0 -ruff==0.0.274 -safetensors==0.3.1 -scipy==1.10.1 +rpds-py==0.13.2 +ruff==0.1.7 +safetensors==0.4.1 +scipy==1.11.4 semantic-version==2.10.0 sentencepiece==0.1.99 -sentry-sdk==1.26.0 -setproctitle==1.3.2 +sentry-sdk==1.38.0 +setproctitle==1.3.3 six==1.16.0 -smmap==5.0.0 +smmap==5.0.1 sniffio==1.3.0 -soupsieve==2.4.1 -sqlalchemy==2.0.17 +soupsieve==2.5 +sqlalchemy==2.0.23 starlette==0.27.0 sympy==1.12 -tenacity==8.2.2 -tiktoken==0.4.0 -tokenizers==0.13.3 +tenacity==8.2.3 +tiktoken==0.5.2 +tokenizers==0.15.0 tomli==2.0.1 toolz==0.12.0 -torch==2.0.1 -tqdm==4.65.0 -transformers @ git+https://github.com/huggingface/transformers@de9255de27abfcae4a1f816b904915f0b1e23cd9 -triton==2.0.0 -typing-extensions==4.5.0 -typing-inspect==0.8.0 +torch==2.1.1 +tqdm==4.66.1 +transformers @ git+https://github.com/huggingface/transformers@v4.35.2 +triton==2.1.0 +typing-extensions==4.8.0 +typing-inspect==0.9.0 tzdata==2023.3 uc-micro-py==1.0.2 -urllib3==1.26.16 -uvicorn==0.22.0 -wandb==0.15.4 +urllib3==2.1.0 +uvicorn==0.24.0.post1 +wandb==0.16.1 websockets==11.0.3 -wheel==0.40.0 -xxhash==3.2.0 -yarl==1.9.2 +wrapt==1.16.0 +xxhash==3.4.1 +yarl==1.9.4 # The following packages are considered to be unsafe in a requirements file: -setuptools==68.0.0 +setuptools==69.0.2 diff --git a/requirements.lock b/requirements.lock index 375757b..e065766 100644 --- a/requirements.lock +++ b/requirements.lock @@ -12,139 +12,143 @@ -e file:. -e file:libs/yuren_core -e file:apps/yuren_trainer -accelerate==0.20.3 -aiofiles==23.1.0 -aiohttp==3.8.4 +accelerate==0.25.0 +aiofiles==23.2.1 +aiohttp==3.9.1 aiosignal==1.3.1 -altair==5.0.1 -anyio==3.7.0 +aiostream==0.5.2 +altair==5.2.0 +annotated-types==0.6.0 +anyio==3.7.1 appdirs==1.4.4 -async-timeout==4.0.2 +async-timeout==4.0.3 attrs==23.1.0 beautifulsoup4==4.12.2 -bitsandbytes==0.39.1 -bs4==0.0.1 -certifi==2023.5.7 -charset-normalizer==3.1.0 -click==8.1.3 -cmake==3.26.4 -contourpy==1.1.0 -cycler==0.11.0 -dataclasses-json==0.5.8 -datasets==2.13.0 -deepspeed==0.9.5 -dill==0.3.6 +bitsandbytes==0.41.3 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +contourpy==1.2.0 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.4 +deprecated==1.2.14 +dill==0.3.7 +distro==1.8.0 docker-pycreds==0.4.0 -einops==0.6.1 -exceptiongroup==1.1.1 -fastapi==0.98.0 -ffmpy==0.3.0 -filelock==3.12.2 -fonttools==4.40.0 -frozenlist==1.3.3 -fsspec==2023.6.0 -gitdb==4.0.10 -gitpython==3.1.31 +einops==0.7.0 +exceptiongroup==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.13.1 +fonttools==4.46.0 +frozenlist==1.4.0 +fsspec==2023.10.0 +gitdb==4.0.11 +gitpython==3.1.40 gradio==3.35.2 -gradio-client==0.2.7 -greenlet==2.0.2 +gradio-client==0.7.1 +greenlet==3.0.1 h11==0.14.0 hjson==3.1.0 -httpcore==0.17.2 -httpx==0.24.1 -huggingface-hub==0.15.1 -idna==3.4 +httpcore==1.0.2 +httpx==0.25.2 +huggingface-hub==0.19.4 +idna==3.6 jinja2==3.1.2 -jsonschema==4.17.3 -kiwisolver==1.4.4 -langchain==0.0.215 -langchainplus-sdk==0.0.17 -latex2mathml==3.76.0 +joblib==1.3.2 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.2 +kiwisolver==1.4.5 +latex2mathml==3.77.0 linkify-it-py==2.0.2 -lit==16.0.6 -llama-index==0.6.33 -markdown==3.4.3 +llama-index==0.9.13 +markdown==3.5.1 markdown-it-py==2.2.0 markupsafe==2.1.3 -marshmallow==3.19.0 -marshmallow-enum==1.5.1 -matplotlib==3.7.1 +marshmallow==3.20.1 +matplotlib==3.8.2 mdit-py-plugins==0.3.3 mdtex2html==1.2.0 mdurl==0.1.2 mpmath==1.3.0 multidict==6.0.4 -multiprocess==0.70.14 +multiprocess==0.70.15 mypy-extensions==1.0.0 -networkx==3.1 -ninja==1.11.1 -numexpr==2.8.4 -numpy==1.25.0 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -openai==0.27.8 -openapi-schema-pydantic==1.2.4 -orjson==3.9.1 -packaging==23.1 -pandas==2.0.2 -pathtools==0.1.2 +nest-asyncio==1.5.8 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +openai==1.3.7 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.3 peft @ git+https://github.com/huggingface/peft -pillow==9.5.0 +pillow==10.1.0 protobuf==3.20.3 -psutil==5.9.5 +psutil==5.9.6 py-cpuinfo==9.0.0 -pyarrow==12.0.1 -pydantic==1.10.9 +pyarrow==14.0.1 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic-core==2.14.5 pydub==0.25.1 -pygments==2.15.1 -pyparsing==3.1.0 -pyrsistent==0.19.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 python-dateutil==2.8.2 python-multipart==0.0.6 -pytz==2023.3 -pyyaml==6.0 -regex==2023.6.3 +pytz==2023.3.post1 +pyyaml==6.0.1 +referencing==0.31.1 +regex==2023.10.3 requests==2.31.0 -safetensors==0.3.1 -scipy==1.10.1 +rpds-py==0.13.2 +safetensors==0.4.1 +scipy==1.11.4 semantic-version==2.10.0 sentencepiece==0.1.99 -sentry-sdk==1.26.0 -setproctitle==1.3.2 +sentry-sdk==1.38.0 +setproctitle==1.3.3 six==1.16.0 -smmap==5.0.0 +smmap==5.0.1 sniffio==1.3.0 -soupsieve==2.4.1 -sqlalchemy==2.0.17 +soupsieve==2.5 +sqlalchemy==2.0.23 starlette==0.27.0 sympy==1.12 -tenacity==8.2.2 -tiktoken==0.4.0 -tokenizers==0.13.3 +tenacity==8.2.3 +tiktoken==0.5.2 +tokenizers==0.15.0 toolz==0.12.0 -torch==2.0.1 -tqdm==4.65.0 -transformers @ git+https://github.com/huggingface/transformers@de9255de27abfcae4a1f816b904915f0b1e23cd9 -triton==2.0.0 -typing-extensions==4.5.0 -typing-inspect==0.8.0 +torch==2.1.1 +tqdm==4.66.1 +transformers @ git+https://github.com/huggingface/transformers@v4.35.2 +triton==2.1.0 +typing-extensions==4.8.0 +typing-inspect==0.9.0 tzdata==2023.3 uc-micro-py==1.0.2 -urllib3==1.26.16 -uvicorn==0.22.0 -wandb==0.15.4 +urllib3==2.1.0 +uvicorn==0.24.0.post1 +wandb==0.16.1 websockets==11.0.3 -wheel==0.40.0 -xxhash==3.2.0 -yarl==1.9.2 +wrapt==1.16.0 +xxhash==3.4.1 +yarl==1.9.4 # The following packages are considered to be unsafe in a requirements file: -setuptools==68.0.0 +setuptools==69.0.2