diff --git a/ChatQnA/serving/tgi_gaudi/build_docker.sh b/ChatQnA/serving/tgi_gaudi/build_docker.sh index 2828258ae7..cd63bf81c6 100644 --- a/ChatQnA/serving/tgi_gaudi/build_docker.sh +++ b/ChatQnA/serving/tgi_gaudi/build_docker.sh @@ -2,4 +2,4 @@ git clone https://github.com/huggingface/tgi-gaudi.git cd ./tgi-gaudi/ -docker build -t tgi_gaudi . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy +docker build -t ghcr.io/huggingface/tgi-gaudi:1.2.1 . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy diff --git a/ChatQnA/serving/tgi_gaudi/launch_tgi_service.sh b/ChatQnA/serving/tgi_gaudi/launch_tgi_service.sh index 85a65093ea..b4e795bec7 100644 --- a/ChatQnA/serving/tgi_gaudi/launch_tgi_service.sh +++ b/ChatQnA/serving/tgi_gaudi/launch_tgi_service.sh @@ -27,9 +27,9 @@ volume=$PWD/data # Build the Docker run command based on the number of cards if [ "$num_cards" -eq 1 ]; then - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi --model-id $model_name" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name" else - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi --model-id $model_name --sharded true --num-shard $num_cards" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name --sharded true --num-shard $num_cards" fi # Execute the Docker run command diff --git a/CodeGen/README.md b/CodeGen/README.md index ca1fee2caf..487bb511a3 100644 --- a/CodeGen/README.md +++ b/CodeGen/README.md @@ -4,9 +4,17 @@ Code generation is a noteworthy application of Large Language Model (LLM) techno # Environment Setup To use [🤗 text-generation-inference](https://github.com/huggingface/text-generation-inference) on Intel Gaudi2, please follow these steps: -## Build TGI Gaudi Docker Image +## Prepare Gaudi Image +Getting started is straightforward with the official Docker container. Simply pull the image using: + +```bash +docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 +``` + +Alternatively, you can build the Docker image yourself with: + ```bash -bash ./tgi_gaudi/build_docker.sh +bash ./serving/tgi_gaudi/build_docker.sh ``` ## Launch TGI Gaudi Service diff --git a/CodeGen/tgi_gaudi/build_docker.sh b/CodeGen/tgi_gaudi/build_docker.sh index 13acf35081..6e38c86d76 100644 --- a/CodeGen/tgi_gaudi/build_docker.sh +++ b/CodeGen/tgi_gaudi/build_docker.sh @@ -16,4 +16,4 @@ git clone https://github.com/huggingface/tgi-gaudi.git cd ./tgi-gaudi/ -docker build -t tgi_gaudi_codegen . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy +docker build -t ghcr.io/huggingface/tgi-gaudi:1.2.1 . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy diff --git a/CodeGen/tgi_gaudi/launch_tgi_service.sh b/CodeGen/tgi_gaudi/launch_tgi_service.sh index 212ee6ebd5..12dfa59e4c 100644 --- a/CodeGen/tgi_gaudi/launch_tgi_service.sh +++ b/CodeGen/tgi_gaudi/launch_tgi_service.sh @@ -41,9 +41,9 @@ volume=$PWD/data # Build the Docker run command based on the number of cards if [ "$num_cards" -eq 1 ]; then - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy tgi_gaudi_codegen --model-id $model_name" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name" else - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy tgi_gaudi_codegen --model-id $model_name --sharded true --num-shard $num_cards" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name --sharded true --num-shard $num_cards" fi # Execute the Docker run command diff --git a/DocSum/serving/tgi_gaudi/build_docker.sh b/DocSum/serving/tgi_gaudi/build_docker.sh index 63bd88d39e..cd63bf81c6 100644 --- a/DocSum/serving/tgi_gaudi/build_docker.sh +++ b/DocSum/serving/tgi_gaudi/build_docker.sh @@ -2,4 +2,4 @@ git clone https://github.com/huggingface/tgi-gaudi.git cd ./tgi-gaudi/ -docker build -t tgi_gaudi_doc_summary . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy +docker build -t ghcr.io/huggingface/tgi-gaudi:1.2.1 . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy diff --git a/DocSum/serving/tgi_gaudi/launch_tgi_service.sh b/DocSum/serving/tgi_gaudi/launch_tgi_service.sh index 5a2909e72c..b4e795bec7 100644 --- a/DocSum/serving/tgi_gaudi/launch_tgi_service.sh +++ b/DocSum/serving/tgi_gaudi/launch_tgi_service.sh @@ -27,9 +27,9 @@ volume=$PWD/data # Build the Docker run command based on the number of cards if [ "$num_cards" -eq 1 ]; then - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy tgi_gaudi_doc_summary --model-id $model_name" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name" else - docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy tgi_gaudi_doc_summary --model-id $model_name --sharded true --num-shard $num_cards" + docker_cmd="docker run -p $port_number:80 -v $volume:/data --runtime=habana -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_name --sharded true --num-shard $num_cards" fi # Execute the Docker run command