Install pip at build time instead of runtime (#39573)

The current list of pip dependencies are installed at runtime through ray.init. That does not seem to work. Install these dependencies at docker build time instead.

Signed-off-by: can <can@anyscale.com>

release/ray_release/byod/byod_finetune_llvms.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the workspace_template_finetuning_llms_with_deepspeed_llama_2_7b test
+
+set -exo pipefail
+
+pip3 install -U \
+    torch==2.0.0 \
+    torchvision==0.15.1 \
+    torchaudio==2.0.1 \
+    deepspeed==0.10.0 \
+    fairscale==0.4.13 \
+    datasets==2.14.4 \
+    accelerate==0.21.0 \
+    evaluate==0.4.0 \
+    bitsandbytes==0.41.1 \
+    wandb==0.15.8 \
+    pytorch-lightning==2.0.6 \
+    "protobuf<3.21.0" \
+    torchmetrics==1.0.3 \
+    lm_eval==0.3.0 \
+    tiktoken==0.1.2 \
+    sentencepiece==0.1.99 \
+    "urllib3<1.27" \
+    git+https://github.com/huggingface/transformers.git@d0c1aeb 

release/release_tests.yaml

@@ -1039,25 +1039,7 @@
     byod:
       type: cu121
       # This needs to be in sync with requirements under go/llm-forge.
-      pip:
-        - torch==2.0.0
-        - torchvision==0.15.1
-        - torchaudio==2.0.1
-        - deepspeed==0.10.0
-        - fairscale==0.4.13
-        - datasets==2.14.4
-        - accelerate==0.21.0
-        - evaluate==0.4.0
-        - bitsandbytes==0.41.1
-        - wandb==0.15.8
-        - pytorch-lightning==2.0.6
-        - protobuf<3.21.0
-        - torchmetrics==1.0.3
-        - lm_eval==0.3.0
-        - tiktoken==0.1.2
-        - sentencepiece==0.1.99
-        - urllib3<1.27
-        - git+https://github.com/huggingface/transformers.git@d0c1aeb
+      post_build_script: byod_finetune_llvms.sh
     cluster_compute: ../testing/compute_configs/04_finetuning_llms_with_deepspeed/aws_7b.yaml
 
   run: