Refactor reqirements handling and model archive creation for TS MME e…

…xample (#4517) * Refactor reqirements handling and model archive creation * Update dependencies * Update source for downloading pretrained lama model * black-nb linting * Update model custom requirements and use Torchserve nightly * Set torchserve version to 0.10.0 --------- Co-authored-by: Naman Nandan <namannan@amazon.com>
aws · Mar 22, 2024 · a1af277 · a1af277
1 parent cc112d5
commit a1af277
Show file tree

Hide file tree

Showing 9 changed files with 45 additions and 66 deletions.
diff --git a/inference/torchserve/mme-gpu/torchserve_multi_model_endpoint.ipynb b/inference/torchserve/mme-gpu/torchserve_multi_model_endpoint.ipynb
@@ -116,7 +116,7 @@
     "    region=region,\n",
     "    py_version=\"py310\",\n",
     "    image_scope=\"inference\",\n",
-    "    version=\"2.0.0\",\n",
+    "    version=\"2.1.0\",\n",
     "    instance_type=\"ml.g5.2xlarge\",\n",
     ")\n",
     "print(baseimage)"
@@ -262,8 +262,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!torch-model-archiver --model-name sam --version 1.0 --serialized-file workspace/sam/sam_vit_h_4b8939.pth --handler workspace/sam/custom_handler.py --config-file workspace/sam/model-config.yaml --archive-format no-archive\n",
-    "!cd sam && tar cvzf sam.tar.gz ."
+    "!torch-model-archiver --model-name sam --version 1.0 --serialized-file workspace/sam/sam_vit_h_4b8939.pth --handler workspace/sam/custom_handler.py --config-file workspace/sam/model-config.yaml --requirements-file workspace/sam/requirements.txt --export-path workspace/sam --archive-format tgz"
    ]
   },
   {
@@ -273,7 +272,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!cd sam && aws s3 cp sam.tar.gz {output_path}/sam.tar.gz"
+    "!aws s3 cp workspace/sam/sam.tar.gz {output_path}/sam.tar.gz"
    ]
   },
   {
@@ -292,7 +291,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install -U torch diffusers==0.13.0 transformers"
+    "!pip install -U torch==2.1.0 diffusers==0.13.0 transformers"
    ]
   },
   {
@@ -376,8 +375,7 @@
    },
    "outputs": [],
    "source": [
-    "!torch-model-archiver --model-name sd --version 1.0 --handler workspace/sd/custom_handler.py --extra-files workspace/sd/model --config-file workspace/sam/model-config.yaml --archive-format no-archive\n",
-    "!cd sd && tar cvzf sd.tar.gz ."
+    "!torch-model-archiver --model-name sd --version 1.0 --handler workspace/sd/custom_handler.py --extra-files workspace/sd/model --config-file workspace/sd/model-config.yaml --requirements-file workspace/sd/requirements.txt --export-path workspace/sd --archive-format tgz"
    ]
   },
   {
@@ -389,24 +387,14 @@
     "#### Download Pre-Trained Model"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7baf9342",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip install wldhx.yadisk-direct --quiet"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "8d850797",
    "metadata": {},
    "outputs": [],
    "source": [
-    "!cd workspace/lama && curl -L $(yadisk-direct https://disk.yandex.ru/d/ouP6l8VJ0HpMZg) -o big-lama.zip && unzip big-lama.zip -d model"
+    "!cd workspace/lama && curl -LJO https://huggingface.co/smartywu/big-lama/resolve/main/big-lama.zip && unzip big-lama.zip -d model"
    ]
   },
   {
@@ -479,8 +467,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!torch-model-archiver --model-name lama --version 1.0 --handler workspace/lama/custom_handler.py --extra-files workspace/lama/model,workspace/lama/lama-repo --config-file workspace/sam/model-config.yaml --archive-format no-archive\n",
-    "!cd lama && tar cvzf lama.tar.gz ."
+    "!torch-model-archiver --model-name lama --version 1.0 --handler workspace/lama/custom_handler.py --extra-files workspace/lama/model,workspace/lama/lama-repo --config-file workspace/lama/model-config.yaml --requirements-file workspace/lama/requirements.txt --export-path workspace/lama --archive-format tgz"
    ]
   },
   {
@@ -521,7 +508,7 @@
     "    image_uri=container,\n",
     "    role=role,\n",
     "    sagemaker_session=smsess,\n",
-    "    env={\"TF_ENABLE_ONEDNN_OPTS\": \"0\"},\n",
+    "    env={\"TF_ENABLE_ONEDNN_OPTS\": \"0\", \"TS_INSTALL_PY_DEP_PER_MODEL\": \"true\"},\n",
     ")\n",
     "\n",
     "mme = MultiDataModel(\n",
@@ -611,7 +598,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "models = [\"sd/sd.tar.gz\", \"lama/lama.tar.gz\"]\n",
+    "models = [\"workspace/sd/sd.tar.gz\", \"workspace/lama/lama.tar.gz\"]\n",
     "for model in models:\n",
     "    mme.add_model(model_data_source=model)"
    ]

diff --git a/inference/torchserve/mme-gpu/workspace/code/requirements.txt b/inference/torchserve/mme-gpu/workspace/code/requirements.txt
diff --git a/inference/torchserve/mme-gpu/workspace/docker/Dockerfile b/inference/torchserve/mme-gpu/workspace/docker/Dockerfile
@@ -2,25 +2,6 @@ ARG BASE_IMAGE
 
 FROM $BASE_IMAGE
 
-#Install any additional libraries
-RUN pip install segment-anything-py==1.0
-RUN pip install opencv-python-headless==4.7.0.68
-RUN pip install matplotlib==3.6.3
-RUN pip install diffusers
-RUN pip install tqdm
-RUN pip install easydict
-RUN pip install scikit-image
-RUN pip install xformers
-RUN pip install tensorflow
-RUN pip install joblib
-RUN pip install matplotlib
-RUN pip install albumentations==0.5.2
-RUN pip install hydra-core==1.1.0
-RUN pip install pytorch-lightning
-RUN pip install tabulate
-RUN pip install kornia==0.5.0
-RUN pip install webdataset
-RUN pip install omegaconf==2.1.2
-RUN pip install transformers==4.28.1
-RUN pip install accelerate
-RUN pip install ftfy
+ENV TEMP=/tmp
+
+RUN pip install torchserve==0.10.0
diff --git a/inference/torchserve/mme-gpu/workspace/lama/model-config.yaml b/inference/torchserve/mme-gpu/workspace/lama/model-config.yaml
@@ -3,3 +3,4 @@ maxWorkers: 1
 batchSize: 1
 maxBatchDelay: 200
 responseTimeout: 300
+useVenv: true
diff --git a/inference/torchserve/mme-gpu/workspace/lama/requirements.txt b/inference/torchserve/mme-gpu/workspace/lama/requirements.txt
@@ -0,0 +1,19 @@
+pyyaml
+tqdm
+numpy
+easydict
+scikit-image
+scikit-learn
+opencv-python
+tensorflow
+joblib
+matplotlib
+pandas
+albumentations==0.5.2
+hydra-core
+pytorch-lightning
+tabulate
+kornia==0.5.0
+webdataset
+packaging
+timm
diff --git a/inference/torchserve/mme-gpu/workspace/sam/model-config.yaml b/inference/torchserve/mme-gpu/workspace/sam/model-config.yaml
@@ -3,3 +3,4 @@ maxWorkers: 1
 batchSize: 1
 maxBatchDelay: 200
 responseTimeout: 300
+useVenv: true
diff --git a/inference/torchserve/mme-gpu/workspace/sam/requirements.txt b/inference/torchserve/mme-gpu/workspace/sam/requirements.txt
@@ -0,0 +1,3 @@
+segment-anything-py==1.0
+opencv-python-headless==4.7.0.68
+matplotlib==3.6.3
diff --git a/inference/torchserve/mme-gpu/workspace/sd/model-config.yaml b/inference/torchserve/mme-gpu/workspace/sd/model-config.yaml
@@ -3,3 +3,4 @@ maxWorkers: 1
 batchSize: 1
 maxBatchDelay: 200
 responseTimeout: 300
+useVenv: true
diff --git a/inference/torchserve/mme-gpu/workspace/sd/requirements.txt b/inference/torchserve/mme-gpu/workspace/sd/requirements.txt
@@ -0,0 +1,8 @@
+--extra-index-url https://download.pytorch.org/whl/cu118
+transformers
+ftfy
+scipy
+accelerate
+diffusers==0.13.0
+opencv-python-headless==4.7.0.68
+xformers==0.0.23+cu118