From 52fe9ebe519db2fda84b01521c94585349302bd9 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:44:11 +0100 Subject: [PATCH 1/9] Bump `transformers`, `diffusers`, `sentence-transformers` et al. --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index d5b3da68..96b815c3 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ # We don't declare our dependency on transformers here because we build with # different packages for different variants -VERSION = "0.5.4" +VERSION = "0.5.5" # Ubuntu packages # libsndfile1-dev: torchaudio requires the development version of the libsndfile package which can be installed via a system package manager. On Ubuntu it can be installed as follows: apt install libsndfile1-dev @@ -13,8 +13,8 @@ # libavcodec-extra : libavcodec-extra includes additional codecs for ffmpeg install_requires = [ - "transformers[sklearn,sentencepiece,audio,vision]==4.48.0", - "huggingface_hub[hf_transfer]==0.27.1", + "transformers[sklearn,sentencepiece,audio,vision]==4.49.0", + "huggingface_hub[hf_transfer]==0.29.2", # vision "Pillow", "librosa", @@ -31,8 +31,8 @@ extras = {} -extras["st"] = ["sentence_transformers==3.3.1"] -extras["diffusers"] = ["diffusers==0.32.1", "accelerate==1.2.1"] +extras["st"] = ["sentence_transformers==3.4.1"] +extras["diffusers"] = ["diffusers==0.32.2", "accelerate==1.4.0"] # Includes `peft` as PEFT requires `torch` so having `peft` as a core dependency # means that `torch` will be installed even if the `torch` extra is not specified. extras["torch"] = ["torch==2.3.1", "torchvision", "torchaudio", "peft==0.14.0"] @@ -57,7 +57,7 @@ version=VERSION, author="Hugging Face", description="Hugging Face Inference Toolkit is for serving 🤗 Transformers models in containers.", - url="", + url="https://github.com/huggingface/huggingface-inference-toolkit", package_dir={"": "src"}, packages=find_packages(where="src"), install_requires=install_requires, From ec8d0c7ca9cc327ed4ad79fde39207ad2ee1455d Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 13:36:05 +0100 Subject: [PATCH 2/9] try just popping from inputs --- src/huggingface_inference_toolkit/handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/huggingface_inference_toolkit/handler.py b/src/huggingface_inference_toolkit/handler.py index e3c721d1..eb82bb8c 100644 --- a/src/huggingface_inference_toolkit/handler.py +++ b/src/huggingface_inference_toolkit/handler.py @@ -50,6 +50,9 @@ def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: else self.pipeline(inputs, **parameters) ) + if self.pipeline.task == "automatic-speech-recognition": + inputs["input_features"] = inputs.pop("inputs") + if self.pipeline.task == "question-answering": if not isinstance(inputs, dict): raise ValueError(f"inputs must be a dict, but a `{type(inputs)}` was provided instead.") From 958809846cad603b1e7d6865c8f0dd42b55310ef Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 15:00:17 +0100 Subject: [PATCH 3/9] Revert "try just popping from inputs" This reverts commit ec8d0c7ca9cc327ed4ad79fde39207ad2ee1455d. --- src/huggingface_inference_toolkit/handler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/huggingface_inference_toolkit/handler.py b/src/huggingface_inference_toolkit/handler.py index eb82bb8c..e3c721d1 100644 --- a/src/huggingface_inference_toolkit/handler.py +++ b/src/huggingface_inference_toolkit/handler.py @@ -50,9 +50,6 @@ def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: else self.pipeline(inputs, **parameters) ) - if self.pipeline.task == "automatic-speech-recognition": - inputs["input_features"] = inputs.pop("inputs") - if self.pipeline.task == "question-answering": if not isinstance(inputs, dict): raise ValueError(f"inputs must be a dict, but a `{type(inputs)}` was provided instead.") From 8201938720df2cb42d286de4704d52e80a175e09 Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 15:25:08 +0100 Subject: [PATCH 4/9] maybe fix torch dtype issue --- src/huggingface_inference_toolkit/diffusers_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/huggingface_inference_toolkit/diffusers_utils.py b/src/huggingface_inference_toolkit/diffusers_utils.py index 47ddf390..751675f6 100644 --- a/src/huggingface_inference_toolkit/diffusers_utils.py +++ b/src/huggingface_inference_toolkit/diffusers_utils.py @@ -73,7 +73,13 @@ def __call__( logger.warning("The `output_type` cannot be modified, and PIL will be used by default instead.") # Call pipeline with parameters - out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) + if self.pipeline.device.type == "cuda": + model_dtype = next(self.pipeline.parameters()).dtype + with torch.autocast("cuda", dtype=model_dtype): + out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) + else: + out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) + return out.images[0] From da5a5874646e2c88a21ee80dec8bc71dca6872f3 Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 15:59:11 +0100 Subject: [PATCH 5/9] maybe we hardcode bfloat16 --- src/huggingface_inference_toolkit/diffusers_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/huggingface_inference_toolkit/diffusers_utils.py b/src/huggingface_inference_toolkit/diffusers_utils.py index 751675f6..366a0db3 100644 --- a/src/huggingface_inference_toolkit/diffusers_utils.py +++ b/src/huggingface_inference_toolkit/diffusers_utils.py @@ -74,8 +74,7 @@ def __call__( # Call pipeline with parameters if self.pipeline.device.type == "cuda": - model_dtype = next(self.pipeline.parameters()).dtype - with torch.autocast("cuda", dtype=model_dtype): + with torch.autocast("cuda", dtype=torch.bfloat16): out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) else: out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) From f877fbd09cee998156ab5e78d90c4db7b8998a82 Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 17:52:27 +0100 Subject: [PATCH 6/9] try with lower diffusers patch --- setup.py | 2 +- src/huggingface_inference_toolkit/diffusers_utils.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 96b815c3..445f6a52 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ extras = {} extras["st"] = ["sentence_transformers==3.4.1"] -extras["diffusers"] = ["diffusers==0.32.2", "accelerate==1.4.0"] +extras["diffusers"] = ["diffusers==0.32.1", "accelerate==1.4.0"] # Includes `peft` as PEFT requires `torch` so having `peft` as a core dependency # means that `torch` will be installed even if the `torch` extra is not specified. extras["torch"] = ["torch==2.3.1", "torchvision", "torchaudio", "peft==0.14.0"] diff --git a/src/huggingface_inference_toolkit/diffusers_utils.py b/src/huggingface_inference_toolkit/diffusers_utils.py index 366a0db3..9f893a81 100644 --- a/src/huggingface_inference_toolkit/diffusers_utils.py +++ b/src/huggingface_inference_toolkit/diffusers_utils.py @@ -73,11 +73,7 @@ def __call__( logger.warning("The `output_type` cannot be modified, and PIL will be used by default instead.") # Call pipeline with parameters - if self.pipeline.device.type == "cuda": - with torch.autocast("cuda", dtype=torch.bfloat16): - out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) - else: - out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) + out = self.pipeline(prompt, num_images_per_prompt=1, **kwargs) return out.images[0] From dfc1ed388c835703e8ef4b4641e352b68cf38746 Mon Sep 17 00:00:00 2001 From: erikkaum Date: Mon, 10 Mar 2025 18:06:24 +0100 Subject: [PATCH 7/9] revers all versions execpt transformers --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 445f6a52..a404471e 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requires = [ "transformers[sklearn,sentencepiece,audio,vision]==4.49.0", - "huggingface_hub[hf_transfer]==0.29.2", + "huggingface_hub[hf_transfer]==0.27.1", # vision "Pillow", "librosa", @@ -31,8 +31,8 @@ extras = {} -extras["st"] = ["sentence_transformers==3.4.1"] -extras["diffusers"] = ["diffusers==0.32.1", "accelerate==1.4.0"] +extras["st"] = ["sentence_transformers==3.3.1"] +extras["diffusers"] = ["diffusers==0.32.1", "accelerate==1.2.1"] # Includes `peft` as PEFT requires `torch` so having `peft` as a core dependency # means that `torch` will be installed even if the `torch` extra is not specified. extras["torch"] = ["torch==2.3.1", "torchvision", "torchaudio", "peft==0.14.0"] From 6d7bdb785637babebf102f3ad9808a760770b884 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:27:53 +0200 Subject: [PATCH 8/9] Drop Python 3.8 support --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 58009ba5..1773699f 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ install_requires=install_requires, extras_require=extras, entry_points={"console_scripts": "serve=sagemaker_huggingface_inference_toolkit.serving:main"}, - python_requires=">=3.8", + python_requires=">=3.9", license="Apache License 2.0", classifiers=[ "Development Status :: 5 - Production/Stable", @@ -76,7 +76,6 @@ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 253555bd22018a6e4a5268d372750736beefc307 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:28:11 +0200 Subject: [PATCH 9/9] Bump `torch`, `transformers` et al. --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 1773699f..28464388 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ # Due to an error affecting kenlm and cmake (see https://github.com/kpu/kenlm/pull/464) # Also see the transformers patch for it https://github.com/huggingface/transformers/pull/37091 "kenlm@git+https://github.com/kpu/kenlm@ba83eafdce6553addd885ed3da461bb0d60f8df7", - "transformers[sklearn,sentencepiece,audio,vision]==4.48.0", - "huggingface_hub[hf_transfer]==0.27.1", + "transformers[sklearn,sentencepiece,audio,vision]==4.51.2", + "huggingface_hub[hf_transfer]==0.30.2", # vision "Pillow", "librosa", @@ -34,11 +34,11 @@ extras = {} -extras["st"] = ["sentence_transformers==3.3.1"] -extras["diffusers"] = ["diffusers==0.32.1", "accelerate==1.2.1"] +extras["st"] = ["sentence_transformers==4.0.2"] +extras["diffusers"] = ["diffusers==0.33.1", "accelerate==1.6.0"] # Includes `peft` as PEFT requires `torch` so having `peft` as a core dependency # means that `torch` will be installed even if the `torch` extra is not specified. -extras["torch"] = ["torch==2.3.1", "torchvision", "torchaudio", "peft==0.14.0"] +extras["torch"] = ["torch==2.5.1", "torchvision", "torchaudio", "peft==0.15.1"] extras["test"] = [ "pytest==7.2.1", "pytest-xdist",