fix(python): pin python deps

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Makefile

@@ -550,4 +550,4 @@ docker-image-intel:
 		--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
 		--build-arg GO_TAGS="none" \
-		--build-arg BUILD_TYPE=sycl_f16 -t $(DOCKER_IMAGE) .
+		--build-arg BUILD_TYPE=sycl_f32 -t $(DOCKER_IMAGE) .

backend/python/common-env/transformers/transformers-nvidia.yml

@@ -23,6 +23,7 @@ dependencies:
   - wheel=0.41.2=py311h06a4308_0
   - xz=5.4.2=h5eee18b_0
   - zlib=1.2.13=h5eee18b_0
+  - packaging
   - pip:
       - accelerate==0.23.0
       - aiohttp==3.8.5
@@ -115,4 +116,7 @@ dependencies:
       - vllm==0.2.7
       - transformers>=4.36.0  # Required for Mixtral.
       - xformers==0.0.23.post1  
+      # mamba
+      - mamba-ssm==1.0.1
+      - causal-conv1d==1.0.0
 prefix: /opt/conda/envs/transformers

backend/python/common-env/transformers/transformers.yml

@@ -23,6 +23,7 @@ dependencies:
   - wheel=0.41.2=py311h06a4308_0
   - xz=5.4.2=h5eee18b_0
   - zlib=1.2.13=h5eee18b_0
+  - packaging
   - pip:
       - accelerate==0.23.0
       - aiohttp==3.8.5
@@ -104,4 +105,7 @@ dependencies:
       - vllm==0.2.7
       - transformers>=4.36.0  # Required for Mixtral.
       - xformers==0.0.23.post1  
+      # mamba
+      - mamba-ssm==1.0.1
+      - causal-conv1d==1.0.0
 prefix: /opt/conda/envs/transformers

backend/python/diffusers/Makefile

@@ -1,8 +1,9 @@
+CONDA_ENV_PATH = "diffusers.yml"
+
 .PHONY: diffusers
 diffusers:
-	@echo "Creating virtual environment..."
-	@conda env create --name diffusers --file diffusers.yml
-	@echo "Virtual environment created."
+	@echo "Installing $(CONDA_ENV_PATH)..."
+	bash install.sh $(CONDA_ENV_PATH)
 
 .PHONY: run
 run:

backend/python/exllama2/install.sh

@@ -1,15 +1,25 @@
 #!/bin/bash
-
+set -e
 ##
 ## A bash script installs the required dependencies of VALL-E-X and prepares the environment
 export PATH=$PATH:/opt/conda/bin
+export SHA=c0ddebaaaf8ffd1b3529c2bb654e650bce2f790f
 
 # Activate conda environment
 source activate transformers
 
 echo $CONDA_PREFIX
 
-git clone https://github.com/turboderp/exllamav2 $CONDA_PREFIX/exllamav2 && pushd $CONDA_PREFIX/exllamav2 && pip install -r requirements.txt && popd
+git clone https://github.com/turboderp/exllamav2 $CONDA_PREFIX/exllamav2
+
+pushd $CONDA_PREFIX/exllamav2
+
+git checkout -b build $SHA
+
+# TODO: this needs to be pinned within the conda environments
+pip install -r requirements.txt
+
+popd
 
 cp -rfv $CONDA_PREFIX/exllamav2/* ./  
 

backend/python/mamba/Makefile

@@ -1,7 +1,6 @@
 .PHONY: mamba
 mamba:
 	$(MAKE) -C ../common-env/transformers
-	bash install.sh
 
 .PHONY: run
 run:

backend/python/vall-e-x/install.sh

@@ -10,10 +10,7 @@ source activate transformers
 
 echo $CONDA_PREFIX
 
-git clone https://github.com/Plachtaa/VALL-E-X.git $CONDA_PREFIX/vall-e-x && pushd $CONDA_PREFIX/vall-e-x && git checkout -b build $SHA && pip install -r requirements.txt && popd
-
-# Pin some dependencies (the upstream requirements is too much loose)
-pip install torchaudio==2.2.0
+git clone https://github.com/Plachtaa/VALL-E-X.git $CONDA_PREFIX/vall-e-x && pushd $CONDA_PREFIX/vall-e-x && git checkout -b build $SHA && popd
 
 cp -rfv $CONDA_PREFIX/vall-e-x/* ./