From c3f15202401ed9b728f785132b822f0828cb26fe Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 10 Aug 2024 12:13:09 +0200 Subject: [PATCH 1/3] Update abstract_graph.py --- scrapegraphai/graphs/abstract_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 83b532bc..c31c5558 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -228,7 +228,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): llm_params["model"] = "/".join(llm_params["model"].split("/")[1:]) except KeyError as exc: raise KeyError("Model not supported") from exc - return ChatNVIDIA(llm_params) + return ChatNVIDIA(**llm_config) # Raise an error if the model did not match any of the previous cases raise ValueError("Model provided by the configuration not supported") From e6bedb6701601e87a6dff99eabec9c3494280411 Mon Sep 17 00:00:00 2001 From: Federico Aguzzi <62149513+f-aguzzi@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:11:45 +0200 Subject: [PATCH 2/3] fix(AbstractGraph): pass kwargs to Ernie and Nvidia models Co-Authored-By: Marco Vinciguerra <88108002+VinciGit00@users.noreply.github.com> --- requirements-dev.lock | 35 -------------------------- requirements.lock | 33 ------------------------ scrapegraphai/graphs/abstract_graph.py | 4 +-- 3 files changed, 2 insertions(+), 70 deletions(-) diff --git a/requirements-dev.lock b/requirements-dev.lock index 39f2747d..6a90165b 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -6,8 +6,6 @@ # features: [] # all-features: false # with-sources: false -# generate-hashes: false -# universal: false -e file:. aiofiles==24.1.0 @@ -112,7 +110,6 @@ filelock==3.15.4 # via huggingface-hub # via torch # via transformers - # via triton fireworks-ai==0.14.0 # via langchain-fireworks fonttools==4.53.1 @@ -362,34 +359,6 @@ numpy==1.26.4 # via shapely # via streamlit # via transformers -nvidia-cublas-cu12==12.1.3.1 - # via nvidia-cudnn-cu12 - # via nvidia-cusolver-cu12 - # via torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via nvidia-cusolver-cu12 - # via torch -nvidia-nccl-cu12==2.19.3 - # via torch -nvidia-nvjitlink-cu12==12.6.20 - # via nvidia-cusolver-cu12 - # via nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch openai==1.37.0 # via burr # via langchain-fireworks @@ -631,8 +600,6 @@ tqdm==4.66.4 transformers==4.43.3 # via langchain-huggingface # via sentence-transformers -triton==2.2.0 - # via torch typer==0.12.3 # via fastapi-cli typing-extensions==4.12.2 @@ -676,8 +643,6 @@ uvicorn==0.30.3 # via fastapi uvloop==0.19.0 # via uvicorn -watchdog==4.0.1 - # via streamlit watchfiles==0.22.0 # via uvicorn websockets==12.0 diff --git a/requirements.lock b/requirements.lock index 7957082f..f449a7b7 100644 --- a/requirements.lock +++ b/requirements.lock @@ -6,8 +6,6 @@ # features: [] # all-features: false # with-sources: false -# generate-hashes: false -# universal: false -e file:. aiohttp==3.9.5 @@ -69,7 +67,6 @@ filelock==3.15.4 # via huggingface-hub # via torch # via transformers - # via triton fireworks-ai==0.14.0 # via langchain-fireworks free-proxy==1.1.1 @@ -267,34 +264,6 @@ numpy==1.26.4 # via sentence-transformers # via shapely # via transformers -nvidia-cublas-cu12==12.1.3.1 - # via nvidia-cudnn-cu12 - # via nvidia-cusolver-cu12 - # via torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via nvidia-cusolver-cu12 - # via torch -nvidia-nccl-cu12==2.19.3 - # via torch -nvidia-nvjitlink-cu12==12.6.20 - # via nvidia-cusolver-cu12 - # via nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch openai==1.37.0 # via langchain-fireworks # via langchain-openai @@ -446,8 +415,6 @@ tqdm==4.66.4 transformers==4.43.3 # via langchain-huggingface # via sentence-transformers -triton==2.2.0 - # via torch typing-extensions==4.12.2 # via anthropic # via anyio diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index c31c5558..16116997 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -211,7 +211,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): except KeyError: print("model not found, using default token size (8192)") self.model_token = 8192 - return ErnieBotChat(llm_params) + return ErnieBotChat(**llm_params) if "oneapi" in llm_params["model"]: # take the model after the last dash @@ -228,7 +228,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): llm_params["model"] = "/".join(llm_params["model"].split("/")[1:]) except KeyError as exc: raise KeyError("Model not supported") from exc - return ChatNVIDIA(**llm_config) + return ChatNVIDIA(**llm_params) # Raise an error if the model did not match any of the previous cases raise ValueError("Model provided by the configuration not supported") From 7fe181f69b3178d2d9d41a00fd660a98e04b777e Mon Sep 17 00:00:00 2001 From: Federico Aguzzi <62149513+f-aguzzi@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:38:16 +0200 Subject: [PATCH 3/3] chore(requirements): update requirements.txt Co-Authored-By: Marco Vinciguerra <88108002+VinciGit00@users.noreply.github.com> --- requirements.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 61f4c477..754eab61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ langchain-openai>=0.1.17 langchain-groq>=0.1.3 langchain-aws>=0.1.3 langchain-anthropic>=0.1.11 +langchain-mistralai>=0.1.12 +langchain-huggingface>=0.0.3 langchain-nvidia-ai-endpoints>=0.1.6 html2text>=2024.2.26 faiss-cpu>=1.8.0 @@ -22,8 +24,4 @@ playwright>=1.43.0 google>=3.0.0 undetected-playwright>=0.3.0 semchunk>=1.0.1 -langchain-fireworks>=0.1.3 -langchain-community>=0.2.9 -langchain-huggingface>=0.0.3 browserbase>=0.3.0 -langchain-mistralai>=0.1.12