diff --git a/requirements-dev.lock b/requirements-dev.lock index ad6fe188..ffcd2d40 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -6,13 +6,13 @@ # features: [] # all-features: false # with-sources: false -# generate-hashes: false -# universal: false -e file:. aiofiles==24.1.0 # via burr -aiohttp==3.9.5 +aiohappyeyeballs==2.3.5 + # via aiohttp +aiohttp==3.10.3 # via langchain # via langchain-community # via langchain-fireworks @@ -21,11 +21,11 @@ aiosignal==1.3.1 # via aiohttp alabaster==0.7.16 # via sphinx -altair==5.3.0 +altair==5.4.0 # via streamlit annotated-types==0.7.0 # via pydantic -anthropic==0.31.2 +anthropic==0.33.0 # via langchain-anthropic anyio==4.4.0 # via anthropic @@ -33,17 +33,16 @@ anyio==4.4.0 # via httpx # via openai # via starlette - # via watchfiles astroid==3.2.4 # via pylint async-timeout==4.0.3 # via aiohttp # via langchain -attrs==23.2.0 +attrs==24.2.0 # via aiohttp # via jsonschema # via referencing -babel==2.15.0 +babel==2.16.0 # via sphinx beautifulsoup4==4.12.3 # via furo @@ -51,9 +50,9 @@ beautifulsoup4==4.12.3 # via scrapegraphai blinker==1.8.2 # via streamlit -boto3==1.34.146 +boto3==1.34.158 # via langchain-aws -botocore==1.34.146 +botocore==1.34.158 # via boto3 # via s3transfer browserbase==0.3.0 @@ -72,7 +71,6 @@ charset-normalizer==3.3.2 click==8.1.7 # via burr # via streamlit - # via typer # via uvicorn contourpy==1.2.1 # via matplotlib @@ -89,31 +87,24 @@ distro==1.9.0 # via anthropic # via groq # via openai -dnspython==2.6.1 - # via email-validator docstring-parser==0.16 # via google-cloud-aiplatform docutils==0.19 # via sphinx -email-validator==2.2.0 - # via fastapi exceptiongroup==1.2.2 # via anyio # via pytest faiss-cpu==1.8.0.post1 # via scrapegraphai -fastapi==0.111.1 +fastapi==0.112.0 # via burr -fastapi-cli==0.0.4 - # via fastapi fastapi-pagination==0.12.26 # via burr filelock==3.15.4 # via huggingface-hub # via torch # via transformers - # via triton -fireworks-ai==0.14.0 +fireworks-ai==0.15.0 # via langchain-fireworks fonttools==4.53.1 # via matplotlib @@ -144,9 +135,9 @@ google-api-core==2.19.1 # via google-cloud-resource-manager # via google-cloud-storage # via google-generativeai -google-api-python-client==2.137.0 +google-api-python-client==2.140.0 # via google-generativeai -google-auth==2.32.0 +google-auth==2.33.0 # via google-ai-generativelanguage # via google-api-core # via google-api-python-client @@ -159,16 +150,16 @@ google-auth==2.32.0 # via google-generativeai google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-aiplatform==1.59.0 +google-cloud-aiplatform==1.61.0 # via langchain-google-vertexai google-cloud-bigquery==3.25.0 # via google-cloud-aiplatform google-cloud-core==2.4.1 # via google-cloud-bigquery # via google-cloud-storage -google-cloud-resource-manager==1.12.4 +google-cloud-resource-manager==1.12.5 # via google-cloud-aiplatform -google-cloud-storage==2.18.0 +google-cloud-storage==2.18.2 # via google-cloud-aiplatform # via langchain-google-vertexai google-crc32c==1.5.0 @@ -176,7 +167,7 @@ google-crc32c==1.5.0 # via google-resumable-media google-generativeai==0.7.2 # via langchain-google-genai -google-resumable-media==2.7.1 +google-resumable-media==2.7.2 # via google-cloud-bigquery # via google-cloud-storage googleapis-common-protos==1.63.2 @@ -193,12 +184,12 @@ groq==0.9.0 # via langchain-groq grpc-google-iam-v1==0.13.1 # via google-cloud-resource-manager -grpcio==1.65.1 +grpcio==1.65.4 # via google-api-core # via googleapis-common-protos # via grpc-google-iam-v1 # via grpcio-status -grpcio-status==1.62.2 +grpcio-status==1.62.3 # via google-api-core h11==0.14.0 # via httpcore @@ -210,12 +201,9 @@ httpcore==1.0.5 httplib2==0.22.0 # via google-api-python-client # via google-auth-httplib2 -httptools==0.6.1 - # via uvicorn httpx==0.27.0 # via anthropic # via browserbase - # via fastapi # via fireworks-ai # via groq # via langchain-mistralai @@ -223,20 +211,19 @@ httpx==0.27.0 httpx-sse==0.4.0 # via fireworks-ai # via langchain-mistralai -huggingface-hub==0.24.1 +huggingface-hub==0.24.5 # via langchain-huggingface # via sentence-transformers # via tokenizers # via transformers idna==3.7 # via anyio - # via email-validator # via httpx # via requests # via yarl imagesize==1.4.1 # via sphinx -importlib-metadata==8.1.0 +importlib-metadata==8.2.0 # via sphinx importlib-resources==6.4.0 # via matplotlib @@ -247,12 +234,12 @@ isort==5.13.2 jinja2==3.1.4 # via altair # via burr - # via fastapi # via pydeck # via sphinx # via torch jiter==0.5.0 # via anthropic + # via openai jmespath==1.0.1 # via boto3 # via botocore @@ -268,16 +255,16 @@ jsonschema-specifications==2023.12.1 # via jsonschema kiwisolver==1.4.5 # via matplotlib -langchain==0.2.11 +langchain==0.2.12 # via langchain-community # via scrapegraphai -langchain-anthropic==0.1.20 +langchain-anthropic==0.1.22 # via scrapegraphai -langchain-aws==0.1.12 +langchain-aws==0.1.16 # via scrapegraphai -langchain-community==0.2.10 +langchain-community==0.2.11 # via scrapegraphai -langchain-core==0.2.28 +langchain-core==0.2.29 # via langchain # via langchain-anthropic # via langchain-aws @@ -291,31 +278,31 @@ langchain-core==0.2.28 # via langchain-nvidia-ai-endpoints # via langchain-openai # via langchain-text-splitters -langchain-fireworks==0.1.5 +langchain-fireworks==0.1.7 # via scrapegraphai langchain-google-genai==1.0.8 # via scrapegraphai -langchain-google-vertexai==1.0.7 +langchain-google-vertexai==1.0.8 # via scrapegraphai -langchain-groq==0.1.6 +langchain-groq==0.1.9 # via scrapegraphai langchain-huggingface==0.0.3 # via scrapegraphai langchain-mistralai==0.1.12 # via scrapegraphai -langchain-nvidia-ai-endpoints==0.1.7 +langchain-nvidia-ai-endpoints==0.2.1 # via scrapegraphai -langchain-openai==0.1.17 +langchain-openai==0.1.21 # via scrapegraphai langchain-text-splitters==0.2.2 # via langchain -langsmith==0.1.93 +langsmith==0.1.99 # via langchain # via langchain-community # via langchain-core loguru==0.7.2 # via burr -lxml==5.2.2 +lxml==5.3.0 # via free-proxy markdown-it-py==3.0.0 # via rich @@ -323,7 +310,7 @@ markupsafe==2.1.5 # via jinja2 marshmallow==3.21.3 # via dataclasses-json -matplotlib==3.9.1 +matplotlib==3.9.1.post1 # via burr mccabe==0.7.0 # via pylint @@ -342,10 +329,11 @@ multiprocess==0.70.16 # via mpire mypy-extensions==1.0.0 # via typing-inspect +narwhals==1.3.0 + # via altair networkx==3.2.1 # via torch numpy==1.26.4 - # via altair # via contourpy # via faiss-cpu # via langchain @@ -362,39 +350,11 @@ numpy==1.26.4 # via shapely # via streamlit # via transformers -nvidia-cublas-cu12==12.1.3.1 - # via nvidia-cudnn-cu12 - # via nvidia-cusolver-cu12 - # via torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via nvidia-cusolver-cu12 - # via torch -nvidia-nccl-cu12==2.19.3 - # via torch -nvidia-nvjitlink-cu12==12.6.20 - # via nvidia-cusolver-cu12 - # via nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch -openai==1.37.0 +openai==1.40.3 # via burr # via langchain-fireworks # via langchain-openai -orjson==3.10.6 +orjson==3.10.7 # via langsmith packaging==24.1 # via altair @@ -410,7 +370,6 @@ packaging==24.1 # via streamlit # via transformers pandas==2.2.2 - # via altair # via scrapegraphai # via sf-hamilton # via streamlit @@ -433,7 +392,7 @@ proto-plus==1.24.0 # via google-api-core # via google-cloud-aiplatform # via google-cloud-resource-manager -protobuf==4.25.3 +protobuf==4.25.4 # via google-ai-generativelanguage # via google-api-core # via google-cloud-aiplatform @@ -490,22 +449,18 @@ python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 # via scrapegraphai - # via uvicorn -python-multipart==0.0.9 - # via fastapi pytz==2024.1 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via huggingface-hub # via langchain # via langchain-community # via langchain-core # via transformers - # via uvicorn referencing==0.35.1 # via jsonschema # via jsonschema-specifications -regex==2024.5.15 +regex==2024.7.24 # via tiktoken # via transformers requests==2.32.3 @@ -525,15 +480,14 @@ requests==2.32.3 # via transformers rich==13.7.1 # via streamlit - # via typer -rpds-py==0.19.0 +rpds-py==0.20.0 # via jsonschema # via referencing rsa==4.9 # via google-auth s3transfer==0.10.2 # via boto3 -safetensors==0.4.3 +safetensors==0.4.4 # via transformers scikit-learn==1.5.1 # via sentence-transformers @@ -544,12 +498,10 @@ semchunk==2.2.0 # via scrapegraphai sentence-transformers==3.0.1 # via langchain-huggingface -sf-hamilton==1.72.1 +sf-hamilton==1.73.1 # via burr shapely==2.0.5 # via google-cloud-aiplatform -shellingham==1.5.4 - # via typer six==1.16.0 # via python-dateutil smmap==5.0.1 @@ -570,26 +522,26 @@ sphinx==6.0.0 # via sphinx-basic-ng sphinx-basic-ng==1.0.0b2 # via furo -sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.6 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.8 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sqlalchemy==2.0.31 +sqlalchemy==2.0.32 # via langchain # via langchain-community starlette==0.37.2 # via fastapi -streamlit==1.36.0 +streamlit==1.37.1 # via burr -sympy==1.13.1 +sympy==1.13.2 # via torch tenacity==8.5.0 # via langchain @@ -613,13 +565,11 @@ tomli==2.0.1 # via pytest tomlkit==0.13.0 # via pylint -toolz==0.12.1 - # via altair torch==2.2.2 # via sentence-transformers tornado==6.4.1 # via streamlit -tqdm==4.66.4 +tqdm==4.66.5 # via google-generativeai # via huggingface-hub # via mpire @@ -628,13 +578,9 @@ tqdm==4.66.4 # via semchunk # via sentence-transformers # via transformers -transformers==4.43.3 +transformers==4.44.0 # via langchain-huggingface # via sentence-transformers -triton==2.2.0 - # via torch -typer==0.12.3 - # via fastapi-cli typing-extensions==4.12.2 # via altair # via anthropic @@ -656,7 +602,6 @@ typing-extensions==4.12.2 # via starlette # via streamlit # via torch - # via typer # via typing-inspect # via uvicorn typing-inspect==0.9.0 @@ -671,19 +616,10 @@ uritemplate==4.1.1 urllib3==1.26.19 # via botocore # via requests -uvicorn==0.30.3 +uvicorn==0.30.5 # via burr - # via fastapi -uvloop==0.19.0 - # via uvicorn -watchdog==4.0.2 - # via streamlit -watchfiles==0.22.0 - # via uvicorn -websockets==12.0 - # via uvicorn yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.20.0 # via importlib-metadata # via importlib-resources diff --git a/requirements.lock b/requirements.lock index 7957082f..f449a7b7 100644 --- a/requirements.lock +++ b/requirements.lock @@ -6,8 +6,6 @@ # features: [] # all-features: false # with-sources: false -# generate-hashes: false -# universal: false -e file:. aiohttp==3.9.5 @@ -69,7 +67,6 @@ filelock==3.15.4 # via huggingface-hub # via torch # via transformers - # via triton fireworks-ai==0.14.0 # via langchain-fireworks free-proxy==1.1.1 @@ -267,34 +264,6 @@ numpy==1.26.4 # via sentence-transformers # via shapely # via transformers -nvidia-cublas-cu12==12.1.3.1 - # via nvidia-cudnn-cu12 - # via nvidia-cusolver-cu12 - # via torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via nvidia-cusolver-cu12 - # via torch -nvidia-nccl-cu12==2.19.3 - # via torch -nvidia-nvjitlink-cu12==12.6.20 - # via nvidia-cusolver-cu12 - # via nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch openai==1.37.0 # via langchain-fireworks # via langchain-openai @@ -446,8 +415,6 @@ tqdm==4.66.4 transformers==4.43.3 # via langchain-huggingface # via sentence-transformers -triton==2.2.0 - # via torch typing-extensions==4.12.2 # via anthropic # via anyio diff --git a/requirements.txt b/requirements.txt index 61f4c477..754eab61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ langchain-openai>=0.1.17 langchain-groq>=0.1.3 langchain-aws>=0.1.3 langchain-anthropic>=0.1.11 +langchain-mistralai>=0.1.12 +langchain-huggingface>=0.0.3 langchain-nvidia-ai-endpoints>=0.1.6 html2text>=2024.2.26 faiss-cpu>=1.8.0 @@ -22,8 +24,4 @@ playwright>=1.43.0 google>=3.0.0 undetected-playwright>=0.3.0 semchunk>=1.0.1 -langchain-fireworks>=0.1.3 -langchain-community>=0.2.9 -langchain-huggingface>=0.0.3 browserbase>=0.3.0 -langchain-mistralai>=0.1.12 diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index ab53862d..eecd2297 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -210,7 +210,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): except KeyError: print("model not found, using default token size (8192)") self.model_token = 8192 - return ErnieBotChat(llm_params) + return ErnieBotChat(**llm_params) if "oneapi" in llm_params["model"]: # take the model after the last dash @@ -227,7 +227,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): llm_params["model"] = "/".join(llm_params["model"].split("/")[1:]) except KeyError as exc: raise KeyError("Model not supported") from exc - return ChatNVIDIA(llm_params) + return ChatNVIDIA(**llm_params) # Raise an error if the model did not match any of the previous cases raise ValueError("Model provided by the configuration not supported")