From 5ddc7dd009ea013b8bf9dce45cc05855492e2e98 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:24:58 +0100 Subject: [PATCH 01/12] Removed new-line --- llm-complete-guide/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llm-complete-guide/run.py b/llm-complete-guide/run.py index 45419f6a..ba001b4f 100644 --- a/llm-complete-guide/run.py +++ b/llm-complete-guide/run.py @@ -208,7 +208,6 @@ def main( rag_deployment.with_options( config_path=config_path, **pipeline_args )() - if deploy: rag_deployment.with_options(**pipeline_args)() if evaluation: From bc1b62665a4741e2307b9d22c6c5b071ee94d8de Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:33:51 +0100 Subject: [PATCH 02/12] Fixed github action --- .github/workflows/run_complete_llm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 978c92d6..34cbe89a 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -10,6 +10,7 @@ concurrency: jobs: run-staging-workflow: + runs-on: ubuntu-latest env: ZENML_HOST: ${{ secrets.ZENML_HOST }} ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }} From 56f32e99128f36fef954c956c589634a73cd2ef2 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:37:19 +0100 Subject: [PATCH 03/12] Fixed workflow --- .github/workflows/run_complete_llm.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 34cbe89a..01e2c9eb 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -30,6 +30,9 @@ jobs: with: python-version: '3.11' + - name: Change directory + run: cd llm-complete-guide + - name: Install requirements run: | pip3 install -r requirements.txt @@ -37,6 +40,7 @@ jobs: - name: Connect to ZenML server run: | + zenml init zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY - name: Set stack (Staging) From 4fa9fbbf7464856744c565816776084e29210192 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:41:51 +0100 Subject: [PATCH 04/12] Fixed github action --- .github/workflows/run_complete_llm.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 01e2c9eb..61722401 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -30,25 +30,26 @@ jobs: with: python-version: '3.11' - - name: Change directory - run: cd llm-complete-guide - - name: Install requirements + working-directory: ./llm-complete-guide run: | pip3 install -r requirements.txt zenml integration install gcp -y - name: Connect to ZenML server + working-directory: ./llm-complete-guide run: | zenml init zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY - name: Set stack (Staging) + working-directory: ./llm-complete-guide if: ${{ github.base_ref == 'staging' }} run: | zenml stack set ${{ env.ZENML_STAGING_STACK }} - name: Run pipeline (Staging) + working-directory: ./llm-complete-guide if: ${{ github.base_ref == 'staging' }} run: | python run.py --rag --evaluation --no-cache From c3fe1f18b0c5f7a412a511015d2e4fd69f0edb9e Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:50:47 +0100 Subject: [PATCH 05/12] Ignore server mismatch --- .github/workflows/run_complete_llm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 61722401..d09f20b3 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -21,6 +21,7 @@ jobs: ZENML_ANALYTICS_OPT_IN: false ZENML_LOGGING_VERBOSITY: INFO ZENML_PROJECT_SECRET_NAME: llm-complete + ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True steps: - name: Check out repository code From 347224ecae5caa5236349dd34e1e07723c91cf11 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 17:55:38 +0100 Subject: [PATCH 06/12] Ignore server mismatch --- .github/workflows/run_complete_llm.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index d09f20b3..97187a68 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -45,12 +45,10 @@ jobs: - name: Set stack (Staging) working-directory: ./llm-complete-guide - if: ${{ github.base_ref == 'staging' }} run: | zenml stack set ${{ env.ZENML_STAGING_STACK }} - name: Run pipeline (Staging) working-directory: ./llm-complete-guide - if: ${{ github.base_ref == 'staging' }} run: | python run.py --rag --evaluation --no-cache From edb4439ea5f7573933b569ab2ed679bec2d7e4e0 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 22:32:02 +0100 Subject: [PATCH 07/12] Added all requirements --- .github/workflows/run_complete_llm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 97187a68..1fd72e88 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -35,6 +35,7 @@ jobs: working-directory: ./llm-complete-guide run: | pip3 install -r requirements.txt + pip3 install -r requirements_argilla.txt zenml integration install gcp -y - name: Connect to ZenML server From 4437d2beb721d32d3a15fa0fe329b2b5d59f1b27 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Mon, 28 Oct 2024 23:33:20 +0100 Subject: [PATCH 08/12] Fixed gh action --- .github/workflows/run_complete_llm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 1fd72e88..95f464de 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -35,7 +35,7 @@ jobs: working-directory: ./llm-complete-guide run: | pip3 install -r requirements.txt - pip3 install -r requirements_argilla.txt + pip3 install -r requirements-argilla.txt zenml integration install gcp -y - name: Connect to ZenML server From 2795e0970b675758073dbc6f4fdb31bf32cc20af Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Tue, 29 Oct 2024 08:23:46 +0100 Subject: [PATCH 09/12] More fine grained control over configs --- .github/workflows/run_complete_llm.yml | 2 +- llm-complete-guide/configs/rag_gcp.yaml | 15 ++++---- llm-complete-guide/configs/rag_local_dev.yaml | 1 + llm-complete-guide/pipelines/llm_basic_rag.py | 37 +++++++++++++++++++ 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/run_complete_llm.yml index 95f464de..ef576751 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/run_complete_llm.yml @@ -52,4 +52,4 @@ jobs: - name: Run pipeline (Staging) working-directory: ./llm-complete-guide run: | - python run.py --rag --evaluation --no-cache + python pipelines/llm_basic_rag.py --config=rag_gcp_yaml --no-cache diff --git a/llm-complete-guide/configs/rag_gcp.yaml b/llm-complete-guide/configs/rag_gcp.yaml index b7f4c028..1c8d30e6 100644 --- a/llm-complete-guide/configs/rag_gcp.yaml +++ b/llm-complete-guide/configs/rag_gcp.yaml @@ -11,6 +11,7 @@ settings: - psycopg2-binary - tiktoken - ratelimit + - rerankers environment: ZENML_PROJECT_SECRET_NAME: llm_complete ZENML_ENABLE_RICH_TRACEBACK: FALSE @@ -20,13 +21,13 @@ steps: url_scraper: parameters: docs_url: https://docs.zenml.io - generate_embeddings: - step_operator: "terraform-gcp-6c0fd52233ca" - settings: - step_operator.vertex: - accelerator_type: "NVIDIA_TESLA_P100" - accelerator_count: 1 - machine_type: "n1-standard-8" +# generate_embeddings: +# step_operator: "terraform-gcp-6c0fd52233ca" +# settings: +# step_operator.vertex: +# accelerator_type: "NVIDIA_TESLA_P100" +# accelerator_count: 1 +# machine_type: "n1-standard-8" # configuration of the Model Control Plane model: diff --git a/llm-complete-guide/configs/rag_local_dev.yaml b/llm-complete-guide/configs/rag_local_dev.yaml index f460d80d..be6a7b03 100644 --- a/llm-complete-guide/configs/rag_local_dev.yaml +++ b/llm-complete-guide/configs/rag_local_dev.yaml @@ -13,6 +13,7 @@ settings: - psycopg2-binary - tiktoken - ratelimit + - rerankers environment: ZENML_PROJECT_SECRET_NAME: llm_complete ZENML_ENABLE_RICH_TRACEBACK: FALSE diff --git a/llm-complete-guide/pipelines/llm_basic_rag.py b/llm-complete-guide/pipelines/llm_basic_rag.py index 6cf99f08..df2d6a94 100644 --- a/llm-complete-guide/pipelines/llm_basic_rag.py +++ b/llm-complete-guide/pipelines/llm_basic_rag.py @@ -14,6 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +from typing import Optional + +import click +from litellm import enable_cache from steps.populate_index import ( generate_embeddings, @@ -41,3 +46,35 @@ def llm_basic_rag() -> None: processed_docs = preprocess_documents(documents=docs) embedded_docs = generate_embeddings(split_documents=processed_docs) index_generator(documents=embedded_docs) + + +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Disable cache.", +) +@click.option( + "--config", + "config", + default="rag_local_dev.yaml", + help="Specify a configuration file" +) +def main( + no_cache: bool, + config: Optional[str]= "rag_local_dev.yaml" +): + config_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "configs", + config, + ) + llm_basic_rag.with_options( + config_path=config_path, + enable_cache=not no_cache + )() + + +if __name__ == "__main__": + main() \ No newline at end of file From 2e4fa23063cc72ff7a043bd75de136fff03d8571 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Tue, 29 Oct 2024 08:35:59 +0100 Subject: [PATCH 10/12] Added defaults --- llm-complete-guide/pipelines/llm_basic_rag.py | 9 +++- llm-complete-guide/pipelines/llm_eval.py | 43 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/llm-complete-guide/pipelines/llm_basic_rag.py b/llm-complete-guide/pipelines/llm_basic_rag.py index df2d6a94..e09e708a 100644 --- a/llm-complete-guide/pipelines/llm_basic_rag.py +++ b/llm-complete-guide/pipelines/llm_basic_rag.py @@ -62,9 +62,16 @@ def llm_basic_rag() -> None: help="Specify a configuration file" ) def main( - no_cache: bool, + no_cache: bool = False, config: Optional[str]= "rag_local_dev.yaml" ): + """ + Executes the pipeline to train a basic RAG model. + + Args: + no_cache (bool): If `True`, cache will be disabled. + config (str): The path to the configuration file. + """ config_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "configs", diff --git a/llm-complete-guide/pipelines/llm_eval.py b/llm-complete-guide/pipelines/llm_eval.py index a0e20bc0..42d1bdb2 100644 --- a/llm-complete-guide/pipelines/llm_eval.py +++ b/llm-complete-guide/pipelines/llm_eval.py @@ -13,6 +13,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +from typing import Optional + +import click from steps.eval_e2e import e2e_evaluation, e2e_evaluation_llm_judged from steps.eval_retrieval import ( @@ -64,3 +68,42 @@ def llm_eval() -> None: average_helpfulness_score, average_relevance_score, ) + + +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Disable cache.", +) +@click.option( + "--config", + "config", + default="rag_local_dev.yaml", + help="Specify a configuration file" +) +def main( + no_cache: bool = False, + config: Optional[str] = "rag_eval.yaml" +): + """ + Executes the pipeline to train a basic RAG model. + + Args: + no_cache (bool): If `True`, cache will be disabled. + config (str): The path to the configuration file. + """ + config_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "configs", + config, + ) + llm_eval.with_options( + config_path=config_path, + enable_cache=not no_cache + )() + + +if __name__ == "__main__": + main() \ No newline at end of file From b0a868b41a859b2be407e81121ddd2f3f5245bba Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Tue, 29 Oct 2024 08:46:05 +0100 Subject: [PATCH 11/12] fixed config path --- llm-complete-guide/pipelines/llm_basic_rag.py | 10 ++++------ llm-complete-guide/pipelines/llm_eval.py | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/llm-complete-guide/pipelines/llm_basic_rag.py b/llm-complete-guide/pipelines/llm_basic_rag.py index e09e708a..9c065284 100644 --- a/llm-complete-guide/pipelines/llm_basic_rag.py +++ b/llm-complete-guide/pipelines/llm_basic_rag.py @@ -15,6 +15,7 @@ # limitations under the License. # import os +from pathlib import Path from typing import Optional import click @@ -72,13 +73,10 @@ def main( no_cache (bool): If `True`, cache will be disabled. config (str): The path to the configuration file. """ - config_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "configs", - config, - ) + config_path = Path(__file__).parent.parent / "configs" / config + llm_basic_rag.with_options( - config_path=config_path, + config_path=str(config_path), enable_cache=not no_cache )() diff --git a/llm-complete-guide/pipelines/llm_eval.py b/llm-complete-guide/pipelines/llm_eval.py index 42d1bdb2..d310fd18 100644 --- a/llm-complete-guide/pipelines/llm_eval.py +++ b/llm-complete-guide/pipelines/llm_eval.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +from pathlib import Path from typing import Optional import click @@ -94,13 +95,10 @@ def main( no_cache (bool): If `True`, cache will be disabled. config (str): The path to the configuration file. """ - config_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "configs", - config, - ) + config_path = Path(__file__).parent.parent / "configs" / config + llm_eval.with_options( - config_path=config_path, + config_path=str(config_path), enable_cache=not no_cache )() From bd2e1b2afaa9ec3e265138975a4be98d931e52e3 Mon Sep 17 00:00:00 2001 From: AlexejPenner Date: Tue, 29 Oct 2024 13:10:19 +0100 Subject: [PATCH 12/12] Automate trigger update --- .../workflows/production_run_complete_llm.yml | 58 ++++++++++ ...e_llm.yml => staging_run_complete_llm.yml} | 2 +- llm-complete-guide/pipelines/llm_basic_rag.py | 43 +------ llm-complete-guide/scripts/gh_action_rag.py | 106 ++++++++++++++++++ 4 files changed, 166 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/production_run_complete_llm.yml rename .github/workflows/{run_complete_llm.yml => staging_run_complete_llm.yml} (95%) create mode 100644 llm-complete-guide/scripts/gh_action_rag.py diff --git a/.github/workflows/production_run_complete_llm.yml b/.github/workflows/production_run_complete_llm.yml new file mode 100644 index 00000000..c5cecdf2 --- /dev/null +++ b/.github/workflows/production_run_complete_llm.yml @@ -0,0 +1,58 @@ +name: Production LLM-COMPLETE +on: + push: + branches: + - main + paths: + - 'llm-complete-guide/**' +concurrency: + # New commit on branch cancels running workflows of the same branch + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-staging-workflow: + runs-on: ubuntu-latest + env: + ZENML_HOST: ${{ secrets.ZENML_HOST }} + ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }} + ZENML_PRODUCTION_STACK : 51a49786-b82a-4646-bde7-a460efb0a9c5 + ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }} + ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }} + ZENML_DEBUG: true + ZENML_ANALYTICS_OPT_IN: false + ZENML_LOGGING_VERBOSITY: INFO + ZENML_PROJECT_SECRET_NAME: llm-complete + ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True + ZENML_ACTION_ID: 23a4d58c-bd2b-47d5-a41d-0a845d2982f8 + + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + working-directory: ./llm-complete-guide + run: | + pip3 install -r requirements.txt + pip3 install -r requirements-argilla.txt + zenml integration install gcp -y + + - name: Connect to ZenML server + working-directory: ./llm-complete-guide + run: | + zenml init + zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY + + - name: Set stack (Production) + working-directory: ./llm-complete-guide + run: | + zenml stack set ${{ env.ZENML_PRODUCTION_STACK }} + + - name: Run pipeline, create pipeline, configure trigger (Production) + working-directory: ./llm-complete-guide + run: | + python scripts/gh_action_rag.py --no-cache --create-template --action-id ${{ env.ZENML_ACTION_ID }} \ No newline at end of file diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/staging_run_complete_llm.yml similarity index 95% rename from .github/workflows/run_complete_llm.yml rename to .github/workflows/staging_run_complete_llm.yml index ef576751..07099bbe 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/staging_run_complete_llm.yml @@ -52,4 +52,4 @@ jobs: - name: Run pipeline (Staging) working-directory: ./llm-complete-guide run: | - python pipelines/llm_basic_rag.py --config=rag_gcp_yaml --no-cache + python scripts/gh_action_rag.py --no-cache \ No newline at end of file diff --git a/llm-complete-guide/pipelines/llm_basic_rag.py b/llm-complete-guide/pipelines/llm_basic_rag.py index 9c065284..3cfb4051 100644 --- a/llm-complete-guide/pipelines/llm_basic_rag.py +++ b/llm-complete-guide/pipelines/llm_basic_rag.py @@ -14,12 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os -from pathlib import Path -from typing import Optional -import click -from litellm import enable_cache +from zenml import pipeline from steps.populate_index import ( generate_embeddings, @@ -28,7 +24,6 @@ ) from steps.url_scraper import url_scraper from steps.web_url_loader import web_url_loader -from zenml import pipeline @pipeline @@ -47,39 +42,3 @@ def llm_basic_rag() -> None: processed_docs = preprocess_documents(documents=docs) embedded_docs = generate_embeddings(split_documents=processed_docs) index_generator(documents=embedded_docs) - - -@click.option( - "--no-cache", - "no_cache", - is_flag=True, - default=False, - help="Disable cache.", -) -@click.option( - "--config", - "config", - default="rag_local_dev.yaml", - help="Specify a configuration file" -) -def main( - no_cache: bool = False, - config: Optional[str]= "rag_local_dev.yaml" -): - """ - Executes the pipeline to train a basic RAG model. - - Args: - no_cache (bool): If `True`, cache will be disabled. - config (str): The path to the configuration file. - """ - config_path = Path(__file__).parent.parent / "configs" / config - - llm_basic_rag.with_options( - config_path=str(config_path), - enable_cache=not no_cache - )() - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/llm-complete-guide/scripts/gh_action_rag.py b/llm-complete-guide/scripts/gh_action_rag.py new file mode 100644 index 00000000..3ccf25fa --- /dev/null +++ b/llm-complete-guide/scripts/gh_action_rag.py @@ -0,0 +1,106 @@ +# Apache Software License 2.0 +# +# Copyright (c) ZenML GmbH 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from datetime import datetime +from pathlib import Path +from typing import Optional +from uuid import UUID + +import click +from zenml.client import Client + +from pipelines import llm_basic_rag + + +@click.command( + help=""" +ZenML LLM Complete - Rag Pipeline +""" +) +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Disable cache.", +) + +@click.option( + "--create-template", + "create_template", + is_flag=True, + default=False, + help="Create a run template.", +) +@click.option( + "--config", + "config", + default="rag_local_dev.yaml", + help="Specify a configuration file" +) +@click.option( + "--action-id", + "action_id", + default=None, + help="Specify an action ID" +) +def main( + no_cache: bool = False, + config: Optional[str]= "rag_local_dev.yaml", + create_template: bool = False, + action_id: Optional[str] = None +): + """ + Executes the pipeline to train a basic RAG model. + + Args: + no_cache (bool): If `True`, cache will be disabled. + config (str): The path to the configuration file. + create_template (bool): If `True`, a run template will be created. + action_id (str): The action ID. + """ + client = Client() + config_path = Path(__file__).parent.parent / "configs" / config + + if create_template: + # run pipeline + run = llm_basic_rag.with_options( + config_path=str(config_path), + enable_cache=not no_cache + )() + # create new run template + rt = client.create_run_template( + name=f"production-llm-complete-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", + deployment_id=run.deployment_id + ) + # update the action with the new template + client.update_action( + name_id_or_prefix=UUID(action_id), + configuration={ + "template_id": str(rt.id) + } + ) + + else: + llm_basic_rag.with_options( + config_path=str(config_path), + enable_cache=not no_cache + )() + + +if __name__ == "__main__": + main() \ No newline at end of file