diff --git a/.github/workflows/production_run_complete_llm.yml b/.github/workflows/production_run_complete_llm.yml new file mode 100644 index 00000000..c5cecdf2 --- /dev/null +++ b/.github/workflows/production_run_complete_llm.yml @@ -0,0 +1,58 @@ +name: Production LLM-COMPLETE +on: + push: + branches: + - main + paths: + - 'llm-complete-guide/**' +concurrency: + # New commit on branch cancels running workflows of the same branch + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-staging-workflow: + runs-on: ubuntu-latest + env: + ZENML_HOST: ${{ secrets.ZENML_HOST }} + ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }} + ZENML_PRODUCTION_STACK : 51a49786-b82a-4646-bde7-a460efb0a9c5 + ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }} + ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }} + ZENML_DEBUG: true + ZENML_ANALYTICS_OPT_IN: false + ZENML_LOGGING_VERBOSITY: INFO + ZENML_PROJECT_SECRET_NAME: llm-complete + ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True + ZENML_ACTION_ID: 23a4d58c-bd2b-47d5-a41d-0a845d2982f8 + + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install requirements + working-directory: ./llm-complete-guide + run: | + pip3 install -r requirements.txt + pip3 install -r requirements-argilla.txt + zenml integration install gcp -y + + - name: Connect to ZenML server + working-directory: ./llm-complete-guide + run: | + zenml init + zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY + + - name: Set stack (Production) + working-directory: ./llm-complete-guide + run: | + zenml stack set ${{ env.ZENML_PRODUCTION_STACK }} + + - name: Run pipeline, create pipeline, configure trigger (Production) + working-directory: ./llm-complete-guide + run: | + python scripts/gh_action_rag.py --no-cache --create-template --action-id ${{ env.ZENML_ACTION_ID }} \ No newline at end of file diff --git a/.github/workflows/run_complete_llm.yml b/.github/workflows/staging_run_complete_llm.yml similarity index 77% rename from .github/workflows/run_complete_llm.yml rename to .github/workflows/staging_run_complete_llm.yml index 978c92d6..07099bbe 100644 --- a/.github/workflows/run_complete_llm.yml +++ b/.github/workflows/staging_run_complete_llm.yml @@ -10,6 +10,7 @@ concurrency: jobs: run-staging-workflow: + runs-on: ubuntu-latest env: ZENML_HOST: ${{ secrets.ZENML_HOST }} ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }} @@ -20,6 +21,7 @@ jobs: ZENML_ANALYTICS_OPT_IN: false ZENML_LOGGING_VERBOSITY: INFO ZENML_PROJECT_SECRET_NAME: llm-complete + ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True steps: - name: Check out repository code @@ -30,20 +32,24 @@ jobs: python-version: '3.11' - name: Install requirements + working-directory: ./llm-complete-guide run: | pip3 install -r requirements.txt + pip3 install -r requirements-argilla.txt zenml integration install gcp -y - name: Connect to ZenML server + working-directory: ./llm-complete-guide run: | + zenml init zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY - name: Set stack (Staging) - if: ${{ github.base_ref == 'staging' }} + working-directory: ./llm-complete-guide run: | zenml stack set ${{ env.ZENML_STAGING_STACK }} - name: Run pipeline (Staging) - if: ${{ github.base_ref == 'staging' }} + working-directory: ./llm-complete-guide run: | - python run.py --rag --evaluation --no-cache + python scripts/gh_action_rag.py --no-cache \ No newline at end of file diff --git a/llm-complete-guide/configs/rag_gcp.yaml b/llm-complete-guide/configs/rag_gcp.yaml index b7f4c028..1c8d30e6 100644 --- a/llm-complete-guide/configs/rag_gcp.yaml +++ b/llm-complete-guide/configs/rag_gcp.yaml @@ -11,6 +11,7 @@ settings: - psycopg2-binary - tiktoken - ratelimit + - rerankers environment: ZENML_PROJECT_SECRET_NAME: llm_complete ZENML_ENABLE_RICH_TRACEBACK: FALSE @@ -20,13 +21,13 @@ steps: url_scraper: parameters: docs_url: https://docs.zenml.io - generate_embeddings: - step_operator: "terraform-gcp-6c0fd52233ca" - settings: - step_operator.vertex: - accelerator_type: "NVIDIA_TESLA_P100" - accelerator_count: 1 - machine_type: "n1-standard-8" +# generate_embeddings: +# step_operator: "terraform-gcp-6c0fd52233ca" +# settings: +# step_operator.vertex: +# accelerator_type: "NVIDIA_TESLA_P100" +# accelerator_count: 1 +# machine_type: "n1-standard-8" # configuration of the Model Control Plane model: diff --git a/llm-complete-guide/configs/rag_local_dev.yaml b/llm-complete-guide/configs/rag_local_dev.yaml index f460d80d..be6a7b03 100644 --- a/llm-complete-guide/configs/rag_local_dev.yaml +++ b/llm-complete-guide/configs/rag_local_dev.yaml @@ -13,6 +13,7 @@ settings: - psycopg2-binary - tiktoken - ratelimit + - rerankers environment: ZENML_PROJECT_SECRET_NAME: llm_complete ZENML_ENABLE_RICH_TRACEBACK: FALSE diff --git a/llm-complete-guide/pipelines/llm_basic_rag.py b/llm-complete-guide/pipelines/llm_basic_rag.py index 6cf99f08..3cfb4051 100644 --- a/llm-complete-guide/pipelines/llm_basic_rag.py +++ b/llm-complete-guide/pipelines/llm_basic_rag.py @@ -15,6 +15,8 @@ # limitations under the License. # +from zenml import pipeline + from steps.populate_index import ( generate_embeddings, index_generator, @@ -22,7 +24,6 @@ ) from steps.url_scraper import url_scraper from steps.web_url_loader import web_url_loader -from zenml import pipeline @pipeline diff --git a/llm-complete-guide/pipelines/llm_eval.py b/llm-complete-guide/pipelines/llm_eval.py index a0e20bc0..d310fd18 100644 --- a/llm-complete-guide/pipelines/llm_eval.py +++ b/llm-complete-guide/pipelines/llm_eval.py @@ -13,6 +13,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +from pathlib import Path +from typing import Optional + +import click from steps.eval_e2e import e2e_evaluation, e2e_evaluation_llm_judged from steps.eval_retrieval import ( @@ -64,3 +69,39 @@ def llm_eval() -> None: average_helpfulness_score, average_relevance_score, ) + + +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Disable cache.", +) +@click.option( + "--config", + "config", + default="rag_local_dev.yaml", + help="Specify a configuration file" +) +def main( + no_cache: bool = False, + config: Optional[str] = "rag_eval.yaml" +): + """ + Executes the pipeline to train a basic RAG model. + + Args: + no_cache (bool): If `True`, cache will be disabled. + config (str): The path to the configuration file. + """ + config_path = Path(__file__).parent.parent / "configs" / config + + llm_eval.with_options( + config_path=str(config_path), + enable_cache=not no_cache + )() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/llm-complete-guide/run.py b/llm-complete-guide/run.py index 45419f6a..ba001b4f 100644 --- a/llm-complete-guide/run.py +++ b/llm-complete-guide/run.py @@ -208,7 +208,6 @@ def main( rag_deployment.with_options( config_path=config_path, **pipeline_args )() - if deploy: rag_deployment.with_options(**pipeline_args)() if evaluation: diff --git a/llm-complete-guide/scripts/gh_action_rag.py b/llm-complete-guide/scripts/gh_action_rag.py new file mode 100644 index 00000000..3ccf25fa --- /dev/null +++ b/llm-complete-guide/scripts/gh_action_rag.py @@ -0,0 +1,106 @@ +# Apache Software License 2.0 +# +# Copyright (c) ZenML GmbH 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from datetime import datetime +from pathlib import Path +from typing import Optional +from uuid import UUID + +import click +from zenml.client import Client + +from pipelines import llm_basic_rag + + +@click.command( + help=""" +ZenML LLM Complete - Rag Pipeline +""" +) +@click.option( + "--no-cache", + "no_cache", + is_flag=True, + default=False, + help="Disable cache.", +) + +@click.option( + "--create-template", + "create_template", + is_flag=True, + default=False, + help="Create a run template.", +) +@click.option( + "--config", + "config", + default="rag_local_dev.yaml", + help="Specify a configuration file" +) +@click.option( + "--action-id", + "action_id", + default=None, + help="Specify an action ID" +) +def main( + no_cache: bool = False, + config: Optional[str]= "rag_local_dev.yaml", + create_template: bool = False, + action_id: Optional[str] = None +): + """ + Executes the pipeline to train a basic RAG model. + + Args: + no_cache (bool): If `True`, cache will be disabled. + config (str): The path to the configuration file. + create_template (bool): If `True`, a run template will be created. + action_id (str): The action ID. + """ + client = Client() + config_path = Path(__file__).parent.parent / "configs" / config + + if create_template: + # run pipeline + run = llm_basic_rag.with_options( + config_path=str(config_path), + enable_cache=not no_cache + )() + # create new run template + rt = client.create_run_template( + name=f"production-llm-complete-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", + deployment_id=run.deployment_id + ) + # update the action with the new template + client.update_action( + name_id_or_prefix=UUID(action_id), + configuration={ + "template_id": str(rt.id) + } + ) + + else: + llm_basic_rag.with_options( + config_path=str(config_path), + enable_cache=not no_cache + )() + + +if __name__ == "__main__": + main() \ No newline at end of file