Refactoring and updates #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LLMOps with PromptFlow | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [ main ] | |
| env: | |
| GROUP: ${{secrets.GROUP}} | |
| WORKSPACE: ${{secrets.WORKSPACE}} | |
| SUBSCRIPTION: ${{secrets.SUBSCRIPTION}} | |
| RUN_NAME: llmopsqa | |
| EVAL_RUN_NAME: llmopsqa_eval | |
| WORKSHOP_PATH: 'Workshop' | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| login-and-run-and-evalpf: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: check out repo | |
| uses: actions/checkout@v2 | |
| - name: install az ml extension | |
| run: az extension add -n ml -y | |
| - name: azure login | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{secrets.AZURE_RBAC_CREDENTIALS}} | |
| - name: Set up Python version | |
| uses: actions/setup-python@v3.0.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Create and start virtual environment | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| - name: 'Resolve Project Dependencies Using Pip' | |
| shell: bash | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| popd | |
| - name: run promptflow | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| pfazure run create -f promptflow/llmopsqa/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/run_info.txt | |
| cat promptflow/llmops-helper/run_info.txt | |
| popd | |
| - name: set run name | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| echo "RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV" | |
| popd | |
| - name: show the current run name | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| echo "Run name is:" ${{env.RUN_NAME}} | |
| popd | |
| - name: show promptflow results | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} | |
| popd | |
| # - name: run promptflow evaluations | |
| # run: pfazure run create -f promptflow/llmopsqa/run_evaluation.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/eval_info.txt | |
| # - name: get eval run name | |
| # run: export EVAL_RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py eval_info.txt) | |
| # - name: show promptflow details | |
| # run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} | |
| # - name: show promptflow metrics | |
| # run: pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > promptflow/llmops-helper/eval_result.json | |
| assert-and-register-model: | |
| needs: login-and-run-and-evalpf | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: check out repo | |
| uses: actions/checkout@v2 | |
| - name: install az ml extension | |
| run: az extension add -n ml -y | |
| - name: azure login | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{secrets.AZURE_RBAC_CREDENTIALS}} | |
| - name: Set up Python version | |
| uses: actions/setup-python@v3.0.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Create and start virtual environment | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| - name: 'Resolve Project Dependencies Using Pip' | |
| shell: bash | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| popd | |
| - name: set default subscription | |
| run: | | |
| az account set -s ${{env.SUBSCRIPTION}} | |
| - name: get assert eval results | |
| id: jobMetricAssert | |
| run: | | |
| export ASSERT=$(python promptflow/llmops-helper/assert.py result.json 0.6) # NOTE <file>.json is the file name and decimal is the threshold for the assertion | |
| echo "::debug::Assert has returned the following value: $ASSERT" | |
| # assert.py will return True or False, but bash expects lowercase. | |
| if ${ASSERT,,} ; then | |
| echo "::debug::Prompt flow run met the quality bar and can be deployed." | |
| echo "::set-output name=result::true" | |
| else | |
| echo "::warning::Prompt flow run didn't meet quality bar." | |
| echo "::set-output name=result::false" | |
| fi | |
| - name: register promptflow model | |
| if: ${{ steps.jobMetricAssert.outputs.result == 'true' }} | |
| run: | | |
| pushd './${{ env.WORKSHOP_PATH }}' | |
| az ml model create --file promptflow/deployment/model.yaml -g ${{env.GROUP}} -w ${{env.WORKSPACE}} | |
| popd |