-
Notifications
You must be signed in to change notification settings - Fork 254
146 lines (120 loc) · 4.82 KB
/
llmopsrunevalpf.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
name: LLMOps with PromptFlow
on:
workflow_dispatch:
push:
branches: [ main ]
env:
GROUP: ${{secrets.GROUP}}
WORKSPACE: ${{secrets.WORKSPACE}}
SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
RUN_NAME: llmopsqa
EVAL_RUN_NAME: llmopsqa_eval
WORKSHOP_PATH: 'Workshop'
PYTHON_VERSION: '3.11'
jobs:
login-and-run-and-evalpf:
runs-on: ubuntu-latest
steps:
- name: check out repo
uses: actions/checkout@v2
- name: install az ml extension
run: az extension add -n ml -y
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_RBAC_CREDENTIALS}}
- name: Set up Python version
uses: actions/setup-python@v3.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Create and start virtual environment
run: |
python -m venv venv
source venv/bin/activate
- name: 'Resolve Project Dependencies Using Pip'
shell: bash
run: |
pushd './${{ env.WORKSHOP_PATH }}'
python -m pip install --upgrade pip
pip install -r requirements.txt
popd
- name: run promptflow
run: |
pushd './${{ env.WORKSHOP_PATH }}'
pfazure run create -f promptflow/llmopsqa/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/run_info.txt
cat promptflow/llmops-helper/run_info.txt
popd
- name: set run name
run: |
pushd './${{ env.WORKSHOP_PATH }}'
echo "RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV"
popd
- name: show the current run name
run: |
pushd './${{ env.WORKSHOP_PATH }}'
echo "Run name is:" ${{env.RUN_NAME}}
popd
- name: show promptflow results
run: |
pushd './${{ env.WORKSHOP_PATH }}'
pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
popd
# - name: run promptflow evaluations
# run: pfazure run create -f promptflow/llmopsqa/run_evaluation.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/eval_info.txt
# - name: get eval run name
# run: export EVAL_RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py eval_info.txt)
# - name: show promptflow details
# run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
# - name: show promptflow metrics
# run: pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > promptflow/llmops-helper/eval_result.json
assert-and-register-model:
needs: login-and-run-and-evalpf
runs-on: ubuntu-latest
steps:
- name: check out repo
uses: actions/checkout@v2
- name: install az ml extension
run: az extension add -n ml -y
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_RBAC_CREDENTIALS}}
- name: Set up Python version
uses: actions/setup-python@v3.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Create and start virtual environment
run: |
python -m venv venv
source venv/bin/activate
- name: 'Resolve Project Dependencies Using Pip'
shell: bash
run: |
pushd './${{ env.WORKSHOP_PATH }}'
python -m pip install --upgrade pip
pip install -r requirements.txt
popd
- name: set default subscription
run: |
az account set -s ${{env.SUBSCRIPTION}}
- name: get assert eval results
id: jobMetricAssert
run: |
export ASSERT=$(python promptflow/llmops-helper/assert.py result.json 0.6) # NOTE <file>.json is the file name and decimal is the threshold for the assertion
echo "::debug::Assert has returned the following value: $ASSERT"
# assert.py will return True or False, but bash expects lowercase.
if ${ASSERT,,} ; then
echo "::debug::Prompt flow run met the quality bar and can be deployed."
echo "::set-output name=result::true"
else
echo "::warning::Prompt flow run didn't meet quality bar."
echo "::set-output name=result::false"
fi
- name: register promptflow model
if: ${{ steps.jobMetricAssert.outputs.result == 'true' }}
run: |
pushd './${{ env.WORKSHOP_PATH }}'
az ml model create --file promptflow/deployment/model.yaml -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
popd