-
Notifications
You must be signed in to change notification settings - Fork 827
123 lines (119 loc) · 4.56 KB
/
promptflow-sdk-cli-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
name: promptflow-sdk-cli-test
on:
schedule:
- cron: "40 18 * * *" # Every day starting at 2:40 BJT
pull_request:
paths:
- src/promptflow-core/**
- src/promptflow-devkit/**
- src/promptflow/**
- src/promptflow-tracing/**
- scripts/building/**
- .github/workflows/promptflow-sdk-cli-test.yml
- src/promptflow-recording/**
workflow_dispatch:
inputs:
filepath:
description: file or paths you want to trigger a test
required: true
default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
type: string
permissions:
id-token: write
contents: read
env:
IS_IN_CI_PIPELINE: "true"
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
jobs:
sdk_cli_tests:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
pythonVersion: ['3.8', '3.9', '3.10', '3.11']
environment:
internal
runs-on: ${{ matrix.os }}
steps:
- name: set test mode
run: |
echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.pythonVersion }}
- uses: snok/install-poetry@v1
- name: install test dependency group
run: |
set -xe
poetry install -E pyarrow --with ci,test
poetry run pip show promptflow-tracing
poetry run pip show promptflow-core
poetry run pip show promptflow-devkit
poetry run pip show promptflow-tools
working-directory: ${{ env.WORKING_DIRECTORY }}
- name: Azure login (non pull_request workflow)
if: github.event_name != 'pull_request'
uses: azure/login@v1
with:
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
tenant-id: ${{secrets.AZURE_TENANT_ID}}
client-id: ${{secrets.AZURE_CLIENT_ID}}
- name: generate live test resources (non pull_request workflow)
if: github.event_name != 'pull_request'
uses: "./.github/actions/step_generate_configs"
with:
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
- name: generate live test resources (pull_request workflow)
if: github.event_name == 'pull_request'
working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
run: |
cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
- name: run devkit tests
run: |
poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
--cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
--ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
working-directory: ${{ env.WORKING_DIRECTORY }}
- name: Upload Test Results
if: always()
uses: actions/upload-artifact@v3
with:
name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
path: |
${{ env.WORKING_DIRECTORY }}/*.xml
${{ env.WORKING_DIRECTORY }}/htmlcov/
${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
- run: poetry install -E executable
working-directory: ${{ env.WORKING_DIRECTORY }}
- name: run devkit executable tests
run: |
poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py
working-directory: ${{ env.WORKING_DIRECTORY }}
publish-test-results-sdk-cli-test:
needs: sdk_cli_tests
runs-on: ubuntu-latest
permissions:
checks: write
pull-requests: write
contents: read
issues: read
if: always()
steps:
- name: checkout
uses: actions/checkout@v4
- name: Publish Test Results
uses: "./.github/actions/step_publish_test_results"
with:
testActionFileName: promptflow-sdk-cli-test.yml
testResultTitle: SDK CLI Test Result
osVersion: ubuntu-latest
pythonVersion: 3.9
coverageThreshold: 40
context: test/sdk_cli