Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 352afa4

Browse files
hiro-vhiro-v
committed
feat: Add openai coverage test for nightly
1 parent 9b2c8bb commit 352afa4

File tree

5 files changed

+255
-0
lines changed

5 files changed

+255
-0
lines changed

‎.github/workflows/cortex-js-openai-coverage.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Test - OpenAI API Coverage - Nightly / Manual
2+
on:
3+
schedule:
4+
- cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday
5+
workflow_dispatch:
6+
inputs:
7+
endpoints:
8+
description: 'comma-separated list (see available at coverage/endpoint_mapping.json e.g. GET /users,POST /transform)'
9+
required: false
10+
default: all
11+
type: string
12+
13+
branch:
14+
description: 'Target github branch - default is dev'
15+
required: false
16+
default: dev
17+
type: string
18+
19+
env:
20+
OPENAI_API_PYTHON_TAG: v1.23.2
21+
TARGET_BRANCH: ${{ github.event.inputs.branch }}
22+
23+
jobs:
24+
openai-python-tests:
25+
runs-on: [self-hosted, Linux, ubuntu-desktop]
26+
steps:
27+
- name: Getting the repo
28+
uses: actions/checkout@v4
29+
with:
30+
fetch-depth: 0
31+
ref: ${{ env.TARGET_BRANCH }}
32+
33+
- name: Installing node
34+
uses: actions/setup-node@v4
35+
with:
36+
node-version: 20
37+
38+
- name: "Cleanup cache"
39+
continue-on-error: true
40+
run: |
41+
npm cache clean --force
42+
43+
- name: Install dependencies
44+
run: |
45+
npm install -g @stoplight/prism-cli
46+
47+
- run: yarn install && yarn build
48+
working-directory: ./cortex-js
49+
50+
- name: Run and extract openapi.json
51+
run:
52+
|
53+
node cortex-js/dist/src/command.js serve --host 127.0.0.1 --port 4010 > cortex.log & cortex_pid=$!
54+
sleep 3
55+
wget --no-verbose -O api.json http://127.0.0.1:4010/api-json
56+
kill $cortex_pid
57+
58+
- name: Create python virtual environment and run test
59+
run: |
60+
python3 -m venv /tmp/jan
61+
source /tmp/jan/bin/activate
62+
63+
# Clone openai-api-python repo
64+
git clone https://github.com/openai/openai-python.git -b $OPENAI_API_PYTHON_TAG
65+
cd openai-python
66+
67+
pip install -r requirements-dev.lock
68+
pip install pytest-reportportal pytest-html
69+
70+
# Create pytest.ini file with content
71+
mv ../coverage/pytest.ini pytest.ini
72+
echo "rp_api_key=${{ secrets.RP_API_KEY }}" >> pytest.ini
73+
echo "rp_endpoint=${{ secrets.RP_ENDPOINT }}" >> pytest.ini
74+
75+
# Append to conftest.py
76+
mv ../coverage/conftest.py tests/conftest.py
77+
mv ../coverage/endpoint_mapping.json tests/endpoints_mapping.json
78+
79+
prism mock ../api.json > prism.log & prism_pid=$!
80+
pytest --endpoint "$ENDPOINTS" --reportportal --html=report.html && kill $prism_pid
81+
deactivate
82+
env:
83+
ENDPOINTS: ${{ github.event.inputs.endpoints }}
84+
85+
86+
- name: Collect RP artifacts
87+
run: |
88+
wget --no-verbose -O total-coverage.json "${{ secrets.RP_ENDPOINT }}/api/v1/openai-api-test/widget/27" --header 'authorization: bearer ${{ secrets.RP_API_KEY }}'
89+
wget --no-verbose -O today-endpoint.json "${{ secrets.RP_ENDPOINT }}/api/v1/openai-api-test/widget/multilevel/32" --header 'authorization: bearer ${{ secrets.RP_API_KEY }}'
90+
cat total-coverage.json
91+
current_date=$(date +"%m-%d-%Y")
92+
cp today-endpoint.json $current_date.json
93+
cat $current_date.json
94+
95+
- name: Upload report json files to S3
96+
run: |
97+
current_date=$(date +"%m-%d-%Y")
98+
aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "openai-api-collection-test/$current_date.json" --body "./$current_date.json" --content-type "application/json"
99+
aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "openai-api-collection-test/total-coverage.json" --body "./total-coverage.json" --content-type "application/json"
100+
env:
101+
AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
102+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
103+
AWS_DEFAULT_REGION: auto
104+
AWS_EC2_METADATA_DISABLED: "true"
105+
106+
- name: Upload Artifact
107+
uses: actions/upload-artifact@v2
108+
with:
109+
name: report
110+
path: |
111+
openai-python/report.html
112+
openai-python/prism.log
113+
openai-python/cortex.log
114+
openai-python/total-coverage.json
115+
openai-python/today-endpoint.json
116+
api.json
117+
118+
- name: Clean up
119+
if: always()
120+
run: |
121+
rm -rf /tmp/jan
122+
rm -rf openai-python
123+
rm -rf report.html
124+
rm -rf report.zip
125+

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,10 @@ dist
44
*.lock
55
node_modules
66
.turbo
7+
8+
# CI - Test - Coverage
9+
cortex.log
10+
api.log
11+
prism.log
12+
api.json
13+
openai-python/*

‎coverage/conftest.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import json
2+
3+
4+
def pytest_addoption(parser):
5+
parser.addoption(
6+
"--endpoint", action="store", default="all", help="my option: endpoints"
7+
)
8+
9+
10+
def pytest_configure(config):
11+
config.addinivalue_line(
12+
"markers", "endpoint(endpoint): this mark select the test based on endpoint"
13+
)
14+
15+
16+
def pytest_runtest_setup(item):
17+
getoption = item.config.getoption("--endpoint").split(",")
18+
if getoption not in (["all"], [''], [""]):
19+
endpoint_names = [mark.args[0] for mark in item.iter_markers(name="endpoint")]
20+
if not endpoint_names or not set(getoption).intersection(set(endpoint_names)):
21+
pytest.skip("Test skipped because endpoint is {!r}".format(endpoint_names))
22+
23+
24+
def pytest_collection_modifyitems(items):
25+
# load the JSON file
26+
with open("tests/endpoints_mapping.json", "r") as json_file:
27+
endpoints_file_mapping = json.load(json_file)
28+
29+
# create a dictionary to map filenames to endpoints
30+
filename_to_endpoint = {}
31+
for endpoint, files in endpoints_file_mapping.items():
32+
for filename in files:
33+
filename_to_endpoint[filename] = endpoint
34+
35+
# add the markers based on the JSON file
36+
for item in items:
37+
# map the name of the file to endpoint, else use default value
38+
filename = item.fspath.basename
39+
marker = filename_to_endpoint.get(filename, filename)
40+
item.add_marker(pytest.mark.endpoint(marker, filename=filename))

‎coverage/endpoint_mapping.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"/embeddings": [
3+
"test_embedding.py"
4+
],
5+
"/audio/translations": [
6+
"test_translations.py"
7+
],
8+
"/audio/transcriptions": [
9+
"test_transcriptions.py"
10+
],
11+
"/moderations": [
12+
"test_moderations.py"
13+
],
14+
"/images/generations": [
15+
"test_images.py"
16+
],
17+
"/batches": [
18+
"test_batches.py"
19+
],
20+
"/vector_stores": [
21+
"test_vector_stores.py"
22+
],
23+
"/fine_tuning/jobs": [
24+
"test_jobs.py",
25+
"test_checkpoints.py"
26+
],
27+
"/assistants": [
28+
"test_assistants.py"
29+
],
30+
"/threads/{thread_id}/runs": [
31+
"test_runs.py"
32+
],
33+
"/threads/{thread_id}/runs/{run_id}/steps": [
34+
"test_steps.py"
35+
],
36+
"/vector_stores/{vector_store_id}/file_batches": [
37+
"test_file_batches.py"
38+
],
39+
"/messages": [
40+
"test_messages.py"
41+
],
42+
"/vector_stores/{vector_store_id}/files": [
43+
"test_files.py"
44+
],
45+
"/chat/completions": [
46+
"test_completions.py"
47+
],
48+
"/threads": [
49+
"test_threads.py"
50+
],
51+
"/audio/speech": [
52+
"test_speech.py"
53+
],
54+
"/models": [
55+
"test_models.py"
56+
],
57+
"native_client_sdk_only": [
58+
"test_streaming.py"
59+
],
60+
"utils": [
61+
"test_response.py",
62+
"test_client.py",
63+
"test_extract_files.py",
64+
"test_typing.py",
65+
"test_legacy_response.py",
66+
"test_module_client.py",
67+
"test_old_api.py",
68+
"test_proxy.py",
69+
"test_qs.py",
70+
"test_required_args.py",
71+
"test_transform.py",
72+
"test_azure.py",
73+
"test_deepcopy.py"
74+
]
75+
}

‎coverage/pytest.ini‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[pytest]
2+
rp_project = openai-api-test
3+
rp_launch = OpenAI Collection Test
4+
rp_launch_description = Full collection to ensure compatibility with OpenAI API
5+
rp_launch_attributes = 'CI'
6+
filterwarnings = ignore::pytest.PytestUnknownMarkWarning
7+
log_format = %(asctime)s %(levelname)s %(message)s
8+
log_date_format = %Y-%m-%d %H:%M:%S

0 commit comments

Comments
 (0)