Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 035c410

Browse files
hiro-vhiro-v
committed
feat: Add CI job for api coverage
1 parent 9b2c8bb commit 035c410

File tree

5 files changed

+249
-0
lines changed

5 files changed

+249
-0
lines changed

‎.github/workflows/cortex-js-openai-coverage.yml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: Test - OpenAI API Coverage - Nightly / Manual
2+
on:
3+
schedule:
4+
- cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday
5+
workflow_dispatch:
6+
inputs:
7+
endpoints:
8+
description: 'comma-separated list (see available at coverage/endpoint_mapping.json e.g. GET /users,POST /transform)'
9+
required: false
10+
default: all
11+
type: string
12+
push:
13+
branches:
14+
- feat/cortex_openai_coverage
15+
16+
env:
17+
OPENAI_API_PYTHON_TAG: v1.23.2
18+
19+
jobs:
20+
openai-python-tests:
21+
runs-on: [self-hosted, Linux, ubuntu-desktop]
22+
steps:
23+
- name: Getting the repo
24+
uses: actions/checkout@v3
25+
with:
26+
fetch-depth: 0
27+
28+
- name: Installing node
29+
uses: actions/setup-node@v3
30+
with:
31+
node-version: 20
32+
33+
- name: "Cleanup cache"
34+
continue-on-error: true
35+
run: |
36+
npm cache clean --force
37+
38+
- name: Install dependencies
39+
run: |
40+
npm install -g @stoplight/prism-cli
41+
42+
- run: yarn install && yarn build
43+
working-directory: ./cortex-js
44+
45+
- run:
46+
|
47+
node cortex-js/dist/src/command.js serve --host 127.0.0.1 --port 4010 > cortex.log & cortex_pid=$!
48+
sleep 1
49+
wget -O api.json http://127.0.0.1:4010/api-json
50+
kill $cortex_pid
51+
52+
- name: Create python virtual environment and run test
53+
run: |
54+
python3 -m venv /tmp/jan
55+
source /tmp/jan/bin/activate
56+
57+
# Clone openai-api-python repo
58+
git clone https://github.com/openai/openai-python.git -b $OPENAI_API_PYTHON_TAG
59+
cd openai-python
60+
61+
pip install -r requirements-dev.lock
62+
pip install pytest-reportportal pytest-html
63+
64+
# Create pytest.ini file with content
65+
cp ../coverage/pytest.ini pytest.ini
66+
echo "rp_api_key=${{ secrets.RP_API_KEY }}" >> pytest.ini
67+
echo "rp_endpoint=${{ secrets.RP_ENDPOINT }}" >> pytest.ini
68+
69+
# Append to conftest.py
70+
cp ../coverage/conftest.py tests/conftest.py
71+
cp ../coverage/endpoint_mapping.json tests/endpoints_mapping.json
72+
73+
prism mock ../api.json > prism.log & prism_pid=$!
74+
# pytest --endpoint "$ENDPOINTS" --reportportal --html=report.html && kill $prism_pid
75+
pytest --endpoint all --reportportal --html=report.html && kill $prism_pid
76+
deactivate
77+
# env:
78+
# ENDPOINTS: ${{ github.event.inputs.endpoints }}
79+
80+
81+
- name: Collect RP artifacts
82+
run: |
83+
wget -O total-coverage.json "${{ secrets.RP_ENDPOINT }}/api/v1/openai-api-test/widget/27" --header 'authorization: bearer ${{ secrets.RP_API_KEY }}'
84+
wget -O today-endpoint.json "${{ secrets.RP_ENDPOINT }}/api/v1/openai-api-test/widget/multilevel/32" --header 'authorization: bearer ${{ secrets.RP_API_KEY }}'
85+
cat total-coverage.json
86+
current_date=$(date +"%m-%d-%Y")
87+
cp today-endpoint.json $current_date.json
88+
cat $current_date.json
89+
90+
- name: Upload report json files to S3
91+
run: |
92+
current_date=$(date +"%m-%d-%Y")
93+
aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "openai-api-collection-test/$current_date.json" --body "./$current_date.json" --content-type "application/json"
94+
aws s3api put-object --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --key "openai-api-collection-test/total-coverage.json" --body "./total-coverage.json" --content-type "application/json"
95+
env:
96+
AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
97+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
98+
AWS_DEFAULT_REGION: auto
99+
AWS_EC2_METADATA_DISABLED: "true"
100+
101+
- name: Upload Artifact
102+
uses: actions/upload-artifact@v2
103+
with:
104+
name: report
105+
path: |
106+
openai-python/report.html
107+
openai-python/prism.log
108+
openai-python/cortex.log
109+
openai-python/total-coverage.json
110+
openai-python/today-endpoint.json
111+
112+
- name: Clean up
113+
if: always()
114+
run: |
115+
rm -rf /tmp/jan
116+
rm -rf openai-python
117+
rm -rf report.html
118+
rm -rf report.zip
119+

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,10 @@ dist
44
*.lock
55
node_modules
66
.turbo
7+
8+
# CI - Test - Coverage
9+
cortex.log
10+
api.log
11+
prism.log
12+
api.json
13+
openai-python/*

‎coverage/conftest.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import json
2+
3+
4+
def pytest_addoption(parser):
5+
parser.addoption(
6+
"--endpoint", action="store", default="all", help="my option: endpoints"
7+
)
8+
9+
10+
def pytest_configure(config):
11+
config.addinivalue_line(
12+
"markers", "endpoint(endpoint): this mark select the test based on endpoint"
13+
)
14+
15+
16+
def pytest_runtest_setup(item):
17+
getoption = item.config.getoption("--endpoint").split(",")
18+
if getoption not in (["all"], [''], [""]):
19+
endpoint_names = [mark.args[0] for mark in item.iter_markers(name="endpoint")]
20+
if not endpoint_names or not set(getoption).intersection(set(endpoint_names)):
21+
pytest.skip("Test skipped because endpoint is {!r}".format(endpoint_names))
22+
23+
24+
def pytest_collection_modifyitems(items):
25+
# load the JSON file
26+
with open("tests/endpoints_mapping.json", "r") as json_file:
27+
endpoints_file_mapping = json.load(json_file)
28+
29+
# create a dictionary to map filenames to endpoints
30+
filename_to_endpoint = {}
31+
for endpoint, files in endpoints_file_mapping.items():
32+
for filename in files:
33+
filename_to_endpoint[filename] = endpoint
34+
35+
# add the markers based on the JSON file
36+
for item in items:
37+
# map the name of the file to endpoint, else use default value
38+
filename = item.fspath.basename
39+
marker = filename_to_endpoint.get(filename, filename)
40+
item.add_marker(pytest.mark.endpoint(marker, filename=filename))

‎coverage/endpoint_mapping.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"/embeddings": [
3+
"test_embedding.py"
4+
],
5+
"/audio/translations": [
6+
"test_translations.py"
7+
],
8+
"/audio/transcriptions": [
9+
"test_transcriptions.py"
10+
],
11+
"/moderations": [
12+
"test_moderations.py"
13+
],
14+
"/images/generations": [
15+
"test_images.py"
16+
],
17+
"/batches": [
18+
"test_batches.py"
19+
],
20+
"/vector_stores": [
21+
"test_vector_stores.py"
22+
],
23+
"/fine_tuning/jobs": [
24+
"test_jobs.py",
25+
"test_checkpoints.py"
26+
],
27+
"/assistants": [
28+
"test_assistants.py"
29+
],
30+
"/threads/{thread_id}/runs": [
31+
"test_runs.py"
32+
],
33+
"/threads/{thread_id}/runs/{run_id}/steps": [
34+
"test_steps.py"
35+
],
36+
"/vector_stores/{vector_store_id}/file_batches": [
37+
"test_file_batches.py"
38+
],
39+
"/messages": [
40+
"test_messages.py"
41+
],
42+
"/vector_stores/{vector_store_id}/files": [
43+
"test_files.py"
44+
],
45+
"/chat/completions": [
46+
"test_completions.py"
47+
],
48+
"/threads": [
49+
"test_threads.py"
50+
],
51+
"/audio/speech": [
52+
"test_speech.py"
53+
],
54+
"/models": [
55+
"test_models.py"
56+
],
57+
"native_client_sdk_only": [
58+
"test_streaming.py"
59+
],
60+
"utils": [
61+
"test_response.py",
62+
"test_client.py",
63+
"test_extract_files.py",
64+
"test_typing.py",
65+
"test_legacy_response.py",
66+
"test_module_client.py",
67+
"test_old_api.py",
68+
"test_proxy.py",
69+
"test_qs.py",
70+
"test_required_args.py",
71+
"test_transform.py",
72+
"test_azure.py",
73+
"test_deepcopy.py"
74+
]
75+
}

‎coverage/pytest.ini‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[pytest]
2+
rp_project = openai-api-test
3+
rp_launch = OpenAI Collection Test
4+
rp_launch_description = Full collection to ensure compatibility with OpenAI API
5+
rp_launch_attributes = 'CI'
6+
filterwarnings = ignore::pytest.PytestUnknownMarkWarning
7+
log_format = %(asctime)s %(levelname)s %(message)s
8+
log_date_format = %Y-%m-%d %H:%M:%S

0 commit comments

Comments
 (0)