Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit cd63b89

Browse files
vansangpfievvansangpfiev
authored
feat: python runtime engine (#559)
1 parent 5603112 commit cd63b89

File tree

15 files changed

+561
-82
lines changed

15 files changed

+561
-82
lines changed

‎.github/scripts/e2e-test-python-linux-and-mac.sh‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/bin/bash
2+
3+
## Example run command
4+
# ./e2e-test-python-linux-and-mac.sh '../../examples/build/server' './e2e-test.py'
5+
6+
# Check for required arguments
7+
if [[ $# -ne 2 ]]; then
8+
echo "Usage: $0 <path_to_binary> <path_to_python_file>"
9+
exit 1
10+
fi
11+
12+
BINARY_PATH=$1
13+
PYTHON_FILE_EXECUTION_PATH=$2
14+
15+
rm /tmp/python-file-execution-res.log /tmp/server.log
16+
17+
# Random port to ensure it's not used
18+
min=10000
19+
max=11000
20+
range=$((max - min + 1))
21+
PORT=$((RANDOM % range + min))
22+
23+
# Install numpy for Python
24+
export PYTHONHOME=$(pwd)/engines/cortex.python/python/
25+
export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH"
26+
export DYLD_FALLBACK_LIBRARY_PATH="$PYTHONHOME:$DYLD_FALLBACK_LIBRARY_PATH"
27+
echo "Set Python HOME to $PYTHONHOME"
28+
echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
29+
./engines/cortex.python/python/bin/python3 -m ensurepip
30+
./engines/cortex.python/python/bin/python3 -m pip install --upgrade pip
31+
./engines/cortex.python/python/bin/python3 -m pip install numpy --target=$PYTHONHOME/lib/python/site-packages/
32+
33+
# Start the binary file
34+
"$BINARY_PATH" 1 127.0.0.1 $PORT >/tmp/server.log &
35+
36+
pid=$!
37+
38+
if ! ps -p $pid >/dev/null; then
39+
echo "server failed to start. Logs:"
40+
cat /tmp/server.log
41+
exit 1
42+
fi
43+
44+
# Wait for a few seconds to let the server start
45+
sleep 3
46+
47+
# Run the curl commands
48+
response1=$(curl --connect-timeout 60 -o /tmp/python-file-execution-res.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/v1/fine_tuning/job" \
49+
--header 'Content-Type: application/json' \
50+
--data '{
51+
"file_execution_path": "'$PYTHON_FILE_EXECUTION_PATH'"
52+
}')
53+
54+
error_occurred=0
55+
56+
# Verify the response
57+
if [[ "$response1" -ne 200 ]]; then
58+
echo "The python file execution curl command failed with status code: $response1"
59+
cat /tmp/python-file-execution-res.log
60+
error_occurred=1
61+
fi
62+
63+
# Verify the output of the Python file in output.txt
64+
OUTPUT_FILE="./output.txt"
65+
EXPECTED_OUTPUT="1 2 3" # Replace with the expected content
66+
67+
if [[ -f "$OUTPUT_FILE" ]]; then
68+
actual_output=$(cat "$OUTPUT_FILE")
69+
if [[ "$actual_output" != "$EXPECTED_OUTPUT" ]]; then
70+
echo "The output of the Python file does not match the expected output."
71+
echo "Expected: $EXPECTED_OUTPUT"
72+
echo "Actual: $actual_output"
73+
error_occurred=1
74+
else
75+
echo "The output of the Python file matches the expected output."
76+
fi
77+
else
78+
echo "Output file $OUTPUT_FILE does not exist."
79+
error_occurred=1
80+
fi
81+
82+
83+
if [[ "$error_occurred" -eq 1 ]]; then
84+
echo "Server test run failed!!!!!!!!!!!!!!!!!!!!!!"
85+
echo "Server Error Logs:"
86+
cat /tmp/server.log
87+
kill $pid
88+
echo "An error occurred while running the server."
89+
exit 1
90+
fi
91+
92+
echo "----------------------"
93+
echo "Log server:"
94+
cat /tmp/server.log
95+
96+
echo "Server test run successfully!"
97+
98+
# Kill the server process
99+
kill $pid

‎.github/scripts/e2e-test-python-windows.bat‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
@echo off
2+
3+
setlocal enabledelayedexpansion
4+
5+
set "TEMP=C:\Users\%UserName%\AppData\Local\Temp"
6+
7+
rem Check for required arguments
8+
if "%~2"=="" (
9+
echo Usage: %~0 ^<path_to_binary^> ^<path_to_python_file^>
10+
exit /b 1
11+
)
12+
13+
set "BINARY_PATH=%~1"
14+
set "PYTHON_FILE_EXECUTION_PATH=%~2"
15+
16+
for %%i in ("%BINARY_PATH%") do set "BINARY_NAME=%%~nxi"
17+
18+
echo BINARY_NAME=%BINARY_NAME%
19+
20+
del %TEMP%\response1.log 2>nul
21+
del %TEMP%\server.log 2>nul
22+
23+
set /a min=9999
24+
set /a max=11000
25+
set /a range=max-min+1
26+
set /a PORT=%min% + %RANDOM% %% %range%
27+
28+
rem Install numpy for Python
29+
set "PYTHONHOME=%cd%\engines\cortex.python\python"
30+
echo Set Python HOME to %PYTHONHOME%
31+
%PYTHONHOME%\python.exe -m ensurepip
32+
%PYTHONHOME%\python.exe -m pip install --upgrade pip
33+
%PYTHONHOME%\python.exe -m pip install numpy --target=%PYTHONHOME%\Lib\site-packages\
34+
35+
rem Start the binary file
36+
start "" /B "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > "%TEMP%\server.log" 2>&1
37+
38+
ping -n 3 127.0.0.1 > nul
39+
40+
rem Capture the PID of the started process with "server" in its name
41+
for /f "tokens=2" %%a in ('tasklist /fi "imagename eq %BINARY_NAME%" /fo list ^| findstr /B "PID:"') do (
42+
set "pid=%%a"
43+
)
44+
45+
echo pid=%pid%
46+
47+
if not defined pid (
48+
echo server failed to start. Logs:
49+
type %TEMP%\server.log
50+
echo.
51+
exit /b 1
52+
)
53+
54+
rem Wait for a few seconds to let the server start
55+
56+
rem Define JSON strings for curl data
57+
call set "PYTHON_FILE_EXECUTION_PATH_STRING=%%PYTHON_FILE_EXECUTION_PATH:\=\\%%"
58+
set "curl_data1={\"file_execution_path\":\"%PYTHON_FILE_EXECUTION_PATH_STRING%\"}"
59+
60+
rem Print the values of curl_data for debugging
61+
echo curl_data1=%curl_data1%
62+
63+
rem Run the curl commands and capture the status code
64+
curl.exe --connect-timeout 60 -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/fine_tuning/job" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1
65+
66+
set "error_occurred=0"
67+
68+
rem Read the status code directly from the response file
69+
set "response1="
70+
for /f %%a in (%TEMP%\response1.log) do set "response1=%%a"
71+
72+
if "%response1%" neq "200" (
73+
echo The first curl command failed with status code: %response1%
74+
type %TEMP%\response1.log
75+
echo.
76+
set "error_occurred=1"
77+
)
78+
79+
echo ----------------------
80+
echo Log python file execution:
81+
type %TEMP%\response1.log
82+
echo.
83+
84+
rem Verification step: Check the contents of output.txt
85+
set "expected_output=1 2 3"
86+
set "actual_output="
87+
if exist "output.txt" (
88+
for /f "delims=" %%x in (output.txt) do set "actual_output=%%x"
89+
if "!actual_output!"=="!expected_output!" (
90+
echo Verification succeeded: output.txt contains the expected data.
91+
) else (
92+
echo Verification failed: output.txt does not contain the expected data.
93+
echo Expected: !expected_output!
94+
echo Actual: !actual_output!
95+
set "error_occurred=1"
96+
)
97+
) else (
98+
echo Verification failed: output.txt does not exist.
99+
set "error_occurred=1"
100+
)
101+
102+
echo ----------------------
103+
echo Server logs:
104+
type %TEMP%\server.log
105+
echo.
106+
107+
if "%error_occurred%"=="1" (
108+
echo Server test run failed!!!!!!!!!!!!!!!!!!!!!!
109+
taskkill /f /pid %pid%
110+
echo An error occurred while running the server.
111+
exit /b 1
112+
)
113+
114+
echo Server test run successfully!
115+
116+
rem Kill the server process
117+
taskkill /f /im server.exe 2>nul || exit /B 0
118+
119+
endlocal

‎.github/scripts/python-file-to-test.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import sys;
2+
for path in sys.path:
3+
print(path)
4+
5+
import numpy as np
6+
print("Numpy version: " + np.__version__)
7+
8+
with open('output.txt', 'w') as file:
9+
file.write(' '.join(map(str, np.array([1, 2, 3]))))

‎.github/workflows/cortex-cpp-quality-gate.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ on:
1212
env:
1313
LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
1414
EMBEDDING_MODEL_URL: https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf
15+
PYTHON_FILE_EXECUTION_PATH: "python-file-to-test.py"
1516

1617
jobs:
1718
build-and-test:
@@ -26,107 +27,126 @@ jobs:
2627
runs-on: "ubuntu-18-04"
2728
cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF"
2829
run-e2e: true
30+
run-python-e2e: true
2931

3032
- os: "linux"
3133
name: "amd64-avx"
3234
runs-on: "ubuntu-18-04"
3335
cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF"
3436
run-e2e: false
37+
run-python-e2e: false
3538

3639
- os: "linux"
3740
name: "amd64-avx512"
3841
runs-on: "ubuntu-18-04"
3942
cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF"
4043
run-e2e: false
44+
run-python-e2e: false
4145

4246
- os: "linux"
4347
name: "amd64-vulkan"
4448
runs-on: "ubuntu-18-04-cuda-11-7"
4549
cmake-flags: "-DLLAMA_VULKAN=ON -DLLAMA_NATIVE=OFF"
4650
run-e2e: false
51+
run-python-e2e: false
4752

4853
- os: "linux"
4954
name: "amd64-cuda-11-7"
5055
runs-on: "ubuntu-18-04-cuda-11-7"
5156
cmake-flags: "-DCUDA_11_7=ON -DLLAMA_NATIVE=OFF -DLLAMA_CUDA=ON"
5257
run-e2e: false
58+
run-python-e2e: false
5359

5460
- os: "linux"
5561
name: "amd64-cuda-12-0"
5662
runs-on: "ubuntu-18-04-cuda-12-0"
5763
cmake-flags: "-DCUDA_12_0=ON -DLLAMA_NATIVE=OFF -DLLAMA_CUDA=ON"
5864
run-e2e: false
65+
run-python-e2e: false
5966

6067
- os: "mac"
6168
name: "amd64"
6269
runs-on: "macos-13"
6370
cmake-flags: ""
6471
run-e2e: true
72+
run-python-e2e: true
6573

6674
- os: "mac"
6775
name: "arm64"
6876
runs-on: "mac-silicon"
6977
cmake-flags: "-DMAC_ARM64=ON"
7078
run-e2e: true
79+
run-python-e2e: true
7180

7281
- os: "windows"
7382
name: "amd64-avx2"
7483
runs-on: "windows-latest"
7584
cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
7685
run-e2e: true
86+
run-python-e2e: true
7787

7888
- os: "windows"
7989
name: "amd64-avx"
8090
runs-on: "windows-latest"
8191
cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
8292
run-e2e: false
93+
run-python-e2e: false
8394

8495
- os: "windows"
8596
name: "amd64-avx512"
8697
runs-on: "windows-latest"
8798
cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
8899
run-e2e: false
100+
run-python-e2e: false
89101

90102
- os: "windows"
91103
name: "amd64-vulkan"
92104
runs-on: "windows-latest"
93105
cmake-flags: "-DLLAMA_VULKAN=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
94106
run-e2e: false
107+
run-python-e2e: false
95108

96109
- os: "windows"
97110
name: "amd64-avx2-cuda-12-0"
98111
runs-on: "windows-cuda-12-0"
99112
cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
100113
run-e2e: false
114+
run-python-e2e: false
101115

102116
- os: "windows"
103117
name: "amd64-avx-cuda-12-0"
104118
runs-on: "windows-cuda-12-0"
105119
cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
106120
run-e2e: false
121+
run-python-e2e: false
107122

108123
- os: "windows"
109124
name: "amd64-avx512-cuda-12-0"
110125
runs-on: "windows-cuda-12-0"
111126
cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
112127
run-e2e: false
128+
run-python-e2e: false
113129

114130
- os: "windows"
115131
name: "amd64-avx2-cuda-11-7"
116132
runs-on: "windows-cuda-11-7"
117133
cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
118134
run-e2e: false
135+
run-python-e2e: false
119136

120137
- os: "windows"
121138
name: "amd64-avx-cuda-11-7"
122139
runs-on: "windows-cuda-11-7"
123140
cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
124141
run-e2e: false
142+
run-python-e2e: false
143+
125144
- os: "windows"
126145
name: "amd64-avx512-cuda-11-7"
127146
runs-on: "windows-cuda-11-7"
128147
cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE"
129148
run-e2e: false
149+
run-python-e2e: false
130150

131151
steps:
132152
- name: Clone
@@ -161,6 +181,12 @@ jobs:
161181
cd cortex-cpp
162182
make run-e2e-test RUN_TESTS=true LLM_MODEL_URL=${{ env.LLM_MODEL_URL }} EMBEDDING_MODEL_URL=${{ env.EMBEDDING_MODEL_URL }}
163183
184+
- name: Run python e2e testing
185+
if: ${{ matrix.run-python-e2e }}
186+
run: |
187+
cd cortex-cpp
188+
make run-python-e2e-test RUN_TESTS=true PYTHON_FILE_EXECUTION_PATH=${{ env.PYTHON_FILE_EXECUTION_PATH }}
189+
164190
- name: Upload Artifact
165191
uses: actions/upload-artifact@v2
166192
with:

‎cortex-cpp/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.5)
22
project(cortex-cpp C CXX)
33

44
include(engines/cortex.llamacpp/engine.cmake)
5+
if(NOT LLAMA_CUDA AND (LLAMA_AVX2 OR APPLE))
6+
include(engines/cortex.python/engine.cmake)
7+
endif()
58
include(CheckIncludeFileCXX)
69

710
check_include_file_cxx(any HAS_ANY)

0 commit comments

Comments
 (0)