Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 45103c0

Browse files
namchuainamchuai
committed
feat: download cuda toolkit
Signed-off-by: James <namnh0122@gmail.com>
1 parent 8fdff72 commit 45103c0

File tree

8 files changed

+291
-54
lines changed

8 files changed

+291
-54
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cortex-js
2+
.DS_Store
23
cortex-js/cortex.db
34
dist
45
*.lock
@@ -18,4 +19,4 @@ cortex-js/package-lock.json
1819
.vscode
1920
cortex-js/command
2021
cortex-js/src/infrastructure/commanders/test/test_data
21-
**/vcpkg_installed
22+
**/vcpkg_installed

‎engine/commands/engine_init_cmd.cc‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "utils/archive_utils.h"
88
#include "utils/system_info_utils.h"
99
// clang-format on
10+
#include "utils/cuda_toolkit_utils.h"
1011
#include "utils/engine_matcher_utils.h"
1112

1213
namespace commands {
@@ -103,9 +104,10 @@ bool EngineInitCmd::Exec() const {
103104
.path = path,
104105
}}};
105106

106-
DownloadService().AddDownloadTask(downloadTask, [](const std::string&
107-
absolute_path,
108-
bool unused) {
107+
DownloadService download_service;
108+
download_service.AddDownloadTask(downloadTask, [](const std::string&
109+
absolute_path,
110+
bool unused) {
109111
// try to unzip the downloaded file
110112
std::filesystem::path downloadedEnginePath{absolute_path};
111113
LOG_INFO << "Downloaded engine path: "
@@ -125,6 +127,58 @@ bool EngineInitCmd::Exec() const {
125127
}
126128
LOG_INFO << "Finished!";
127129
});
130+
if (system_info.os == "mac" || engineName_ == "cortex.onnx") {
131+
return false;
132+
}
133+
// download cuda toolkit
134+
const std::string jan_host = "https://catalog.jan.ai";
135+
const std::string cuda_toolkit_file_name = "cuda.tar.gz";
136+
const std::string download_id = "cuda";
137+
138+
auto gpu_driver_version = system_info_utils::GetDriverVersion();
139+
140+
auto cuda_runtime_version =
141+
cuda_toolkit_utils::GetCompatibleCudaToolkitVersion(
142+
gpu_driver_version, system_info.os, engineName_);
143+
144+
std::ostringstream cuda_toolkit_path;
145+
cuda_toolkit_path << "dist/cuda-dependencies/" << 11.7 << "/"
146+
<< system_info.os << "/"
147+
<< cuda_toolkit_file_name;
148+
149+
LOG_DEBUG << "Cuda toolkit download url: " << jan_host
150+
<< cuda_toolkit_path.str();
151+
152+
auto downloadCudaToolkitTask = DownloadTask{
153+
.id = download_id,
154+
.type = DownloadType::CudaToolkit,
155+
.error = std::nullopt,
156+
.items = {DownloadItem{
157+
.id = download_id,
158+
.host = jan_host,
159+
.fileName = cuda_toolkit_file_name,
160+
.type = DownloadType::CudaToolkit,
161+
.path = cuda_toolkit_path.str(),
162+
}},
163+
};
164+
165+
download_service.AddDownloadTask(
166+
downloadCudaToolkitTask,
167+
[](const std::string& absolute_path, bool unused) {
168+
LOG_DEBUG << "Downloaded cuda path: " << absolute_path;
169+
// try to unzip the downloaded file
170+
std::filesystem::path downloaded_path{absolute_path};
171+
172+
archive_utils::ExtractArchive(
173+
absolute_path,
174+
downloaded_path.parent_path().parent_path().string());
175+
176+
try {
177+
std::filesystem::remove(absolute_path);
178+
} catch (std::exception& e) {
179+
LOG_ERROR << "Error removing downloaded file: " << e.what();
180+
}
181+
});
128182

129183
return true;
130184
}

‎engine/main.cc‎

Lines changed: 50 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -22,53 +22,53 @@
2222
#error "Unsupported platform!"
2323
#endif
2424

25-
void RunServer(){
25+
void RunServer() {
2626
// Create logs/ folder and setup log to file
27-
std::filesystem::create_directory(cortex_utils::logs_folder);
28-
trantor::AsyncFileLogger asyncFileLogger;
29-
asyncFileLogger.setFileName(cortex_utils::logs_base_name);
30-
asyncFileLogger.startLogging();
31-
trantor::Logger::setOutputFunction(
32-
[&](const char* msg, const uint64_t len) {
33-
asyncFileLogger.output(msg, len);
34-
},
35-
[&]() { asyncFileLogger.flush(); });
36-
asyncFileLogger.setFileSizeLimit(cortex_utils::log_file_size_limit);
37-
// Number of cortex.cpp threads
38-
// if (argc > 1) {
39-
// thread_num = std::atoi(argv[1]);
40-
// }
27+
std::filesystem::create_directory(cortex_utils::logs_folder);
28+
trantor::AsyncFileLogger asyncFileLogger;
29+
asyncFileLogger.setFileName(cortex_utils::logs_base_name);
30+
asyncFileLogger.startLogging();
31+
trantor::Logger::setOutputFunction(
32+
[&](const char* msg, const uint64_t len) {
33+
asyncFileLogger.output(msg, len);
34+
},
35+
[&]() { asyncFileLogger.flush(); });
36+
asyncFileLogger.setFileSizeLimit(cortex_utils::log_file_size_limit);
37+
// Number of cortex.cpp threads
38+
// if (argc > 1) {
39+
// thread_num = std::atoi(argv[1]);
40+
// }
4141

42-
// // Check for host argument
43-
// if (argc > 2) {
44-
// host = argv[2];
45-
// }
42+
// // Check for host argument
43+
// if (argc > 2) {
44+
// host = argv[2];
45+
// }
4646

47-
// // Check for port argument
48-
// if (argc > 3) {
49-
// port = std::atoi(argv[3]); // Convert string argument to int
50-
// }
51-
int thread_num = 1;
52-
std::string host = "127.0.0.1";
53-
int port = 3928;
47+
// // Check for port argument
48+
// if (argc > 3) {
49+
// port = std::atoi(argv[3]); // Convert string argument to int
50+
// }
51+
int thread_num = 1;
52+
std::string host = "127.0.0.1";
53+
int port = 3928;
5454

55-
int logical_cores = std::thread::hardware_concurrency();
56-
int drogon_thread_num = std::max(thread_num, logical_cores);
57-
// cortex_utils::nitro_logo();
55+
int logical_cores = std::thread::hardware_concurrency();
56+
int drogon_thread_num = std::max(thread_num, logical_cores);
57+
// cortex_utils::nitro_logo();
5858
#ifdef CORTEX_CPP_VERSION
59-
LOG_INFO << "cortex.cpp version: " << CORTEX_CPP_VERSION;
59+
LOG_INFO << "cortex.cpp version: " << CORTEX_CPP_VERSION;
6060
#else
61-
LOG_INFO << "cortex.cpp version: undefined";
61+
LOG_INFO << "cortex.cpp version: undefined";
6262
#endif
6363

64-
LOG_INFO << "Server started, listening at: " << host << ":" << port;
65-
LOG_INFO << "Please load your model";
66-
drogon::app().addListener(host, port);
67-
drogon::app().setThreadNum(drogon_thread_num);
68-
LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
64+
LOG_INFO << "Server started, listening at: " << host << ":" << port;
65+
LOG_INFO << "Please load your model";
66+
drogon::app().addListener(host, port);
67+
drogon::app().setThreadNum(drogon_thread_num);
68+
LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
6969

70-
drogon::app().run();
71-
// return 0;
70+
drogon::app().run();
71+
// return 0;
7272
}
7373

7474
void ForkProcess() {
@@ -80,19 +80,21 @@ void ForkProcess() {
8080
ZeroMemory(&si, sizeof(si));
8181
si.cb = sizeof(si);
8282
ZeroMemory(&pi, sizeof(pi));
83-
std::string cmds = cortex_utils::GetCurrentPath() + "/cortex-cpp.exe --start-server";
83+
std::string cmds =
84+
cortex_utils::GetCurrentPath() + "/cortex-cpp.exe --start-server";
8485
// Create child process
8586
if (!CreateProcess(
8687
NULL, // No module name (use command line)
87-
const_cast<char*>(cmds.c_str()), // Command line (replace with your actual executable)
88-
NULL, // Process handle not inheritable
89-
NULL, // Thread handle not inheritable
90-
FALSE, // Set handle inheritance to FALSE
91-
0, // No creation flags
92-
NULL, // Use parent's environment block
93-
NULL, // Use parent's starting directory
94-
&si, // Pointer to STARTUPINFO structure
95-
&pi)) // Pointer to PROCESS_INFORMATION structure
88+
const_cast<char*>(
89+
cmds.c_str()), // Command line (replace with your actual executable)
90+
NULL, // Process handle not inheritable
91+
NULL, // Thread handle not inheritable
92+
FALSE, // Set handle inheritance to FALSE
93+
0, // No creation flags
94+
NULL, // Use parent's environment block
95+
NULL, // Use parent's starting directory
96+
&si, // Pointer to STARTUPINFO structure
97+
&pi)) // Pointer to PROCESS_INFORMATION structure
9698
{
9799
std::cout << "Could not start server: " << GetLastError() << std::endl;
98100
} else {

‎engine/services/download_service.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include <optional>
55
#include <vector>
66

7-
enum class DownloadType { Model, Engine, Miscellaneous };
7+
enum class DownloadType { Model, Engine, Miscellaneous, CudaToolkit };
88

99
enum class DownloadStatus {
1010
Pending,

‎engine/utils/cuda_toolkit_utils.h‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#include <string>
2+
#include "utils/semantic_version_utils.h"
3+
4+
namespace cuda_toolkit_utils {
5+
// those semantic versions are based on: https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html#f1
6+
inline std::string GetCompatibleCudaToolkitVersion(
7+
const std::string& driver_semantic_version, const std::string& os,
8+
const std::string& engine) {
9+
10+
if (engine == "cortex.tensorrt-llm") {
11+
// if the engine is cortex.tensorrt-llm, the minimum required CUDA version is 12.4
12+
if (os == "windows") {
13+
if (semantic_version_utils::CompareSemanticVersion(
14+
driver_semantic_version, "527.41") >= 0) {
15+
return "12.4";
16+
} else {
17+
throw std::runtime_error(
18+
"GPU driver version not supported. Minimum "
19+
"required driver version is 527.41");
20+
}
21+
} else if (os == "linux") {
22+
if (semantic_version_utils::CompareSemanticVersion(
23+
driver_semantic_version, "525.60.13") >= 0) {
24+
return "12.4";
25+
} else {
26+
throw std::runtime_error(
27+
"GPU driver version not supported. Minimum required driver version "
28+
"is 525.60.13");
29+
}
30+
} else {
31+
throw std::runtime_error("Unsupported OS");
32+
}
33+
}
34+
35+
if (os == "windows") {
36+
if (semantic_version_utils::CompareSemanticVersion(driver_semantic_version,
37+
"527.41") >= 0) {
38+
return "12.4";
39+
} else if (semantic_version_utils::CompareSemanticVersion(
40+
driver_semantic_version, "452.39") >= 0) {
41+
return "11.7";
42+
} else {
43+
throw std::runtime_error(
44+
"GPU driver version not supported. Minimum "
45+
"required driver version is 452.39");
46+
}
47+
} else if (os == "linux") {
48+
if (semantic_version_utils::CompareSemanticVersion(driver_semantic_version,
49+
"525.60.13") >= 0) {
50+
return "12.4";
51+
} else if (semantic_version_utils::CompareSemanticVersion(
52+
driver_semantic_version, "450.80.02") >= 0) {
53+
return "11.7";
54+
} else {
55+
throw std::runtime_error(
56+
"GPU driver version not supported. Minimum "
57+
"required driver version is 450.80.02");
58+
}
59+
} else {
60+
throw std::runtime_error("Unsupported OS");
61+
}
62+
}
63+
} // namespace cuda_toolkit_utils

‎engine/utils/file_manager_utils.h‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,63 @@
44
#include <string>
55
#include <string_view>
66

7+
#if defined(__APPLE__) && defined(__MACH__)
8+
#include <mach-o/dyld.h>
9+
#elif defined(__linux__)
10+
#include <unistd.h>
11+
#elif defined(_WIN32)
12+
#include <windows.h>
13+
#endif
14+
715
namespace file_manager_utils {
816

17+
inline std::filesystem::path GetExecutableFolderContainerPath() {
18+
#if defined(__APPLE__) && defined(__MACH__)
19+
char buffer[1024];
20+
uint32_t size = sizeof(buffer);
21+
22+
if (_NSGetExecutablePath(buffer, &size) == 0) {
23+
LOG_INFO << "Executable path: " << buffer;
24+
return std::filesystem::path{buffer}.parent_path();
25+
} else {
26+
LOG_ERROR << "Failed to get executable path";
27+
return std::filesystem::current_path();
28+
}
29+
#elif defined(__linux__)
30+
// TODO: haven't tested
31+
char buffer[1024];
32+
ssize_t len = readlink("/proc/self/exe", buffer, sizeof(buffer) - 1);
33+
if (len != -1) {
34+
buffer[len] = '\0';
35+
LOG_INFO << "Executable path: " << buffer;
36+
return std::filesystem::path{buffer}.parent_path();
37+
} else {
38+
LOG_ERROR << "Failed to get executable path";
39+
return std::filesystem::current_path();
40+
}
41+
#elif defined(_WIN32)
42+
// TODO: haven't tested
43+
char buffer[MAX_PATH];
44+
GetModuleFileNameA(NULL, buffer, MAX_PATH);
45+
LOG_INFO << "Executable path: " << buffer;
46+
return std::filesystem::path{buffer}.parent_path();
47+
#else
48+
LOG_ERROR << "Unsupported platform!";
49+
return std::filesystem::current_path();
50+
#endif
51+
}
52+
953
inline std::filesystem::path GetContainerFolderPath(
1054
const std::string_view type) {
11-
const auto current_path{std::filesystem::current_path()};
55+
const auto current_path{GetExecutableFolderContainerPath()};
1256
auto container_folder_path = std::filesystem::path{};
1357

1458
if (type == "Model") {
1559
container_folder_path = current_path / "models";
1660
} else if (type == "Engine") {
1761
container_folder_path = current_path / "engines";
62+
} else if (type == "CudaToolkit") {
63+
container_folder_path = current_path;
1864
} else {
1965
container_folder_path = current_path / "misc";
2066
}
@@ -35,6 +81,8 @@ inline std::string downloadTypeToString(DownloadType type) {
3581
return "Engine";
3682
case DownloadType::Miscellaneous:
3783
return "Misc";
84+
case DownloadType::CudaToolkit:
85+
return "CudaToolkit";
3886
default:
3987
return "UNKNOWN";
4088
}

0 commit comments

Comments
 (0)