Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 025d814

Browse files
namchuainamchuai
committed
feat: download cuda toolkit
Signed-off-by: James <namnh0122@gmail.com>
1 parent bbc3e31 commit 025d814

File tree

8 files changed

+253
-18
lines changed

8 files changed

+253
-18
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cortex-js
2+
.DS_Store
23
cortex-js/cortex.db
34
dist
45
*.lock
@@ -18,4 +19,4 @@ cortex-js/package-lock.json
1819
.vscode
1920
cortex-js/command
2021
cortex-js/src/infrastructure/commanders/test/test_data
21-
**/vcpkg_installed
22+
**/vcpkg_installed

‎engine/commands/engine_init_cmd.cc‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "utils/archive_utils.h"
88
#include "utils/system_info_utils.h"
99
// clang-format on
10+
#include "utils/cuda_toolkit_utils.h"
1011
#include "utils/engine_matcher_utils.h"
1112

1213
namespace commands {
@@ -102,8 +103,8 @@ void EngineInitCmd::Exec() const {
102103
.type = DownloadType::Engine,
103104
.path = path,
104105
}}};
105-
106-
DownloadService().AddDownloadTask(
106+
DownloadService downloadService;
107+
downloadService.AddDownloadTask(
107108
downloadTask, [](const std::string& absolute_path) {
108109
// try to unzip the downloaded file
109110
std::filesystem::path downloadedEnginePath{absolute_path};
@@ -116,9 +117,62 @@ void EngineInitCmd::Exec() const {
116117
.parent_path()
117118
.string());
118119

119-
// remove the downloaded file
120-
std::filesystem::remove(absolute_path);
121-
LOG_INFO << "Finished!";
120+
try {
121+
std::filesystem::remove(absolute_path);
122+
} catch (std::exception& e) {
123+
LOG_ERROR << "Error removing downloaded file: " << e.what();
124+
}
125+
});
126+
if (system_info.os == "mac" || engineName_ == "cortex.onnx") {
127+
return;
128+
}
129+
// download cuda toolkit
130+
const std::string jan_host = "https://catalog.jan.ai";
131+
const std::string cuda_toolkit_file_name = "cuda.tar.gz";
132+
const std::string download_id = "cuda";
133+
134+
auto gpu_driver_version = system_info_utils::GetDriverVersion();
135+
136+
auto cuda_runtime_version =
137+
cuda_toolkit_utils::GetCompatibleCudaToolkitVersion(
138+
gpu_driver_version, system_info.os, engineName_);
139+
140+
std::ostringstream cuda_toolkit_path;
141+
cuda_toolkit_path << "dist/cuda-dependencies/" << 11.7 << "/"
142+
<< system_info.os << "/"
143+
<< cuda_toolkit_file_name;
144+
145+
LOG_DEBUG << "Cuda toolkit download url: " << jan_host
146+
<< cuda_toolkit_path.str();
147+
148+
auto downloadCudaToolkitTask = DownloadTask{
149+
.id = download_id,
150+
.type = DownloadType::CudaToolkit,
151+
.error = std::nullopt,
152+
.items = {DownloadItem{
153+
.id = download_id,
154+
.host = jan_host,
155+
.fileName = cuda_toolkit_file_name,
156+
.type = DownloadType::CudaToolkit,
157+
.path = cuda_toolkit_path.str(),
158+
}},
159+
};
160+
161+
downloadService.AddDownloadTask(
162+
downloadCudaToolkitTask, [](const std::string& absolute_path) {
163+
LOG_DEBUG << "Downloaded cuda path: " << absolute_path;
164+
// try to unzip the downloaded file
165+
std::filesystem::path downloaded_path{absolute_path};
166+
167+
archive_utils::ExtractArchive(
168+
absolute_path,
169+
downloaded_path.parent_path().parent_path().string());
170+
171+
try {
172+
std::filesystem::remove(absolute_path);
173+
} catch (std::exception& e) {
174+
LOG_ERROR << "Error removing downloaded file: " << e.what();
175+
}
122176
});
123177

124178
return;

‎engine/main.cc‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,16 @@ int main(int argc, char* argv[]) {
5050
}
5151

5252
// Create logs/ folder and setup log to file
53-
std::filesystem::create_directory(cortex_utils::logs_folder);
54-
trantor::AsyncFileLogger asyncFileLogger;
55-
asyncFileLogger.setFileName(cortex_utils::logs_base_name);
56-
asyncFileLogger.startLogging();
57-
trantor::Logger::setOutputFunction(
58-
[&](const char* msg, const uint64_t len) {
59-
asyncFileLogger.output(msg, len);
60-
},
61-
[&]() { asyncFileLogger.flush(); });
62-
asyncFileLogger.setFileSizeLimit(cortex_utils::log_file_size_limit);
53+
// std::filesystem::create_directory(cortex_utils::logs_folder);
54+
// trantor::AsyncFileLogger asyncFileLogger;
55+
// asyncFileLogger.setFileName(cortex_utils::logs_base_name);
56+
// asyncFileLogger.startLogging();
57+
// trantor::Logger::setOutputFunction(
58+
// [&](const char* msg, const uint64_t len) {
59+
// asyncFileLogger.output(msg, len);
60+
// },
61+
// [&]() { asyncFileLogger.flush(); });
62+
// asyncFileLogger.setFileSizeLimit(cortex_utils::log_file_size_limit);
6363

6464
int thread_num = 1;
6565
std::string host = "127.0.0.1";

‎engine/services/download_service.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include <optional>
55
#include <vector>
66

7-
enum class DownloadType { Model, Engine, Miscellaneous };
7+
enum class DownloadType { Model, Engine, Miscellaneous, CudaToolkit };
88

99
enum class DownloadStatus {
1010
Pending,

‎engine/utils/cuda_toolkit_utils.h‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#include <string>
2+
#include "utils/semantic_version_utils.h"
3+
4+
namespace cuda_toolkit_utils {
5+
// those semantic versions are based on: https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html#f1
6+
inline std::string GetCompatibleCudaToolkitVersion(
7+
const std::string& driver_semantic_version, const std::string& os,
8+
const std::string& engine) {
9+
10+
if (engine == "cortex.tensorrt-llm") {
11+
// if the engine is cortex.tensorrt-llm, the minimum required CUDA version is 12.4
12+
if (os == "windows") {
13+
if (semantic_version_utils::CompareSemanticVersion(
14+
driver_semantic_version, "527.41") >= 0) {
15+
return "12.4";
16+
} else {
17+
throw std::runtime_error(
18+
"GPU driver version not supported. Minimum "
19+
"required driver version is 527.41");
20+
}
21+
} else if (os == "linux") {
22+
if (semantic_version_utils::CompareSemanticVersion(
23+
driver_semantic_version, "525.60.13") >= 0) {
24+
return "12.4";
25+
} else {
26+
throw std::runtime_error(
27+
"GPU driver version not supported. Minimum required driver version "
28+
"is 525.60.13");
29+
}
30+
} else {
31+
throw std::runtime_error("Unsupported OS");
32+
}
33+
}
34+
35+
if (os == "windows") {
36+
if (semantic_version_utils::CompareSemanticVersion(driver_semantic_version,
37+
"527.41") >= 0) {
38+
return "12.4";
39+
} else if (semantic_version_utils::CompareSemanticVersion(
40+
driver_semantic_version, "452.39") >= 0) {
41+
return "11.7";
42+
} else {
43+
throw std::runtime_error(
44+
"GPU driver version not supported. Minimum "
45+
"required driver version is 452.39");
46+
}
47+
} else if (os == "linux") {
48+
if (semantic_version_utils::CompareSemanticVersion(driver_semantic_version,
49+
"525.60.13") >= 0) {
50+
return "12.4";
51+
} else if (semantic_version_utils::CompareSemanticVersion(
52+
driver_semantic_version, "450.80.02") >= 0) {
53+
return "11.7";
54+
} else {
55+
throw std::runtime_error(
56+
"GPU driver version not supported. Minimum "
57+
"required driver version is 450.80.02");
58+
}
59+
} else {
60+
throw std::runtime_error("Unsupported OS");
61+
}
62+
}
63+
} // namespace cuda_toolkit_utils

‎engine/utils/file_manager_utils.h‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,63 @@
44
#include <string>
55
#include <string_view>
66

7+
#if defined(__APPLE__) && defined(__MACH__)
8+
#include <mach-o/dyld.h>
9+
#elif defined(__linux__)
10+
#include <unistd.h>
11+
#elif defined(_WIN32)
12+
#include <windows.h>
13+
#endif
14+
715
namespace file_manager_utils {
816

17+
inline std::filesystem::path GetExecutableFolderContainerPath() {
18+
#if defined(__APPLE__) && defined(__MACH__)
19+
char buffer[1024];
20+
uint32_t size = sizeof(buffer);
21+
22+
if (_NSGetExecutablePath(buffer, &size) == 0) {
23+
LOG_INFO << "Executable path: " << buffer;
24+
return std::filesystem::path{buffer}.parent_path();
25+
} else {
26+
LOG_ERROR << "Failed to get executable path";
27+
return std::filesystem::current_path();
28+
}
29+
#elif defined(__linux__)
30+
// TODO: haven't tested
31+
char buffer[1024];
32+
ssize_t len = readlink("/proc/self/exe", buffer, sizeof(buffer) - 1);
33+
if (len != -1) {
34+
buffer[len] = '\0';
35+
LOG_INFO << "Executable path: " << buffer;
36+
return std::filesystem::path{buffer}.parent_path();
37+
} else {
38+
LOG_ERROR << "Failed to get executable path";
39+
return std::filesystem::current_path();
40+
}
41+
#elif defined(_WIN32)
42+
// TODO: haven't tested
43+
char buffer[MAX_PATH];
44+
GetModuleFileNameA(NULL, buffer, MAX_PATH);
45+
LOG_INFO << "Executable path: " << buffer;
46+
return std::filesystem::path{buffer}.parent_path();
47+
#else
48+
LOG_ERROR << "Unsupported platform!";
49+
return std::filesystem::current_path();
50+
#endif
51+
}
52+
953
inline std::filesystem::path GetContainerFolderPath(
1054
const std::string_view type) {
11-
const auto current_path{std::filesystem::current_path()};
55+
const auto current_path{GetExecutableFolderContainerPath()};
1256
auto container_folder_path = std::filesystem::path{};
1357

1458
if (type == "Model") {
1559
container_folder_path = current_path / "models";
1660
} else if (type == "Engine") {
1761
container_folder_path = current_path / "engines";
62+
} else if (type == "CudaToolkit") {
63+
container_folder_path = current_path;
1864
} else {
1965
container_folder_path = current_path / "misc";
2066
}
@@ -35,6 +81,8 @@ inline std::string downloadTypeToString(DownloadType type) {
3581
return "Engine";
3682
case DownloadType::Miscellaneous:
3783
return "Misc";
84+
case DownloadType::CudaToolkit:
85+
return "CudaToolkit";
3886
default:
3987
return "UNKNOWN";
4088
}

‎engine/utils/semantic_version_utils.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#include <sstream>
2+
#include <vector>
3+
4+
namespace semantic_version_utils {
5+
inline std::vector<int> SplitVersion(const std::string& version) {
6+
std::vector<int> parts;
7+
std::stringstream ss(version);
8+
std::string part;
9+
10+
while (std::getline(ss, part, '.')) {
11+
parts.push_back(std::stoi(part));
12+
}
13+
14+
while (parts.size() < 3) {
15+
parts.push_back(0);
16+
}
17+
18+
return parts;
19+
}
20+
21+
inline int CompareSemanticVersion(const std::string& version1,
22+
const std::string& version2) {
23+
std::vector<int> v1 = SplitVersion(version1);
24+
std::vector<int> v2 = SplitVersion(version2);
25+
26+
for (size_t i = 0; i < 3; ++i) {
27+
if (v1[i] < v2[i])
28+
return -1;
29+
if (v1[i] > v2[i])
30+
return 1;
31+
}
32+
return 0;
33+
}
34+
} // namespace semantic_version_utils

‎engine/utils/system_info_utils.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace system_info_utils {
1212

1313
constexpr static auto kUnsupported{"Unsupported"};
1414
constexpr static auto kCudaVersionRegex{R"(CUDA Version:\s*([\d\.]+))"};
15+
constexpr static auto kDriverVersionRegex{R"(Driver Version:\s*(\d+\.\d+))"};
1516
constexpr static auto kGpuQueryCommand{
1617
"nvidia-smi --query-gpu=index,memory.total,name,compute_cap "
1718
"--format=csv,noheader,nounits"};
@@ -177,6 +178,31 @@ inline bool IsNvidiaSmiAvailable() {
177178
#endif
178179
}
179180

181+
inline std::string GetDriverVersion() {
182+
if (!IsNvidiaSmiAvailable()) {
183+
LOG_INFO << "nvidia-smi is not available!";
184+
return "";
185+
}
186+
try {
187+
CommandExecutor cmd("nvidia-smi");
188+
auto output = cmd.execute();
189+
190+
const std::regex driver_version_reg(kDriverVersionRegex);
191+
std::smatch match;
192+
193+
if (std::regex_search(output, match, driver_version_reg)) {
194+
LOG_INFO << "Gpu Driver Version: " << match[1].str();
195+
return match[1].str();
196+
} else {
197+
LOG_ERROR << "Gpu Driver not found!";
198+
return "";
199+
}
200+
} catch (const std::exception& e) {
201+
LOG_ERROR << "Error: " << e.what();
202+
return "";
203+
}
204+
}
205+
180206
inline std::string GetCudaVersion() {
181207
if (!IsNvidiaSmiAvailable()) {
182208
LOG_INFO << "nvidia-smi is not available!";
@@ -207,6 +233,9 @@ struct GpuInfo {
207233
std::string vram;
208234
std::string name;
209235
std::string arch;
236+
// nvidia driver version. Haven't checked for AMD GPU.
237+
std::optional<std::string> driver_version;
238+
std::optional<std::string> cuda_driver_version;
210239
std::optional<std::string> compute_cap;
211240
};
212241

@@ -271,6 +300,10 @@ inline std::vector<GpuInfo> GetGpuInfoList() {
271300
std::vector<GpuInfo> gpuInfoList;
272301

273302
try {
303+
// TODO: improve by parsing both in one command execution
304+
auto driver_version = GetDriverVersion();
305+
auto cuda_version = GetCudaVersion();
306+
274307
CommandExecutor cmd(kGpuQueryCommand);
275308
auto output = cmd.execute();
276309

@@ -285,6 +318,8 @@ inline std::vector<GpuInfo> GetGpuInfoList() {
285318
match[2].str(), // vram
286319
match[3].str(), // name
287320
GetGpuArch(match[3].str()), // arch
321+
driver_version, // driver_version
322+
cuda_version, // cuda_driver_version
288323
match[4].str() // compute_cap
289324
};
290325
gpuInfoList.push_back(gpuInfo);

0 commit comments

Comments
 (0)