From ab406f014dbbd334ac9c7b35ea047fe56fe9ea7c Mon Sep 17 00:00:00 2001 From: tikikun Date: Wed, 4 Oct 2023 21:29:29 +0700 Subject: [PATCH 1/5] fix cuda --- CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 391c128df..191e6044b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,16 @@ set(OPENSSL_USE_STATIC_LIBS TRUE) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/build_deps/_install) # This is the critical line for installing another package +if (LLAMA_CUBLAS) + cmake_minimum_required(VERSION 3.17) + + find_package(CUDAToolkit) + if (CUDAToolkit_FOUND) + message(STATUS "cuBLAS found") + add_compile_definitions(GGML_USE_CUBLAS) + endif() +endif() + add_subdirectory(llama.cpp) add_executable(${PROJECT_NAME} main.cc) From 3dc2d3e08211b4afbd2047d85621758028767184 Mon Sep 17 00:00:00 2001 From: tikikun Date: Wed, 4 Oct 2023 21:30:06 +0700 Subject: [PATCH 2/5] bug fix: patch the crash of spash screen --- controllers/nitro_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/nitro_utils.h b/controllers/nitro_utils.h index 0115aaf93..94b50d7f1 100644 --- a/controllers/nitro_utils.h +++ b/controllers/nitro_utils.h @@ -50,7 +50,7 @@ inline void nitro_logo(){ std::cout << resetColor << c; colorIndex = 0; } else { - std::cout << rainbowColors[colorIndex % 6] << c; + std::cout << rainbowColors[colorIndex % 2] << c; colorIndex++; } } From b20b93499ecb0d1faac0f2814d78fe910e333b80 Mon Sep 17 00:00:00 2001 From: tikikun Date: Wed, 4 Oct 2023 21:32:21 +0700 Subject: [PATCH 3/5] feat: add cublas setups --- controllers/llamaCPP.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controllers/llamaCPP.h b/controllers/llamaCPP.h index 43bc4aa81..fe44cd1b6 100644 --- a/controllers/llamaCPP.h +++ b/controllers/llamaCPP.h @@ -1385,6 +1385,10 @@ class llamaCPP : public drogon::HttpSimpleController { params.model = conf["llama_model_path"].asString(); params.n_gpu_layers = conf["ngl"].asInt(); params.n_ctx = conf["ctx_len"].asInt(); + #ifdef GGML_USE_CUBLAS + LOG_INFO << "Setting up GGML CUBLAS PARAMS"; + params.mul_mat_q = false; + #endif // GGML_USE_CUBLAS if (params.model_alias == "unknown") { params.model_alias = params.model; } From 13d19c22f1d26ec065b5fcf93c59543914c0fcf2 Mon Sep 17 00:00:00 2001 From: tikikun Date: Wed, 4 Oct 2023 21:33:20 +0700 Subject: [PATCH 4/5] bug fix: add regex to make gcc work --- controllers/llamaCPP.h | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/llamaCPP.h b/controllers/llamaCPP.h index fe44cd1b6..2494d317f 100644 --- a/controllers/llamaCPP.h +++ b/controllers/llamaCPP.h @@ -7,6 +7,7 @@ #include "llama.h" #include "build-info.h" #include "grammar-parser.h" +#include #ifndef NDEBUG // crash the server in debug mode, otherwise send an http 500 error From 5d50b53605cae870c7545d5c6fcd5f579d5032a5 Mon Sep 17 00:00:00 2001 From: tikikun Date: Wed, 4 Oct 2023 21:33:50 +0700 Subject: [PATCH 5/5] bug fix: add regex to make gcc work --- controllers/llamaCPP.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controllers/llamaCPP.cc b/controllers/llamaCPP.cc index d87b8f235..8b8fc62a5 100644 --- a/controllers/llamaCPP.cc +++ b/controllers/llamaCPP.cc @@ -4,6 +4,8 @@ #include #include #include +#include + std::string create_return_json(const std::string &id, const std::string &model, const std::string &content, Json::Value finish_reason = Json::Value()) {