From 3f80512df77c9bb30c3889a11e06e8a147ab0935 Mon Sep 17 00:00:00 2001 From: hiro Date: Thu, 18 Jan 2024 14:18:08 +0700 Subject: [PATCH 1/4] feat: Add nitro utils function for base64 and image load --- utils/nitro_utils.h | 134 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/utils/nitro_utils.h b/utils/nitro_utils.h index 987fbd7e1..a50847fe3 100644 --- a/utils/nitro_utils.h +++ b/utils/nitro_utils.h @@ -3,14 +3,17 @@ #include "random" #include "string" #include +#include #include +#include #include #include #include +#include // Include platform-specific headers #ifdef _WIN32 -#include #include +#include #else #include #endif @@ -32,6 +35,135 @@ inline std::string extractBase64(const std::string &input) { return ""; } +// Helper function to encode data to Base64 +inline std::string base64Encode(const std::vector &data) { + static const char encodingTable[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string encodedData; + int i = 0; + int j = 0; + unsigned char array3[3]; + unsigned char array4[4]; + + for (unsigned char c : data) { + array3[i++] = c; + if (i == 3) { + array4[0] = (array3[0] & 0xfc) >> 2; + array4[1] = ((array3[0] & 0x03) << 4) + ((array3[1] & 0xf0) >> 4); + array4[2] = ((array3[1] & 0x0f) << 2) + ((array3[2] & 0xc0) >> 6); + array4[3] = array3[2] & 0x3f; + + for (i = 0; i < 4; i++) + encodedData += encodingTable[array4[i]]; + i = 0; + } + } + + if (i) { + for (j = i; j < 3; j++) + array3[j] = '\0'; + + array4[0] = (array3[0] & 0xfc) >> 2; + array4[1] = ((array3[0] & 0x03) << 4) + ((array3[1] & 0xf0) >> 4); + array4[2] = ((array3[1] & 0x0f) << 2) + ((array3[2] & 0xc0) >> 6); + + for (j = 0; j < i + 1; j++) + encodedData += encodingTable[array4[j]]; + + while (i++ < 3) + encodedData += '='; + } + + return encodedData; +} + +// Function to load an image and convert it to Base64 +inline std::string imageToBase64(const std::string &imagePath) { + std::ifstream imageFile(imagePath, std::ios::binary); + if (!imageFile.is_open()) { + throw std::runtime_error("Could not open the image file."); + } + + std::vector buffer(std::istreambuf_iterator(imageFile), + {}); + return base64Encode(buffer); +} + +// Helper function to generate a unique filename +inline std::string generateUniqueFilename(const std::string &prefix, + const std::string &extension) { + // Get current time as a timestamp + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + auto epoch = now_ms.time_since_epoch(); + auto value = std::chrono::duration_cast(epoch); + + // Generate a random number + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(1000, 9999); + + std::stringstream ss; + ss << prefix << value.count() << "_" << dis(gen) << extension; + return ss.str(); +} + +// Function to download an image +inline void downloadImage(const std::string &url, const std::string &savePath, + std::function callback) { + auto client = drogon::HttpClient::newHttpClient(url); + client->sendRequest( + drogon::HttpRequest::newHttpRequest(), + [savePath, callback](drogon::ReqResult result, + const drogon::HttpResponsePtr &response) { + if (result == drogon::ReqResult::Ok) { + // Save the image to the specified path + std::ofstream outFile(savePath, std::ios::binary); + outFile.write(response->body().data(), response->body().size()); + outFile.close(); + + // Invoke the callback with true to indicate success + callback(true); + } else { + std::cerr << "Failed to download the image: " << result << std::endl; + // Invoke the callback with false to indicate failure + callback(false); + } + }); +} + +inline void +processRemoteImage(const std::string &url, + std::function callback) { + std::string localPath = + generateUniqueFilename("temp_", ".jpg"); // Generate a unique filename + + downloadImage(url, localPath, [localPath, callback](bool success) { + if (success) { + try { + std::string base64Image = imageToBase64(localPath); + std::remove(localPath.c_str()); // Delete the local file + callback(base64Image); // Invoke the callback with the Base64 string + } catch (const std::exception &e) { + std::cerr << "Error during processing: " << e.what() << std::endl; + } + } else { + std::cerr << "Failed to download the image." << std::endl; + } + }); +} + +inline void +processLocalImage(const std::string &localPath, + std::function callback) { + try { + std::string base64Image = imageToBase64(localPath); + callback(base64Image); // Invoke the callback with the Base64 string + } catch (const std::exception &e) { + std::cerr << "Error during processing: " << e.what() << std::endl; + } +} + inline std::vector listFilesInDir(const std::string &path) { std::vector files; From 2140754dea1cf50e287b1e72df930ae0ce5f9627 Mon Sep 17 00:00:00 2001 From: hiro Date: Thu, 18 Jan 2024 14:18:24 +0700 Subject: [PATCH 2/4] feat: Add handler for remote and local image url --- controllers/llamaCPP.cc | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/controllers/llamaCPP.cc b/controllers/llamaCPP.cc index 3f7eb9dd0..d99c0663f 100644 --- a/controllers/llamaCPP.cc +++ b/controllers/llamaCPP.cc @@ -239,17 +239,38 @@ void llamaCPP::chatCompletion( for (auto content_piece : message["content"]) { role = user_prompt; + json content_piece_image_data; + auto content_piece_type = content_piece["type"].asString(); if (content_piece_type == "text") { auto text = content_piece["text"].asString(); formatted_output += text; } else if (content_piece_type == "image_url") { auto image_url = content_piece["image_url"]["url"].asString(); - auto base64_image_data = nitro_utils::extractBase64(image_url); - LOG_INFO << base64_image_data; - formatted_output += "[img-" + std::to_string(no_images) + "]"; + std::string + base64_image_data; // Declare the variable 'base64_image_data' + if (image_url.find("http") != std::string::npos) { + // If image url is a remote link, extract and use convert to + // base64 + nitro_utils::processRemoteImage( + image_url, [](const std::string &base64Image) { + auto base64_image_data = base64Image; + LOG_INFO << base64_image_data; + }); + } else if (image_url.find("data:image") != std::string::npos) { + // If image url is already in base64, use it directly + auto base64_image_data = nitro_utils::extractBase64(image_url); + LOG_INFO << base64_image_data; + } else { + // If image url is a local file, convert to base64 + nitro_utils::processLocalImage( + image_url, [](const std::string &base64Image) { + auto base64_image_data = base64Image; + LOG_INFO << base64_image_data; + }); + } - json content_piece_image_data; + formatted_output += "[img-" + std::to_string(no_images) + "]"; content_piece_image_data["data"] = base64_image_data; content_piece_image_data["id"] = no_images; data["image_data"].push_back(content_piece_image_data); From 119cd81bf346528f429408e8a1ae1ebf75c700b9 Mon Sep 17 00:00:00 2001 From: hiro Date: Thu, 18 Jan 2024 17:37:10 +0700 Subject: [PATCH 3/4] fix: Remove support for remote URL --- controllers/llamaCPP.cc | 26 +++++++++--------------- llama.cpp | 2 +- utils/nitro_utils.h | 45 ----------------------------------------- 3 files changed, 11 insertions(+), 62 deletions(-) diff --git a/controllers/llamaCPP.cc b/controllers/llamaCPP.cc index d99c0663f..c4c83b2c6 100644 --- a/controllers/llamaCPP.cc +++ b/controllers/llamaCPP.cc @@ -240,6 +240,7 @@ void llamaCPP::chatCompletion( role = user_prompt; json content_piece_image_data; + content_piece_image_data["data"] = ""; auto content_piece_type = content_piece["type"].asString(); if (content_piece_type == "text") { @@ -247,31 +248,24 @@ void llamaCPP::chatCompletion( formatted_output += text; } else if (content_piece_type == "image_url") { auto image_url = content_piece["image_url"]["url"].asString(); - std::string - base64_image_data; // Declare the variable 'base64_image_data' + std::string base64_image_data; if (image_url.find("http") != std::string::npos) { - // If image url is a remote link, extract and use convert to - // base64 - nitro_utils::processRemoteImage( - image_url, [](const std::string &base64Image) { - auto base64_image_data = base64Image; - LOG_INFO << base64_image_data; - }); + LOG_INFO << "Remote image detected but not supported yet"; } else if (image_url.find("data:image") != std::string::npos) { - // If image url is already in base64, use it directly - auto base64_image_data = nitro_utils::extractBase64(image_url); + LOG_INFO << "Base64 image detected"; + base64_image_data = nitro_utils::extractBase64(image_url); LOG_INFO << base64_image_data; } else { - // If image url is a local file, convert to base64 + LOG_INFO << "Local image detected"; nitro_utils::processLocalImage( - image_url, [](const std::string &base64Image) { - auto base64_image_data = base64Image; - LOG_INFO << base64_image_data; + image_url, [&](const std::string &base64Image) { + base64_image_data = base64Image; }); + LOG_INFO << base64_image_data; } + content_piece_image_data["data"] = base64_image_data; formatted_output += "[img-" + std::to_string(no_images) + "]"; - content_piece_image_data["data"] = base64_image_data; content_piece_image_data["id"] = no_images; data["image_data"].push_back(content_piece_image_data); no_images++; diff --git a/llama.cpp b/llama.cpp index 862f5e41a..1e605f410 160000 --- a/llama.cpp +++ b/llama.cpp @@ -1 +1 @@ -Subproject commit 862f5e41ab1fdf12d6f59455aad3f5dd8258f805 +Subproject commit 1e605f4102c7ea8dc0dff82f5eaa6a71973d549f diff --git a/utils/nitro_utils.h b/utils/nitro_utils.h index a50847fe3..d37a6dec5 100644 --- a/utils/nitro_utils.h +++ b/utils/nitro_utils.h @@ -108,51 +108,6 @@ inline std::string generateUniqueFilename(const std::string &prefix, return ss.str(); } -// Function to download an image -inline void downloadImage(const std::string &url, const std::string &savePath, - std::function callback) { - auto client = drogon::HttpClient::newHttpClient(url); - client->sendRequest( - drogon::HttpRequest::newHttpRequest(), - [savePath, callback](drogon::ReqResult result, - const drogon::HttpResponsePtr &response) { - if (result == drogon::ReqResult::Ok) { - // Save the image to the specified path - std::ofstream outFile(savePath, std::ios::binary); - outFile.write(response->body().data(), response->body().size()); - outFile.close(); - - // Invoke the callback with true to indicate success - callback(true); - } else { - std::cerr << "Failed to download the image: " << result << std::endl; - // Invoke the callback with false to indicate failure - callback(false); - } - }); -} - -inline void -processRemoteImage(const std::string &url, - std::function callback) { - std::string localPath = - generateUniqueFilename("temp_", ".jpg"); // Generate a unique filename - - downloadImage(url, localPath, [localPath, callback](bool success) { - if (success) { - try { - std::string base64Image = imageToBase64(localPath); - std::remove(localPath.c_str()); // Delete the local file - callback(base64Image); // Invoke the callback with the Base64 string - } catch (const std::exception &e) { - std::cerr << "Error during processing: " << e.what() << std::endl; - } - } else { - std::cerr << "Failed to download the image." << std::endl; - } - }); -} - inline void processLocalImage(const std::string &localPath, std::function callback) { From 236e3ba64d23edd72b91fa74379e3678a9d5b337 Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 23 Jan 2024 17:33:50 +0700 Subject: [PATCH 4/4] fix: Reoder include to prevent bugs on windows --- utils/nitro_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/nitro_utils.h b/utils/nitro_utils.h index d37a6dec5..ef4690e7d 100644 --- a/utils/nitro_utils.h +++ b/utils/nitro_utils.h @@ -12,8 +12,8 @@ #include // Include platform-specific headers #ifdef _WIN32 -#include #include +#include #else #include #endif