From 36b6d3cb1cddf5d2f992b2baf5ca5f696bdb3539 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 16 Dec 2024 08:26:32 +0700 Subject: [PATCH] fix: add support image url for jan --- engine/common/message.h | 21 ++++++++++++ engine/common/message_content_image_url.h | 42 ++++++++++++++++------- engine/common/message_content_text.h | 3 +- engine/test/components/test_models_db.cc | 3 +- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/engine/common/message.h b/engine/common/message.h index 3bff6f048..d31c4f0d3 100644 --- a/engine/common/message.h +++ b/engine/common/message.h @@ -137,6 +137,27 @@ struct Message : JsonSerializable { if (root["content"].isArray() && !root["content"].empty()) { if (root["content"][0]["type"].asString() == "text") { message.content = ParseContents(std::move(root["content"])).value(); + } else if (root["content"][0]["type"].asString() == "image") { + // deprecated, for supporting jan and should be removed in the future + auto text_str = root["content"][0]["text"]["value"].asString(); + auto img_url = + root["content"][0]["text"]["annotations"][0].asString(); + auto text_content = std::make_unique(); + { + auto text = OpenAi::Text(); + auto empty_annotations = + std::vector>(); + text.value = std::move(text_str); + text.annotations = std::move(empty_annotations); + text_content->text = std::move(text); + } + + auto image_url_obj = OpenAi::ImageUrl(img_url, "auto"); + auto image_url_content = std::make_unique( + "image_url", std::move(image_url_obj)); + + message.content.push_back(std::move(text_content)); + message.content.push_back(std::move(image_url_content)); } else { // deprecated, for supporting jan and should be removed in the future // check if annotations is empty diff --git a/engine/common/message_content_image_url.h b/engine/common/message_content_image_url.h index b86544e38..336cf01d3 100644 --- a/engine/common/message_content_image_url.h +++ b/engine/common/message_content_image_url.h @@ -4,14 +4,21 @@ namespace OpenAi { -struct ImageUrl { - // The external URL of the image, must be a supported image types: jpeg, jpg, png, gif, webp. +struct ImageUrl : public JsonSerializable { + /** + * The external URL of the image, must be a supported image types: + * jpeg, jpg, png, gif, webp. + */ std::string url; - // Specifies the detail level of the image. low uses fewer tokens, you can opt in to high resolution using high. Default value is auto + /** + * Specifies the detail level of the image. low uses fewer tokens, you + * can opt in to high resolution using high. Default value is auto + */ std::string detail; - ImageUrl() = default; + ImageUrl(const std::string& url, const std::string& detail = "auto") + : url{url}, detail{detail} {} ImageUrl(ImageUrl&&) noexcept = default; @@ -20,13 +27,25 @@ struct ImageUrl { ImageUrl(const ImageUrl&) = delete; ImageUrl& operator=(const ImageUrl&) = delete; + + cpp::result ToJson() override { + try { + Json::Value root; + root["url"] = url; + root["detail"] = detail; + return root; + } catch (const std::exception& e) { + return cpp::fail(std::string("ToJson failed: ") + e.what()); + } + } }; // References an image URL in the content of a message. struct ImageUrlContent : Content { // The type of the content part. - ImageUrlContent(const std::string& type) : Content(type) {} + explicit ImageUrlContent(const std::string& type, ImageUrl&& image_url) + : Content(type), image_url{std::move(image_url)} {} ImageUrlContent(ImageUrlContent&&) noexcept = default; @@ -38,6 +57,8 @@ struct ImageUrlContent : Content { ImageUrl image_url; + ~ImageUrlContent() override = default; + static cpp::result FromJson( Json::Value&& json) { if (json.empty()) { @@ -45,11 +66,9 @@ struct ImageUrlContent : Content { } try { - ImageUrlContent content{"image_url"}; - ImageUrl image_url; - image_url.url = std::move(json["image_url"]["url"].asString()); - image_url.detail = std::move(json["image_url"]["detail"].asString()); - content.image_url = std::move(image_url); + auto image_url = ImageUrl(json["image_url"]["url"].asString(), + json["image_url"]["detail"].asString()); + ImageUrlContent content{"image_url", std::move(image_url)}; return content; } catch (const std::exception& e) { return cpp::fail(std::string("FromJson failed: ") + e.what()); @@ -60,8 +79,7 @@ struct ImageUrlContent : Content { try { Json::Value json; json["type"] = type; - json["image_url"]["url"] = image_url.url; - json["image_url"]["detail"] = image_url.detail; + json["image_url"] = image_url.ToJson().value(); return json; } catch (const std::exception& e) { return cpp::fail(std::string("ToJson failed: ") + e.what()); diff --git a/engine/common/message_content_text.h b/engine/common/message_content_text.h index ea6aab1ab..5ede2582d 100644 --- a/engine/common/message_content_text.h +++ b/engine/common/message_content_text.h @@ -122,7 +122,6 @@ struct FilePathWrapper : Annotation { struct Text : JsonSerializable { // The data that makes up the text. - Text() = default; Text(Text&&) noexcept = default; @@ -214,6 +213,8 @@ struct TextContent : Content { Text text; + ~TextContent() override = default; + static cpp::result FromJson(Json::Value&& json) { if (json.empty()) { return cpp::fail("Json string is empty"); diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc index 06294aa8c..0cc9b0344 100644 --- a/engine/test/components/test_models_db.cc +++ b/engine/test/components/test_models_db.cc @@ -1,6 +1,5 @@ #include "database/models.h" #include "gtest/gtest.h" -#include "utils/file_manager_utils.h" namespace cortex::db { namespace { @@ -122,4 +121,4 @@ TEST_F(ModelsTestSuite, TestHasModel) { EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value()); } -} // namespace cortex::db \ No newline at end of file +} // namespace cortex::db