diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..87d39608 --- /dev/null +++ b/.clang-format @@ -0,0 +1,167 @@ +--- +Language: Cpp +BasedOnStyle: Google +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: false +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 0 + - Regex: '^<.*\.h>' + Priority: 1 + SortPriority: 0 + - Regex: '^<.*' + Priority: 2 + SortPriority: 0 + - Regex: '.*' + Priority: 3 + SortPriority: 0 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: true +SortIncludes: false +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..cb65ec91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.vscode/settings.json +projects/core135_llm_product_test_ui +projects/imx678_test \ No newline at end of file diff --git a/SDK b/SDK index c767769d..e6f2432c 160000 --- a/SDK +++ b/SDK @@ -1 +1 @@ -Subproject commit c767769d28ec511b613a24fc8680ec3acd4c769a +Subproject commit e6f2432c564324360b762d738698cc170b95f68d diff --git a/doc/projects_llm_framework_doc/llm_camera.md b/doc/projects_llm_framework_doc/llm_camera.md new file mode 100644 index 00000000..891c7cb6 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_camera.md @@ -0,0 +1,147 @@ +# llm-camera +视频源单元，用于从 USB V4L2 视频设备中获取视频流到内部通道。 + +## setup +配置单元工作。 + +发送 json： +```json +{ + "request_id":"2", + "work_id":"camera", + "action":"setup", + "object":"camera.setup", + "data":{ + "response_format":"camera.raw", + "input":"/dev/video0", + "enoutput":false, + "frame_width":320, + "frame_height":320 + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `camera`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `camera.setup`。 +- response_format：返回结果为 `camera.raw`，是 yuv422 格式。 +- input：读取的设备名。 +- frame_width：输出的视频帧宽。 +- frame_height：输出的视频帧高。 +- enoutput：是否起用用户结果输出。如果不需要获取摄像头图片，请不要开启该参数，视频流会增加信道的通信压力。 + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"camera.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "camera.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"camera.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["camera.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"camera.tasklist", + "request_id":"2", + "work_id":"camera" +} +``` + +获取任务运行参数。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "camera.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "response_format":"camera.raw", + "input":"/dev/video0", + "frame_width":320, + "frame_height":320 + }, + "error":{ + "code":0, + "message":"" + }, + "object":"camera.taskinfo", + "request_id":"2", + "work_id":"camera.1003" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_yolo_zh.md b/doc/projects_llm_framework_doc/llm_yolo_zh.md new file mode 100644 index 00000000..66c0f6d0 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_yolo_zh.md @@ -0,0 +1,144 @@ +# llm-yolo +yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo 模型。 + +## setup +配置单元工作。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "setup", + "object": "yolo.setup", + "data": { + "model": "yolov5", + "response_format": "yolo.box", + "input": "yolo.jpg.base64", + "enoutput": true + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `yolo`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `yolo.setup`。 +- model：使用的模型为 `yolov5` 模型。 +- response_format：返回结果为 `yolo.box`。 +- input：输入的为 `yolo.jpg.base64`,代表的是从用户输入，数据类型为 jpg, base64编码。 +- enoutput：是否起用用户结果输出。 + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"yolo.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "tts.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"tts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["yolo.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"yolo.tasklist", + "request_id":"2", + "work_id":"yolo" +} +``` + +获取任务运行参数。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["yolo.jpg.base64"], + "model":"yolov5", + "response_format":"yolo.box" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"tts.taskinfo", + "request_id":"2", + "work_id":"yolo.1003" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/ext_components/StackFlow/SConstruct b/ext_components/StackFlow/SConstruct index 8007eb97..abac5610 100644 --- a/ext_components/StackFlow/SConstruct +++ b/ext_components/StackFlow/SConstruct @@ -20,7 +20,7 @@ if 'CONFIG_STACKFLOW_ENABLED' in os.environ: INCLUDE.append(ADir("stackflow")) PRIVATE_INCLUDE.append(ADir("stackflow/libzmq")) - REQUIREMENTS += ['eventpp', 'utilities', 'zmq'] + REQUIREMENTS += ['eventpp', 'utilities', 'zmq', 'simdjson_component'] env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), 'SRCS':SRCS, diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index dace7cb4..236cc7ab 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -10,35 +10,48 @@ using namespace StackFlows; std::string llm_channel_obj::uart_push_url; +// Deprecated +#define RPC_PARSE_TO_PARAM_OLD(obj) \ + sample_json_str_get(obj, "zmq_com"), sample_unescapeString(sample_json_str_get(obj, "raw_data")) + +#define RPC_PARSE_TO_PARAM(obj) RPC_PARSE_TO_FIRST(obj), RPC_PARSE_TO_SECOND(obj) + llm_channel_obj::llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name) - : unit_name_(unit_name), inference_url_(inference_url) { + : unit_name_(unit_name), inference_url_(inference_url) +{ zmq_url_index_ = -1000; - zmq_[-1] = std::make_unique(_publisher_url, ZMQ_PUB); + zmq_[-1] = std::make_shared(_publisher_url, ZMQ_PUB); zmq_[-2].reset(); } -llm_channel_obj::~llm_channel_obj() { +llm_channel_obj::~llm_channel_obj() +{ } void llm_channel_obj::subscriber_event_call(const std::function &call, - const std::string &raw) { - // SLOGI("object%s", sample_json_str_get(raw, "object").c_str()); - // SLOGI("request_id%s", sample_json_str_get(raw, "request_id").c_str()); - // SLOGI("work_id%s", sample_json_str_get(raw, "work_id").c_str()); - // SLOGI("action%s", sample_json_str_get(raw, "action").c_str()); - // SLOGI("data%s", sample_json_str_get(raw, "data").c_str()); - if (sample_json_str_get(raw, "action") == "inference") { - std::string zmq_com = sample_json_str_get(raw, "zmq_com"); - if (!zmq_com.empty()) set_push_url(zmq_com); - request_id_ = sample_json_str_get(raw, "request_id"); - work_id_ = sample_json_str_get(raw, "work_id"); + pzmq *_pzmq, const std::string &raw) +{ + const char *user_inference_flage_str = "\"action\""; + std::size_t pos = raw.find(user_inference_flage_str); + while (true) { + if (pos == std::string::npos) { + break; + } else if ((pos > 0) && (raw[pos - 1] != '\\')) { + std::string zmq_com = sample_json_str_get(raw, "zmq_com"); + if (!zmq_com.empty()) set_push_url(zmq_com); + request_id_ = sample_json_str_get(raw, "request_id"); + work_id_ = sample_json_str_get(raw, "work_id"); + break; + } + pos = raw.find(user_inference_flage_str, pos + sizeof(user_inference_flage_str)); } call(sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } int llm_channel_obj::subscriber_work_id(const std::string &work_id, - const std::function &call) { + const std::function &call) +{ int id_num; std::string subscriber_url; std::regex pattern(R"((\w+)\.(\d+))"); @@ -48,9 +61,7 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, // std::string part1 = matches[1].str(); id_num = std::stoi(matches[2].str()); std::string input_url_name = work_id + ".out_port"; - std::string input_url; - pzmq _call("sys"); - _call.call_rpc_action("sql_select", input_url_name, [&](const std::string &data) { input_url = data; }); + std::string input_url = unit_call("sys", "sql_select", input_url_name); if (input_url.empty()) { return -1; } @@ -60,12 +71,14 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, id_num = 0; subscriber_url = inference_url_; } - zmq_[id_num] = std::make_unique( - subscriber_url, ZMQ_SUB, std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1)); + zmq_[id_num] = std::make_shared( + subscriber_url, ZMQ_SUB, + std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1, std::placeholders::_2)); return 0; } -void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) { +void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) +{ int id_num; std::regex pattern(R"((\w+)\.(\d+))"); std::smatch matches; @@ -80,12 +93,14 @@ void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) { if (zmq_.find(id_num) != zmq_.end()) zmq_.erase(id_num); } -void llm_channel_obj::subscriber(const std::string &zmq_url, const std::function &call) { +void llm_channel_obj::subscriber(const std::string &zmq_url, const pzmq::msg_callback_fun &call) +{ zmq_url_map_[zmq_url] = zmq_url_index_--; - zmq_[zmq_url_map_[zmq_url]] = std::make_unique(zmq_url, ZMQ_SUB, call); + zmq_[zmq_url_map_[zmq_url]] = std::make_shared(zmq_url, ZMQ_SUB, call); } -void llm_channel_obj::stop_subscriber(const std::string &zmq_url) { +void llm_channel_obj::stop_subscriber(const std::string &zmq_url) +{ if (zmq_url.empty()) { zmq_.clear(); zmq_url_map_.clear(); @@ -95,11 +110,13 @@ void llm_channel_obj::stop_subscriber(const std::string &zmq_url) { } } -int llm_channel_obj::send_raw_to_pub(const std::string &raw) { +int llm_channel_obj::send_raw_to_pub(const std::string &raw) +{ return zmq_[-1]->send_data(raw); } -int llm_channel_obj::send_raw_to_usr(const std::string &raw) { +int llm_channel_obj::send_raw_to_usr(const std::string &raw) +{ if (zmq_[-2]) { return zmq_[-2]->send_data(raw); } else { @@ -107,28 +124,33 @@ int llm_channel_obj::send_raw_to_usr(const std::string &raw) { } } -void llm_channel_obj::set_push_url(const std::string &url) { +void llm_channel_obj::set_push_url(const std::string &url) +{ if (output_url_ != url) { output_url_ = url; zmq_[-2].reset(new pzmq(output_url_, ZMQ_PUSH)); } } -void llm_channel_obj::cear_push_url() { +void llm_channel_obj::cear_push_url() +{ zmq_[-2].reset(); } -int llm_channel_obj::send_raw_for_url(const std::string &zmq_url, const std::string &raw) { +int llm_channel_obj::send_raw_for_url(const std::string &zmq_url, const std::string &raw) +{ pzmq _zmq(zmq_url, ZMQ_PUSH); return _zmq.send_data(raw); } -int llm_channel_obj::output_to_uart(const std::string &data) { +int llm_channel_obj::output_to_uart(const std::string &data) +{ return send_raw_for_url(uart_push_url, data); } StackFlow::StackFlow::StackFlow(const std::string &unit_name) - : unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) { + : work_id_num_cout_(1000), unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) +{ event_queue_.appendListener(EVENT_NONE, std::bind(&StackFlow::_none_event, this, std::placeholders::_1, std::placeholders::_2)); event_queue_.appendListener(EVENT_PAUSE, @@ -149,27 +171,47 @@ StackFlow::StackFlow::StackFlow(const std::string &unit_name) std::bind(&StackFlow::_sys_init, this, std::placeholders::_1, std::placeholders::_2)); event_queue_.appendListener( EVENT_REPEAT_EVENT, std::bind(&StackFlow::_repeat_loop, this, std::placeholders::_1, std::placeholders::_2)); - rpc_ctx_->register_rpc_action("setup", std::bind(&StackFlow::_rpc_setup, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("pause", std::bind(&StackFlow::_rpc_pause, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("work", std::bind(&StackFlow::_rpc_work, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("exit", std::bind(&StackFlow::_rpc_exit, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("link", std::bind(&StackFlow::_rpc_link, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("unlink", std::bind(&StackFlow::_rpc_unlink, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("taskinfo", std::bind(&StackFlow::_rpc_taskinfo, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action( + "setup", std::bind(&StackFlow::_rpc_setup, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "pause", std::bind(&StackFlow::_rpc_pause, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("work", + std::bind(&StackFlow::_rpc_work, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("exit", + std::bind(&StackFlow::_rpc_exit, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("link", + std::bind(&StackFlow::_rpc_link, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "unlink", std::bind(&StackFlow::_rpc_unlink, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "taskinfo", std::bind(&StackFlow::_rpc_taskinfo, this, std::placeholders::_1, std::placeholders::_2)); status_.store(0); exit_flage_.store(false); - even_loop_thread_ = std::make_unique(std::bind(&StackFlow::even_loop, this)); - event_queue_.enqueue(EVENT_SYS_INIT, "", ""); + even_loop_thread_ = std::make_unique(std::bind(&StackFlow::even_loop, this)); + llm_channel_obj::uart_push_url = std::string("ipc:///tmp/llm/5556.sock"); + status_.store(1); + repeat_event(1000, [this]() { + std::string serial_zmq_url = this->sys_sql_select("serial_zmq_url"); + if (!serial_zmq_url.empty()) { + SLOGI("serial_zmq_url:%s", serial_zmq_url.c_str()); + llm_channel_obj::uart_push_url = serial_zmq_url; + return 0; + } else { + return 1; + } + }); } -StackFlow::~StackFlow() { +StackFlow::~StackFlow() +{ while (1) { auto iteam = llm_task_channel_.begin(); if (iteam == llm_task_channel_.end()) { break; } sys_release_unit(iteam->first, ""); + iteam->second.reset(); llm_task_channel_.erase(iteam->first); } exit_flage_.store(true); @@ -177,38 +219,31 @@ StackFlow::~StackFlow() { even_loop_thread_->join(); } -void StackFlow::even_loop() { +void StackFlow::even_loop() +{ while (!exit_flage_.load()) { event_queue_.wait(); event_queue_.process(); } } -void StackFlow::_none_event(const std::string &data1, const std::string &data2) { +void StackFlow::_none_event(const std::string &data1, const std::string &data2) +{ } -void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) { - // serial_zmq_url - std::string serial_zmq_url; - serial_zmq_url = sys_sql_select("serial_zmq_url"); - if (!serial_zmq_url.empty()) { - SLOGI("serial_zmq_url:%s", serial_zmq_url.c_str()); - llm_channel_obj::uart_push_url = serial_zmq_url; - status_.store(1); - } else { - sleep(1); - event_queue_.enqueue(EVENT_SYS_INIT, "", ""); - } +void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) +{ + // todo:... } -std::string StackFlow::_rpc_setup(const std::string &data) { - SLOGI("_rpc_setup:%s", data.c_str()); - event_queue_.enqueue(EVENT_SETUP, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_setup(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_SETUP, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -int StackFlow::setup(const std::string &zmq_url, const std::string &raw) { +int StackFlow::setup(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::setup raw zmq_url:%s raw:%s", zmq_url.c_str(), raw.c_str()); int workid_num = sys_register_unit(unit_name_); std::string work_id = unit_name_ + "." + std::to_string(workid_num); @@ -222,7 +257,8 @@ int StackFlow::setup(const std::string &zmq_url, const std::string &raw) { return 0; } -int StackFlow::setup(const std::string &work_id, const std::string &object, const std::string &data) { +int StackFlow::setup(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::setup"); if (_setup_) { return _setup_(work_id, object, data); @@ -234,13 +270,14 @@ int StackFlow::setup(const std::string &work_id, const std::string &object, cons return -1; } -std::string StackFlow::_rpc_link(const std::string &data) { - event_queue_.enqueue(EVENT_LINK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_link(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_LINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -void StackFlow::link(const std::string &zmq_url, const std::string &raw) { +void StackFlow::link(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::link raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -251,7 +288,8 @@ void StackFlow::link(const std::string &zmq_url, const std::string &raw) { link(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::link(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::link(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::link"); if (_link_) { _link_(work_id, object, data); @@ -263,13 +301,14 @@ void StackFlow::link(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_unlink(const std::string &data) { - event_queue_.enqueue(EVENT_UNLINK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_unlink(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_UNLINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) { +void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::unlink raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -280,7 +319,8 @@ void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) { unlink(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::unlink(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::unlink(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::unlink"); if (_unlink_) { _unlink_(work_id, object, data); @@ -292,13 +332,14 @@ void StackFlow::unlink(const std::string &work_id, const std::string &object, co send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_work(const std::string &data) { - event_queue_.enqueue(EVENT_WORK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_work(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_WORK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -void StackFlow::work(const std::string &zmq_url, const std::string &raw) { +void StackFlow::work(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::work raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -309,7 +350,8 @@ void StackFlow::work(const std::string &zmq_url, const std::string &raw) { work(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::work(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::work(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::work"); if (_work_) { _work_(work_id, object, data); @@ -321,13 +363,14 @@ void StackFlow::work(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_exit(const std::string &data) { - event_queue_.enqueue(EVENT_EXIT, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_exit(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_EXIT, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -int StackFlow::exit(const std::string &zmq_url, const std::string &raw) { +int StackFlow::exit(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::exit raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -341,7 +384,8 @@ int StackFlow::exit(const std::string &zmq_url, const std::string &raw) { return 0; } -int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string &data) { +int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::exit"); if (_exit_) { return _exit_(work_id, object, data); @@ -353,13 +397,14 @@ int StackFlow::exit(const std::string &work_id, const std::string &object, const return 0; } -std::string StackFlow::_rpc_pause(const std::string &data) { - event_queue_.enqueue(EVENT_PAUSE, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_pause(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_PAUSE, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -void StackFlow::pause(const std::string &zmq_url, const std::string &raw) { +void StackFlow::pause(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::pause raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -370,7 +415,8 @@ void StackFlow::pause(const std::string &zmq_url, const std::string &raw) { pause(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::pause(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::pause(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::pause"); if (_pause_) { _pause_(work_id, object, data); @@ -382,13 +428,14 @@ void StackFlow::pause(const std::string &work_id, const std::string &object, con send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_taskinfo(const std::string &data) { - event_queue_.enqueue(EVENT_TASKINFO, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); +std::string StackFlow::_rpc_taskinfo(pzmq *_pzmq, const std::string &data) +{ + event_queue_.enqueue(EVENT_TASKINFO, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } -void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) { +void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::taskinfo raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -399,7 +446,8 @@ void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) { taskinfo(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::taskinfo(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::taskinfo(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::taskinfo"); if (_taskinfo_) { _taskinfo_(work_id, object, data); @@ -411,24 +459,24 @@ void StackFlow::taskinfo(const std::string &work_id, const std::string &object, send("None", "None", error_body, work_id); } -int StackFlow::sys_register_unit(const std::string &unit_name) { +int StackFlow::sys_register_unit(const std::string &unit_name) +{ int work_id_number; - std::string out_port; - std::string inference_port; - - pzmq _call("sys"); - _call.call_rpc_action("register_unit", unit_name, [&](const std::string &unit_info) { - work_id_number = std::stoi(sample_json_str_get(unit_info, "work_id_number")); - out_port = sample_json_str_get(unit_info, "out_port"); - inference_port = sample_json_str_get(unit_info, "inference_port"); - }); + std::string component_msg = unit_call("sys", "register_unit", unit_name); + std::string str_port = RPC_PARSE_TO_FIRST(component_msg); + work_id_number = std::stoi(str_port); + std::string tmp_buf = RPC_PARSE_TO_SECOND(component_msg); + std::string out_port = RPC_PARSE_TO_FIRST(tmp_buf); + std::string inference_port = RPC_PARSE_TO_SECOND(tmp_buf); + SLOGI("work_id_number:%d, out_port:%s, inference_port:%s ", work_id_number, out_port.c_str(), inference_port.c_str()); llm_task_channel_[work_id_number] = std::make_shared(out_port, inference_port, unit_name_); return work_id_number; } -bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) { +bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) +{ std::string _work_id; int _work_id_num; if (work_id.empty()) { @@ -438,44 +486,33 @@ bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) { _work_id = work_id; _work_id_num = sample_get_work_id_num(work_id); } - pzmq _call("sys"); - _call.call_rpc_action("release_unit", _work_id, [&](const std::string &unit_info) {}); + unit_call("sys", "release_unit", _work_id); llm_task_channel_[_work_id_num].reset(); llm_task_channel_.erase(_work_id_num); SLOGI("release work_id %s success", _work_id.c_str()); return false; } -std::string StackFlow::sys_sql_select(const std::string &key) { - std::string val; - pzmq _call("sys"); - _call.call_rpc_action("sql_select", key, [&val](const std::string &data) { val = data; }); - return val; +std::string StackFlow::sys_sql_select(const std::string &key) +{ + return sample_unescapeString(unit_call("sys", "sql_select", key)); } -void StackFlow::sys_sql_set(const std::string &key, const std::string &val) { +void StackFlow::sys_sql_set(const std::string &key, const std::string &val) +{ nlohmann::json out_body; out_body["key"] = key; out_body["val"] = val; - pzmq _call("sys"); - _call.call_rpc_action("sql_set", out_body.dump(), [](const std::string &data) {}); -} - -void StackFlow::sys_sql_unset(const std::string &key) { - std::string val; - pzmq _call("sys"); - _call.call_rpc_action("sql_select", key, [](const std::string &data) {}); + unit_call("sys", "sql_set", out_body.dump()); } -std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, - const std::string &data) { - std::string value; - pzmq _call(unit_name); - _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); - return value; +void StackFlow::sys_sql_unset(const std::string &key) +{ + unit_call("sys", "sql_unset", key); } -void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) { +void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) +{ repeat_callback_fun_mutex_.lock(); const auto call_fun = repeat_callback_fun_[action]; repeat_callback_fun_mutex_.unlock(); @@ -496,7 +533,8 @@ void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) { } } -void StackFlow::repeat_event(int ms, std::function repeat_fun, bool now) { +void StackFlow::repeat_event(int ms, std::function repeat_fun, bool now) +{ repeat_callback_fun_mutex_.lock(); std::string action = std::to_string(repeat_callback_fun_.size() + 1); repeat_callback_fun_[action] = repeat_fun; diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index f7c911fc..2098458c 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -27,30 +27,34 @@ namespace StackFlows { template class ThreadSafeWrapper { - public: +public: template - ThreadSafeWrapper(Args &&...args) : object(std::make_unique(std::forward(args)...)) { + ThreadSafeWrapper(Args &&...args) : object(std::make_unique(std::forward(args)...)) + { } template - auto access(Func func) -> decltype(func(std::declval())) { + auto access(Func func) -> decltype(func(std::declval())) + { std::lock_guard lock(mutex); return func(*object); } - ThreadSafeWrapper &operator=(const T &newValue) { + ThreadSafeWrapper &operator=(const T &newValue) + { std::lock_guard lock(mutex); *object = newValue; return *this; } - ThreadSafeWrapper &operator=(T &&newValue) { + ThreadSafeWrapper &operator=(T &&newValue) + { std::lock_guard lock(mutex); *object = std::move(newValue); return *this; } - private: +private: std::unique_ptr object; mutable std::mutex mutex; }; @@ -59,12 +63,12 @@ class ThreadSafeWrapper { #define LLM_NONE std::string("None") class llm_channel_obj { - private: - std::unordered_map> zmq_; +private: + std::unordered_map> zmq_; std::atomic zmq_url_index_; std::unordered_map zmq_url_map_; - public: +public: std::string unit_name_; bool enoutput_; bool enstream_; @@ -77,30 +81,35 @@ class llm_channel_obj { llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name); ~llm_channel_obj(); - inline void set_output(bool flage) { + inline void set_output(bool flage) + { enoutput_ = flage; } - inline bool get_output() { + inline bool get_output() + { return enoutput_; } - inline void set_stream(bool flage) { + inline void set_stream(bool flage) + { enstream_ = flage; } - inline bool get_stream() { + inline bool get_stream() + { return enstream_; } - void subscriber_event_call(const std::function &call, + void subscriber_event_call(const std::function &call, pzmq *_pzmq, const std::string &raw); int subscriber_work_id(const std::string &work_id, const std::function &call); void stop_subscriber_work_id(const std::string &work_id); - void subscriber(const std::string &zmq_url, const std::function &call); + void subscriber(const std::string &zmq_url, const pzmq::msg_callback_fun &call); void stop_subscriber(const std::string &zmq_url); int check_zmq_errno(void *ctx, void *com, int code); int send_raw_to_pub(const std::string &raw); int send_raw_to_usr(const std::string &raw); template - int output_data(const std::string &object, const T &data, const U &error_msg) { + int output_data(const std::string &object, const T &data, const U &error_msg) + { return output_data(request_id_, work_id_, object, data, error_msg); } void set_push_url(const std::string &url); @@ -110,7 +119,8 @@ class llm_channel_obj { template static int output_data_for_url(const std::string &zmq_url, const std::string &request_id, const std::string &work_id, const std::string &object, const U &data, - const T &error_msg, bool outuart = false) { + const T &error_msg, bool outuart = false) + { nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -123,6 +133,7 @@ class llm_channel_obj { } else out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; if (outuart) return output_to_uart(out); else @@ -130,7 +141,8 @@ class llm_channel_obj { } template int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id = "", - bool outuart = false) { + bool outuart = false) + { nlohmann::json out_body; out_body["request_id"] = request_id_; out_body["work_id"] = work_id.empty() ? work_id_ : work_id; @@ -144,13 +156,15 @@ class llm_channel_obj { out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; send_raw_to_pub(out); if (enoutput_) return send_raw_to_usr(out); return 0; } template int output_data(const std::string &request_id, const std::string &work_id, const std::string &object, const T &data, - const U &error_msg) { + const U &error_msg) + { nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -164,6 +178,7 @@ class llm_channel_obj { out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; send_raw_to_pub(out); if (enoutput_) return send_raw_to_usr(out); return 0; @@ -171,8 +186,10 @@ class llm_channel_obj { }; class StackFlow { - private: - protected: +private: + std::atomic_int work_id_num_cout_; + +protected: std::string unit_name_; typedef enum { EVENT_NONE = 0, @@ -202,7 +219,7 @@ class StackFlow { void _repeat_loop(const std::string &zmq_url, const std::string &raw); - public: +public: std::string request_id_; std::string out_zmq_url_; @@ -222,7 +239,8 @@ class StackFlow { void _none_event(const std::string &data1, const std::string &data2); template - std::shared_ptr get_channel(T workid) { + std::shared_ptr get_channel(T workid) + { int _work_id_num; if constexpr (std::is_same::value) { _work_id_num = workid; @@ -234,8 +252,9 @@ class StackFlow { return llm_task_channel_.at(_work_id_num); } - std::string _rpc_setup(const std::string &data); - void _setup(const std::string &zmq_url, const std::string &data) { + std::string _rpc_setup(pzmq *_pzmq, const std::string &data); + void _setup(const std::string &zmq_url, const std::string &data) + { // printf("void _setup run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -244,8 +263,9 @@ class StackFlow { virtual int setup(const std::string &zmq_url, const std::string &raw); virtual int setup(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_link(const std::string &data); - void _link(const std::string &zmq_url, const std::string &data) { + std::string _rpc_link(pzmq *_pzmq, const std::string &data); + void _link(const std::string &zmq_url, const std::string &data) + { // printf("void _link run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -254,8 +274,9 @@ class StackFlow { virtual void link(const std::string &zmq_url, const std::string &raw); virtual void link(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_unlink(const std::string &data); - void _unlink(const std::string &zmq_url, const std::string &data) { + std::string _rpc_unlink(pzmq *_pzmq, const std::string &data); + void _unlink(const std::string &zmq_url, const std::string &data) + { // printf("void _unlink run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -264,8 +285,9 @@ class StackFlow { virtual void unlink(const std::string &zmq_url, const std::string &raw); virtual void unlink(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_exit(const std::string &data); - void _exit(const std::string &zmq_url, const std::string &data) { + std::string _rpc_exit(pzmq *_pzmq, const std::string &data); + void _exit(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) exit(zmq_url, data); @@ -273,8 +295,9 @@ class StackFlow { virtual int exit(const std::string &zmq_url, const std::string &raw); virtual int exit(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_work(const std::string &data); - void _work(const std::string &zmq_url, const std::string &data) { + std::string _rpc_work(pzmq *_pzmq, const std::string &data); + void _work(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) work(zmq_url, data); @@ -282,8 +305,9 @@ class StackFlow { virtual void work(const std::string &zmq_url, const std::string &raw); virtual void work(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_pause(const std::string &data); - void _pause(const std::string &zmq_url, const std::string &data) { + std::string _rpc_pause(pzmq *_pzmq, const std::string &data); + void _pause(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) pause(zmq_url, data); @@ -291,8 +315,9 @@ class StackFlow { virtual void pause(const std::string &zmq_url, const std::string &raw); virtual void pause(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_taskinfo(const std::string &data); - void _taskinfo(const std::string &zmq_url, const std::string &data) { + std::string _rpc_taskinfo(pzmq *_pzmq, const std::string &data); + void _taskinfo(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) taskinfo(zmq_url, data); @@ -305,7 +330,8 @@ class StackFlow { void user_output(const std::string &zmq_url, const std::string &request_id, const std::string &data); template int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id, - const std::string &zmq_url = "") { + const std::string &zmq_url = "") + { nlohmann::json out_body; out_body["request_id"] = request_id_; out_body["work_id"] = work_id; @@ -319,23 +345,28 @@ class StackFlow { out_body["error"] = error_msg; if (zmq_url.empty()) { pzmq _zmq(out_zmq_url_, ZMQ_PUSH); - return _zmq.send_data(out_body.dump()); + std::string out = out_body.dump(); + out += "\n"; + return _zmq.send_data(out); } else { pzmq _zmq(zmq_url, ZMQ_PUSH); - return _zmq.send_data(out_body.dump()); + std::string out = out_body.dump(); + out += "\n"; + return _zmq.send_data(out); } } - void llm_firework_exit() { + void llm_firework_exit() + { } - std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); std::string sys_sql_select(const std::string &key); void sys_sql_set(const std::string &key, const std::string &val); void sys_sql_unset(const std::string &key); int sys_register_unit(const std::string &unit_name); template - bool sys_release_unit(T workid) { + bool sys_release_unit(T workid) + { std::string _work_id; int _work_id_num; if constexpr (std::is_same::value) { @@ -348,7 +379,7 @@ class StackFlow { return false; } pzmq _call("sys"); - _call.call_rpc_action("release_unit", _work_id, [](const std::string &data) {}); + _call.call_rpc_action("release_unit", _work_id, [](pzmq *_pzmq, const std::string &data) {}); llm_task_channel_[_work_id_num].reset(); llm_task_channel_.erase(_work_id_num); // SLOGI("release work_id %s success", _work_id.c_str()); diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index fde8ecfc..e7bbb44a 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -5,8 +5,10 @@ */ #include "StackFlowUtil.h" #include +#include "pzmq.hpp" -std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { +std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) +{ std::string key_val; std::string format_val; // SLOGD("json_str: %s json_key:%s\n", json_str.c_str(), json_key.c_str()); @@ -81,7 +83,8 @@ std::string StackFlows::sample_json_str_get(const std::string &json_str, const s return key_val; } -int StackFlows::sample_get_work_id_num(const std::string &work_id) { +int StackFlows::sample_get_work_id_num(const std::string &work_id) +{ int a = work_id.find("."); if ((a == std::string::npos) || (a == work_id.length() - 1)) { return WORK_ID_NONE; @@ -89,7 +92,8 @@ int StackFlows::sample_get_work_id_num(const std::string &work_id) { return std::stoi(work_id.substr(a + 1)); } -std::string StackFlows::sample_get_work_id_name(const std::string &work_id) { +std::string StackFlows::sample_get_work_id_name(const std::string &work_id) +{ int a = work_id.find("."); if (a == std::string::npos) { return work_id; @@ -98,72 +102,41 @@ std::string StackFlows::sample_get_work_id_name(const std::string &work_id) { } } -std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &unit_name) { +std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &unit_name) +{ return unit_name + "." + std::to_string(work_id_num); } - -std::string StackFlows::sample_escapeString(const std::string &input) { +// clang-format off +std::string StackFlows::sample_escapeString(const std::string &input) +{ std::string escaped; for (char c : input) { switch (c) { - case '\n': - escaped += "\\n"; - break; - case '\t': - escaped += "\\t"; - break; - case '\\': - escaped += "\\\\"; - break; - case '\"': - escaped += "\\\""; - break; - case '\r': - escaped += "\\r"; - break; - case '\b': - escaped += "\\b"; - break; - default: - escaped += c; - break; + case '\n':escaped += "\\n" ;break; + case '\t':escaped += "\\t" ;break; + case '\\':escaped += "\\\\";break; + case '\"':escaped += "\\\"";break; + case '\r':escaped += "\\r" ;break; + case '\b':escaped += "\\b" ;break; + default :escaped += c ;break; } } return escaped; } -std::string StackFlows::sample_unescapeString(const std::string &input) { +std::string StackFlows::sample_unescapeString(const std::string &input) +{ std::string unescaped; for (size_t i = 0; i < input.length(); ++i) { if (input[i] == '\\' && i + 1 < input.length()) { switch (input[i + 1]) { - case 'n': - unescaped += '\n'; - ++i; - break; - case 't': - unescaped += '\t'; - ++i; - break; - case '\\': - unescaped += '\\'; - ++i; - break; - case '\"': - unescaped += '\"'; - ++i; - break; - case 'r': - unescaped += '\r'; - ++i; - break; - case 'b': - unescaped += '\b'; - ++i; - break; - default: - unescaped += input[i]; - break; + case 'n' :unescaped += '\n';++i;break; + case 't' :unescaped += '\t';++i;break; + case '\\':unescaped += '\\';++i;break; + case '\"':unescaped += '\"';++i;break; + case 'r' :unescaped += '\r';++i;break; + case 'b' :unescaped += '\b';++i;break; + default :unescaped += input[i];break; } } else { unescaped += input[i]; @@ -171,25 +144,20 @@ std::string StackFlows::sample_unescapeString(const std::string &input) { } return unescaped; } - +// clang-format on bool StackFlows::decode_stream(const std::string &in, std::string &out, - std::unordered_map &stream_buff) { - try { - int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); - std::string finish = StackFlows::sample_json_str_get(in, "finish"); - std::string delta = StackFlows::sample_json_str_get(in, "delta"); - stream_buff[index] = delta; - if (finish == "true") { - for (size_t i = 0; i < stream_buff.size(); i++) { - out += stream_buff.at(i); - } - stream_buff.clear(); - return false; - } else if (finish != "false") { - throw true; + std::unordered_map &stream_buff) +{ + int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); + std::string finish = StackFlows::sample_json_str_get(in, "finish"); + stream_buff[index] = StackFlows::sample_json_str_get(in, "delta"); + // sample find flage: false:true + if (finish.find("f") == std::string::npos) { + for (size_t i = 0; i < stream_buff.size(); i++) { + out += stream_buff.at(i); } - } catch (...) { stream_buff.clear(); + return false; } return true; } @@ -198,124 +166,23 @@ bool StackFlows::decode_stream(const std::string &in, std::string &out, #define BASE64_DECODE_OUT_SIZE(s) (((s)) / 4 * 3) #include /* BASE 64 encode table */ -static const char base64en[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', -}; +static const char base64en[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; #define BASE64_PAD '=' #define BASE64DE_FIRST '+' #define BASE64DE_LAST 'z' /* ASCII order for BASE 64 decode, -1 in unused character */ static const signed char base64de[] = { - /* '+', ',', '-', '.', '/', '0', '1', '2', */ - 62, - -1, - -1, - -1, - 63, - 52, - 53, - 54, - - /* '3', '4', '5', '6', '7', '8', '9', ':', */ - 55, - 56, - 57, - 58, - 59, - 60, - 61, - -1, - - /* ';', '<', '=', '>', '?', '@', 'A', 'B', */ - -1, - -1, - -1, - -1, - -1, - -1, - 0, - 1, - - /* 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', */ - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - - /* 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', */ - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - - /* 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', */ - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - - /* '[', '\', ']', '^', '_', '`', 'a', 'b', */ - -1, - -1, - -1, - -1, - -1, - -1, - 26, - 27, - - /* 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', */ - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - - /* 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', */ - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - - /* 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', */ - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, + 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, }; -static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) { +static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) +{ unsigned int i = 0, j = 0; - for (; i < inlen; i++) { int s = i % 3; - switch (s) { case 0: out[j++] = base64en[(in[i] >> 2) & 0x3F]; @@ -328,10 +195,8 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) out[j++] = base64en[in[i] & 0x3F]; } } - /* move back */ i -= 1; - /* check the last and add padding */ if ((i % 3) == 0) { out[j++] = base64en[(in[i] & 0x3) << 4]; @@ -341,34 +206,28 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) out[j++] = base64en[(in[i] & 0xF) << 2]; out[j++] = BASE64_PAD; } - return j; } -static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) { +static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) +{ unsigned int i = 0, j = 0; - for (; i < inlen; i++) { int c; int s = i % 4; - if (in[i] == '=') return j; - if (in[i] < BASE64DE_FIRST || in[i] > BASE64DE_LAST || (c = base64de[in[i] - BASE64DE_FIRST]) == -1) return -1; - switch (s) { case 0: out[j] = ((unsigned int)c << 2) & 0xFF; continue; case 1: out[j++] += ((unsigned int)c >> 4) & 0x3; - /* if not last char with padding */ if (i < (inlen - 3) || in[inlen - 2] != '=') out[j] = ((unsigned int)c & 0xF) << 4; continue; case 2: out[j++] += ((unsigned int)c >> 2) & 0xF; - /* if not last char with padding */ if (i < (inlen - 2) || in[inlen - 1] != '=') out[j] = ((unsigned int)c & 0x3) << 6; continue; @@ -380,12 +239,63 @@ static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) return j; } -int StackFlows::decode_base64(const std::string &in, std::string &out) { - out.resize(BASE64_DECODE_OUT_SIZE(in.length())); - return base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); +int StackFlows::decode_base64(const std::string &in, std::string &out) +{ + int out_size = BASE64_DECODE_OUT_SIZE(in.length()); + out.resize(out_size); + int ret = base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); + if ((ret > 0) && (ret != out_size)) out.erase(ret); + return ret; +} + +int StackFlows::encode_base64(const std::string &in, std::string &out) +{ + int out_size = BASE64_ENCODE_OUT_SIZE(in.length()); + out.resize(out_size); + int ret = base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); + if ((ret > 0) && (ret != out_size)) out.erase(ret); + return ret; +} + +std::string StackFlows::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) +{ + std::string value; + pzmq _call(unit_name); + _call.call_rpc_action(unit_action, data, [&value](pzmq *_pzmq, const std::string &raw) { value = raw; }); + return value; } -int StackFlows::encode_base64(const std::string &in, std::string &out) { - out.resize(BASE64_ENCODE_OUT_SIZE(in.length())); - return base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); +std::list StackFlows::get_config_file_paths(std::string &base_model_path, + std::string &base_model_config_path, + const std::string &mode_name) +{ + if (base_model_path.empty()) { + base_model_path = sample_unescapeString(unit_call("sys", "sql_select", "config_base_mode_path")); + if (base_model_path.empty()) { + base_model_path = "/opt/m5stack/data/"; + } + } + if (base_model_config_path.empty()) { + base_model_config_path = sample_unescapeString(unit_call("sys", "sql_select", "config_base_mode_config_path")); + if (base_model_config_path.empty()) { + base_model_config_path = "/opt/m5stack/etc/"; + } + } + std::string config_model_d = sample_unescapeString(unit_call("sys", "sql_select", "config_model_d")); + if (config_model_d.empty()) config_model_d = "/opt/m5stack/data/models/"; + + std::list config_file_paths; + config_file_paths.push_back(std::string("./") + mode_name + ".json"); + config_file_paths.push_back(std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + mode_name + std::string("/") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + mode_name + std::string("/") + std::string("./mode_") + mode_name + + ".json"); + config_file_paths.push_back(config_model_d + mode_name + ".json"); + config_file_paths.push_back(config_model_d + std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_config_path + mode_name + ".json"); + config_file_paths.push_back(base_model_config_path + std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + std::string("../share/") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + std::string("../share/") + std::string("./mode_") + mode_name + + ".json"); + return config_file_paths; } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h index 1b5df5f8..d1f6f8a7 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.h +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -5,8 +5,22 @@ */ #pragma once #include +#include #include +#include #define WORK_ID_NONE -100 + +#define RPC_PUSH_PARAM(_obj, _data1, _data2) \ + do { \ + _obj.resize(1 + _data1.size() + _data2.size()); \ + _obj[0] = (unsigned char)_data1.size(); \ + memcpy((void *)(_obj.data() + 1), (void *)_data1.data(), _data1.size()); \ + memcpy((void *)(_obj.data() + _data1.size() + 1), (void *)_data2.data(), _data2.size()); \ + } while (0) + +#define RPC_PARSE_TO_FIRST(_obj) _obj.substr(1, static_cast(_obj[0])) +#define RPC_PARSE_TO_SECOND(_obj) _obj.substr(static_cast(_obj[0]) + 1) + namespace StackFlows { std::string sample_json_str_get(const std::string &json_str, const std::string &json_key); int sample_get_work_id_num(const std::string &work_id); @@ -17,4 +31,6 @@ std::string sample_unescapeString(const std::string &input); bool decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff); int decode_base64(const std::string &in, std::string &out); int encode_base64(const std::string &in, std::string &out); +std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); +std::list get_config_file_paths(std::string &base_model_path, std::string &base_model_config_path, const std::string &mode_name); }; // namespace StackFlows diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index b334097d..1a7baf90 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -12,16 +12,24 @@ #include #include #include +#include +#include #define ZMQ_RPC_FUN (ZMQ_REP | 0x80) #define ZMQ_RPC_CALL (ZMQ_REQ | 0x80) + namespace StackFlows { class pzmq { - private: - const std::string unix_socket_head_ = "ipc://"; - const std::string rpc_url_head_ = "ipc:///tmp/rpc."; +public: + typedef std::function rpc_callback_fun; + typedef std::function msg_callback_fun; + +private: + const int rpc_url_head_length = 6; + std::string rpc_url_head_ = "ipc:///tmp/rpc."; void *zmq_ctx_; void *zmq_socket_; - std::unordered_map> zmq_fun_; + std::unordered_map zmq_fun_; + std::mutex zmq_fun_mtx_; std::atomic flage_; std::unique_ptr zmq_thread_; int mode_; @@ -29,41 +37,99 @@ class pzmq { std::string zmq_url_; int timeout_; - public: - pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) { + bool is_bind() + { + if ((mode_ == ZMQ_PUB) || (mode_ == ZMQ_PULL) || (mode_ == ZMQ_RPC_FUN)) + return true; + else + return false; + } + +public: + pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) + { } pzmq(const std::string &server) - : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) { + : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) + { + if (server.find("://") != std::string::npos) { + rpc_url_head_.clear(); + } } - pzmq(const std::string &url, int mode, const std::function &raw_call = nullptr) - : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) { + pzmq(const std::string &url, int mode, const msg_callback_fun &raw_call = nullptr) + : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) + { + if ((url[0] != 'i') && (url[1] != 'p')) { + rpc_url_head_.clear(); + } if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call); } - void set_timeout(int ms) { + void set_timeout(int ms) + { timeout_ = ms; } - int get_timeout() { + int get_timeout() + { return timeout_; } - int register_rpc_action(const std::string &action, - const std::function &raw_call) { + std::string get_zmq_url() + { + if ((!flage_.load()) && is_bind() && (zmq_url_.find("tcp://") != std::string::npos) && + (zmq_url_.find(":*") != std::string::npos)) { + std::vector zmq_url(zmq_url_.length() + 8, 0); + size_t addr_len = zmq_url.size(); + if (zmq_getsockopt(zmq_socket_, ZMQ_LAST_ENDPOINT, zmq_url.data(), &addr_len) != 0) { + return zmq_url_; + } + zmq_url_ = std::string(zmq_url.data()); + } + return zmq_url_; + } + std::string _rpc_list_action(pzmq *self, const std::string &_None) + { + std::string action_list; + action_list.reserve(128); + action_list = "{\"actions\":["; + for (auto i = zmq_fun_.begin();;) { + action_list += "\""; + action_list += i->first; + action_list += "\""; + if (++i == zmq_fun_.end()) { + action_list += "]}"; + break; + } else { + action_list += ","; + } + } + return action_list; + } + int register_rpc_action(const std::string &action, const rpc_callback_fun &raw_call) + { + int ret = 0; + std::unique_lock lock(zmq_fun_mtx_); if (zmq_fun_.find(action) != zmq_fun_.end()) { zmq_fun_[action] = raw_call; - return 0; + return ret; } - std::string url = rpc_url_head_ + rpc_server_; - if (!flage_) { - flage_ = true; - zmq_ctx_shutdown(zmq_ctx_); - zmq_thread_->join(); - close_zmq(); + if (zmq_fun_.empty()) { + std::string url = rpc_url_head_ + rpc_server_; + mode_ = ZMQ_RPC_FUN; + zmq_fun_["list_action"] = + std::bind(&pzmq::_rpc_list_action, this, std::placeholders::_1, std::placeholders::_2); + ret = creat(url); } zmq_fun_[action] = raw_call; - mode_ = ZMQ_RPC_FUN; - return creat(url); + return ret; } - int call_rpc_action(const std::string &action, const std::string &data, - const std::function &raw_call) { + void unregister_rpc_action(const std::string &action) + { + std::unique_lock lock(zmq_fun_mtx_); + if (zmq_fun_.find(action) != zmq_fun_.end()) { + zmq_fun_.erase(action); + } + } + int call_rpc_action(const std::string &action, const std::string &data, const msg_callback_fun &raw_call) + { int ret; zmq_msg_t msg; zmq_msg_init(&msg); @@ -83,8 +149,10 @@ class pzmq { zmq_send(zmq_socket_, data.c_str(), data.length(), 0); } // action - { zmq_msg_recv(&msg, zmq_socket_, 0); } - raw_call(std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); + { + zmq_msg_recv(&msg, zmq_socket_, 0); + } + raw_call(this, std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); } catch (int e) { ret = e; } @@ -92,7 +160,8 @@ class pzmq { close_zmq(); return ret; } - int creat(const std::string &url, const std::function &raw_call = nullptr) { + int creat(const std::string &url, const msg_callback_fun &raw_call = nullptr) + { zmq_url_ = url; do { zmq_ctx_ = zmq_ctx_new(); @@ -117,7 +186,6 @@ class pzmq { case ZMQ_PUSH: { int reconnect_interval = 100; zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL, &reconnect_interval, sizeof(reconnect_interval)); - int max_reconnect_interval = 1000; // 5 seconds zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL_MAX, &max_reconnect_interval, sizeof(max_reconnect_interval)); @@ -137,7 +205,8 @@ class pzmq { } return 0; } - int check_zmq_errno(int code) { + int check_zmq_errno(int code) + { if (code == -1) { // SLOGE("Error occurred: %s", zmq_strerror(errno)); switch (errno) { @@ -162,44 +231,49 @@ class pzmq { } return code; } - int send_data(const std::string &raw) { + int send_data(const std::string &raw) + { return zmq_send(zmq_socket_, raw.c_str(), raw.length(), 0); } - int send_data(const char *raw, int size) { + int send_data(const char *raw, int size) + { return zmq_send(zmq_socket_, raw, size, 0); } - inline int creat_pub(const std::string &url) { + inline int creat_pub(const std::string &url) + { return zmq_bind(zmq_socket_, url.c_str()); } - inline int creat_push(const std::string &url) { - int timeout = 3000; - zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout, sizeof(timeout)); - zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout, sizeof(timeout)); + inline int creat_push(const std::string &url) + { + zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout_, sizeof(timeout_)); return zmq_connect(zmq_socket_, url.c_str()); } - inline int creat_pull(const std::string &url, const std::function &raw_call) { + inline int creat_pull(const std::string &url, const msg_callback_fun &raw_call) + { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int subscriber_url(const std::string &url, const std::function &raw_call) { + inline int subscriber_url(const std::string &url, const msg_callback_fun &raw_call) + { int ret = zmq_connect(zmq_socket_, url.c_str()); zmq_setsockopt(zmq_socket_, ZMQ_SUBSCRIBE, "", 0); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int creat_rep(const std::string &url, const std::function &raw_call) { + inline int creat_rep(const std::string &url, const msg_callback_fun &raw_call) + { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int creat_req(const std::string &url) { - int pos = url.find(unix_socket_head_); - if (pos != std::string::npos) { - std::string socket_file = url.substr(pos + unix_socket_head_.length()); + inline int creat_req(const std::string &url) + { + if (!rpc_url_head_.empty()) { + std::string socket_file = url.substr(rpc_url_head_length); if (access(socket_file.c_str(), F_OK) != 0) { return -1; } @@ -208,7 +282,8 @@ class pzmq { zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout_, sizeof(timeout_)); return zmq_connect(zmq_socket_, url.c_str()); } - void zmq_event_loop(const std::function &raw_call) { + void zmq_event_loop(const msg_callback_fun &raw_call) + { int ret; zmq_pollitem_t items[1]; if (mode_ == ZMQ_PULL) { @@ -230,10 +305,8 @@ class pzmq { continue; } } - // SLOGI("zmq_msg_recv"); ret = zmq_msg_recv(&msg, zmq_socket_, 0); if (ret <= 0) { - // SLOGE("zmq_connect false:%s", zmq_strerror(zmq_errno())); zmq_msg_close(&msg); continue; } @@ -245,26 +318,26 @@ class pzmq { std::string _raw_data((const char *)zmq_msg_data(&msg1), zmq_msg_size(&msg1)); std::string retval; try { - retval = zmq_fun_.at(raw_data)(_raw_data); + std::unique_lock lock(zmq_fun_mtx_); + retval = zmq_fun_.at(raw_data)(this, _raw_data); } catch (...) { retval = "NotAction"; } zmq_send(zmq_socket_, retval.c_str(), retval.length(), 0); zmq_msg_close(&msg1); } else { - // SLOGI("zmq_msg:%s", zmq_msg_data(&msg)); - raw_call(raw_data); + raw_call(this, raw_data); } zmq_msg_close(&msg); } } - void close_zmq() { + void close_zmq() + { zmq_close(zmq_socket_); zmq_ctx_destroy(zmq_ctx_); - int pos = zmq_url_.find(unix_socket_head_); if ((mode_ == ZMQ_PUB) || (mode_ == ZMQ_PULL) || (mode_ == ZMQ_RPC_FUN)) { - if (pos != std::string::npos) { - std::string socket_file = zmq_url_.substr(pos + unix_socket_head_.length()); + if (!rpc_url_head_.empty()) { + std::string socket_file = zmq_url_.substr(rpc_url_head_length); if (access(socket_file.c_str(), F_OK) == 0) { remove(socket_file.c_str()); } @@ -273,16 +346,100 @@ class pzmq { zmq_socket_ = NULL; zmq_ctx_ = NULL; } - ~pzmq() { + ~pzmq() + { if (!zmq_socket_) { return; } + flage_ = true; + zmq_ctx_shutdown(zmq_ctx_); if (zmq_thread_) { - flage_ = true; - zmq_ctx_shutdown(zmq_ctx_); zmq_thread_->join(); } close_zmq(); } + +private: + std::shared_ptr context_ptr_; + std::weak_ptr wcontext_ptr_; + void *ctx_; + +public: + // context + void *context() + { + return ctx_; + } + void setContext(void *ctx) + { + ctx_ = ctx; + } + template + T *newContext() + { + ctx_ = new T; + return (T *)ctx_; + } + template + T *getContext() + { + return (T *)ctx_; + } + template + void deleteContext() + { + if (ctx_) { + delete (T *)ctx_; + ctx_ = NULL; + } + } + + // contextPtr + std::shared_ptr contextPtr() + { + return context_ptr_; + } + void setContextPtr(const std::shared_ptr &ctx) + { + context_ptr_ = ctx; + } + void setContextPtr(std::shared_ptr &&ctx) + { + context_ptr_ = std::move(ctx); + } + template + std::shared_ptr newContextPtr() + { + context_ptr_ = std::make_shared(); + return std::static_pointer_cast(context_ptr_); + } + template + std::shared_ptr getContextPtr() + { + return std::static_pointer_cast(context_ptr_); + } + void deleteContextPtr() + { + context_ptr_.reset(); + } + + // wcontextPtr + std::shared_ptr wcontextPtr() + { + return wcontext_ptr_.lock(); + } + void wsetContextPtr(const std::shared_ptr &ctx) + { + wcontext_ptr_ = ctx; + } + template + std::shared_ptr wgetContextPtr() + { + return std::static_pointer_cast(wcontext_ptr_.lock()); + } + void wdeleteContextPtr() + { + wcontext_ptr_.reset(); + } }; }; // namespace StackFlows \ No newline at end of file diff --git a/ext_components/ax-samples/Kconfig b/ext_components/ax-samples/Kconfig new file mode 100644 index 00000000..d21f5ac8 --- /dev/null +++ b/ext_components/ax-samples/Kconfig @@ -0,0 +1,6 @@ +menuconfig AX_SAMPLES_ENABLED + bool "Enable ax-samples support" + default n + depends on AX_MSP_ENABLED + help + enable axera ax-samples support diff --git a/ext_components/ax-samples/SConstruct b/ext_components/ax-samples/SConstruct new file mode 100644 index 00000000..1408b20b --- /dev/null +++ b/ext_components/ax-samples/SConstruct @@ -0,0 +1,58 @@ +# component2/SConscript +Import("env") +import os +from pathlib import Path + +with open(env["PROJECT_TOOL_S"]) as f: + exec(f.read()) + +_SDK_PATH = os.path.normpath( + os.environ.get("SDK_PATH", str(Path(os.getcwd()) / ".." / "..")) +) +env["GIT_REPO_LISTS"]["ax-samples"] = { + "url": "https://github.com/AXERA-TECH/ax-samples.git", + "commit": "146154ca03929a0e119ca53650e3b05fa40c3e7a", + "path": str(Path(_SDK_PATH) / "github_source" / "ax-samples"), +} + +if "CONFIG_AX_SAMPLES_ENABLED" in os.environ: + check_component("ax-samples") + SRCS = [] + INCLUDE = [] + PRIVATE_INCLUDE = [] + REQUIREMENTS = [] + STATIC_LIB = [] + DYNAMIC_LIB = [] + DEFINITIONS = [] + DEFINITIONS_PRIVATE = [] + LDFLAGS = [] + LINK_SEARCH_PATH = [] + + INCLUDE += [ + os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/base"), + os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/utilities"), + ] + if "CONFIG_AX_620E_MSP_ENABLED" in os.environ: + INCLUDE += [ + os.path.join( + env["GIT_REPO_LISTS"]["ax-samples"]["path"], + "examples/ax620e/middleware", + ) + ] + + env["COMPONENTS"].append( + { + "target": os.path.basename(env["component_dir"]), + "SRCS": SRCS, + "INCLUDE": INCLUDE, + "PRIVATE_INCLUDE": PRIVATE_INCLUDE, + "REQUIREMENTS": REQUIREMENTS, + "STATIC_LIB": STATIC_LIB, + "DYNAMIC_LIB": DYNAMIC_LIB, + "DEFINITIONS": DEFINITIONS, + "DEFINITIONS_PRIVATE": DEFINITIONS_PRIVATE, + "LDFLAGS": LDFLAGS, + "LINK_SEARCH_PATH": LINK_SEARCH_PATH, + "REGISTER": "static", + } + ) diff --git a/ext_components/ax_msp/SConstruct b/ext_components/ax_msp/SConstruct index a49c7d75..7e73e778 100644 --- a/ext_components/ax_msp/SConstruct +++ b/ext_components/ax_msp/SConstruct @@ -5,45 +5,7 @@ with open(env['PROJECT_TOOL_S']) as f: exec(f.read()) if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: - MSP_PATH = os.path.join(os.environ['GIT_REPO_PATH'], 'm5stack_msp') - if 'MSP_PATH' in os.environ: - MSP_PATH = os.environ['MSP_PATH'] - else: - if not os.path.exists(MSP_PATH): - zip_file = MSP_PATH + '.tar.gz' - zip_file_extrpath = MSP_PATH + '_tmp' - down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_msp.tar.gz" - - if 'CONFIG_REPO_AUTOMATION' in os.environ: - down = 'y' - else: - down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('m5stack_msp.tar.gz')) - down = down.lower() - if down == 'y': - # from git import Repo - import requests - import parse - import tarfile - import shutil - try: - # Downloading via HTTP (more common) - if not os.path.exists(zip_file): - response = requests.get(down_url) - if response.status_code == 200: - with open(zip_file, 'wb') as file: - file.write(response.content) - else: - env.Fatal("{} down failed".format(down_url)) - with tarfile.open(zip_file, 'r:gz') as tar: - tar.extractall(path=zip_file_extrpath) - shutil.move(zip_file_extrpath, MSP_PATH) - # shutil.rmtree(zip_file_extrpath) - print("The {} download successful.".format(down_url)) - except Exception as e: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - env.Fatal("Cloning failed.: {}".format(e)) - else: - env.Fatal('Please manually download {} to {} .'.format(down_url, zip_file)) + MSP_PATH = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_msp.tar.gz", 'm5stack_msp.tar.gz') SRCS=[] INCLUDE=[] PRIVATE_INCLUDE=[] @@ -55,14 +17,18 @@ if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: LDFLAGS=[] LINK_SEARCH_PATH=[] - INCLUDE += [os.path.join(MSP_PATH, 'out/arm64_glibc/include'), os.path.join(MSP_PATH, 'sample/common')] + env["MSP_PATH"] = MSP_PATH + INCLUDE += [os.path.join(MSP_PATH, 'out/arm64_glibc/include'), os.path.join(MSP_PATH, 'sample'), os.path.join(MSP_PATH, 'sample/common'), os.path.join(MSP_PATH, 'sample/rtsp')] LINK_SEARCH_PATH += [os.path.join(MSP_PATH, 'out/arm64_glibc/lib')] REQUIREMENTS += ['dl', 'm', 'pthread'] - third_party = ['cmdline','drm','faac','fdk-aac','ffmpeg','libsamplerate','live','mp4','opencv','openssl','optee','opus','swupdate','tinyalsa'] + third_party = ['drm','faac','fdk-aac','ffmpeg','libsamplerate','mp4','opencv','opus','tinyalsa'] for dirn in third_party: INCLUDE.append(os.path.join(MSP_PATH,'third-party',dirn,'include')) LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party',dirn,'lib/arm64/glibc')) - + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/arm64/glibc/include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/live/out/arm64/glibc/lib')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/openssl/arm64/include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/openssl/arm64/glibc/lib')) LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'app/lib')) env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), diff --git a/projects/llm_framework/.gitignore b/projects/llm_framework/.gitignore index e31df2ba..20085e28 100644 --- a/projects/llm_framework/.gitignore +++ b/projects/llm_framework/.gitignore @@ -8,3 +8,4 @@ static_lib** *.tar.gz m5stack_* debian* +gitignore_* \ No newline at end of file diff --git a/projects/llm_framework/README.md b/projects/llm_framework/README.md index 3f790874..f59c9045 100644 --- a/projects/llm_framework/README.md +++ b/projects/llm_framework/README.md @@ -167,3 +167,65 @@ waiting return status: ``` **Please enjoy the large model voice assistant.** + + +## cv mode +1、reset + +send : + +```json +{ + "request_id": "11212155", + "work_id": "sys", + "action": "reset" +} +``` +waiting reset over! + +2、init camera unit + +send : + +```json +{ + "request_id":"4", + "work_id":"camera", + "action":"setup", + "object":"camera.setup", + "data":{ + "response_format":"camera.raw", + "input":"/dev/video0", + "enoutput":false, + "frame_width":320, + "frame_height":320 + } +} +``` +waiting return work_id: +```json +{"created":1731488371,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"camera.1000"} +``` + +2、init yolo unit + +send : + +```json +{ + "request_id":"5", + "work_id":"yolo", + "action":"setup", + "object":"yolo.setup", + "data":{ + "model":"yolo11n_anquanmao", + "response_format":"yolo.yolobox", + "input":"camera.1000", + "enoutput":true + } +} +``` +waiting return work_id: +```json +{"created":1731488371,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"5","work_id":"yolo.1001"} +``` diff --git a/projects/llm_framework/SConstruct b/projects/llm_framework/SConstruct index af1dbcdc..4c0f64e2 100644 --- a/projects/llm_framework/SConstruct +++ b/projects/llm_framework/SConstruct @@ -8,6 +8,10 @@ os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/ version = 'v0.0.5' static_lib = 'static_lib' update = False + +with open(str(Path(os.getcwd())/'..'/'..'/'SDK'/'tools'/'scons'/'project.py')) as f: + exec(f.read()) + if not os.path.exists(static_lib): update = True else: @@ -17,50 +21,9 @@ else: update = True except: update = True - if update: - zip_file = static_lib + '_{}.tar.gz'.format(version) - zip_file_extrpath = static_lib + '_tmp' + with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/static_lib_{}.tar.gz".format(version) - - if 'CONFIG_REPO_AUTOMATION' in os.environ: - down = 'y' - else: - down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('static_lib_{}.tar.gz').format(version)) - down = down.lower() - if down == 'y': - # from git import Repo - import requests - import parse - import tarfile - import shutil - try: - # Downloading via HTTP (more common) - if not os.path.exists(zip_file): - response = requests.get(down_url) - if response.status_code == 200: - with open(zip_file, 'wb') as file: - file.write(response.content) - else: - print("{} down failed".format(down_url)) - try: - shutil.rmtree(static_lib) - except: - pass - with tarfile.open(zip_file, 'r:gz') as tar: - tar.extractall(path=zip_file_extrpath) - shutil.move(zip_file_extrpath, static_lib) - try: - os.remove(zip_file) - except: - pass - print("The {} download successful.".format(down_url)) - except Exception as e: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - print("Cloning failed.: {}".format(e)) - else: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - - -with open(str(Path(os.getcwd())/'..'/'..'/'SDK'/'tools'/'scons'/'project.py')) as f: - exec(f.read()) + down_path = check_wget_down(down_url, "static_lib_{}.tar.gz".format(version)) + shutil.move(down_path, static_lib) diff --git a/projects/llm_framework/config_defaults.mk b/projects/llm_framework/config_defaults.mk index e09f3056..a61fc047 100644 --- a/projects/llm_framework/config_defaults.mk +++ b/projects/llm_framework/config_defaults.mk @@ -13,7 +13,7 @@ CONFIG_DEVICE_UART_ENABLED=y # CONFIG_LHV_WITH_EVPP=y CONFIG_UTILITIES_ENABLED=y CONFIG_BACKWARD_CPP_ENABLED=y -CONFIG_TOOLCHAIN_FLAGS="-g" +CONFIG_TOOLCHAIN_FLAGS="-O0" CONFIG_EVENTPP_ENABLED=y CONFIG_LHV_ENABLED=y CONFIG_LHV_WITH_EVPP=y @@ -21,3 +21,5 @@ CONFIG_EVENTPP_ENABLED=y CONFIG_STACKFLOW_ENABLED=y CONFIG_UTILITIES_BASE64_ENABLED=y CONFIG_SINGLE_HEADER_LIBS_ENABLED=y +CONFIG_AX_SAMPLES_ENABLED=y +CONFIG_SIMDJSON_COMPENENT_ENABLED=y diff --git a/projects/llm_framework/main_melotts/src/runner/utils/checker.h b/projects/llm_framework/include/utils/checker.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/checker.h rename to projects/llm_framework/include/utils/checker.h diff --git a/projects/llm_framework/main_melotts/src/runner/utils/io.hpp b/projects/llm_framework/include/utils/io.hpp similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/io.hpp rename to projects/llm_framework/include/utils/io.hpp diff --git a/projects/llm_framework/main_melotts/src/runner/utils/logger.h b/projects/llm_framework/include/utils/logger.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/logger.h rename to projects/llm_framework/include/utils/logger.h diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 9c834ff9..116a52da 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -13,13 +13,14 @@ #include using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ main_exit_flage = 1; } typedef std::function task_callback_t; class llm_task { - private: - public: +private: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -27,11 +28,13 @@ class llm_task { bool enoutput_; bool enstream_; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -52,37 +55,48 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } return 0; } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { std::cout << msg << std::endl; if (out_callback_) out_callback_(std::string("hello"), true); } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - public: - llm_llm() : StackFlow("test") { +public: + llm_llm() : StackFlow("test") + { task_count_ = 1; } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_channel->enstream_) { static int count = 0; nlohmann::json data_body; @@ -100,21 +114,50 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; @@ -122,7 +165,8 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -146,10 +190,13 @@ class llm_llm : public StackFlow { if (ret == 0) { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); - llm_channel->subscriber_work_id("", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + llm_channel->subscriber_work_id( + "", + std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_[work_id_num] = llm_task_obj; send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -161,7 +208,8 @@ class llm_llm : public StackFlow { } } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { @@ -181,12 +229,13 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { @@ -202,19 +251,22 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index af59ecb3..c947a9c5 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -24,7 +24,7 @@ INCLUDE += [ADir('../include/sherpa'), ADir('../include/sherpa/sherpa-ncnn')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')] REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] -STATIC_FILES += [AFile('sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json'), AFile('sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_asr', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json b/projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json similarity index 100% rename from projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json rename to projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json diff --git a/projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json b/projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json similarity index 100% rename from projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json rename to projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 4ed598f1..0648433b 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -24,7 +24,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_asr will be exit!"); main_exit_flage = 1; } @@ -41,12 +42,12 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: +private: sherpa_ncnn::RecognizerConfig mode_config_; std::unique_ptr recognizer_; std::unique_ptr recognizer_stream_; - public: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -62,7 +63,8 @@ class llm_task { std::function pause; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -84,14 +86,14 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -163,7 +165,7 @@ class llm_task { mode_config_.model_config.decoder_bin = base_model + mode_config_.model_config.decoder_bin; mode_config_.model_config.joiner_param = base_model + mode_config_.model_config.joiner_param; mode_config_.model_config.joiner_bin = base_model + mode_config_.model_config.joiner_bin; - recognizer_ = std::make_unique(mode_config_); + recognizer_ = std::make_unique(mode_config_); } catch (...) { SLOGE("config false"); return -3; @@ -171,11 +173,13 @@ class llm_task { return 0; } - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) { + void sys_pcm_on_data(const std::string &raw) + { static int count = 0; if (count < delay_audio_frame_) { buffer_write_char(pcmdata, raw.c_str(), raw.length()); @@ -225,22 +229,26 @@ class llm_task { } } - void kws_awake() { + void kws_awake() + { awake_flage_ = true; } - bool delete_model() { + bool delete_model() + { recognizer_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { ensleep_ = false; awake_flage_ = false; pcmdata = buffer_create(); } - ~llm_task() { + ~llm_task() + { if (recognizer_stream_) { recognizer_stream_.reset(); } @@ -251,38 +259,30 @@ class llm_task { #undef CONFIG_AUTO_SET class llm_asr : public StackFlow { - public: +public: enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_TASK_PAUSE }; - private: +private: int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_asr::_load_config success"); - return 0; - } - } - public: - llm_asr() : StackFlow("asr") { +public: + llm_asr() : StackFlow("asr") + { task_count_ = 1; - repeat_event(1000, std::bind(&llm_asr::_load_config, this)); event_queue_.appendListener( EVENT_TASK_PAUSE, std::bind(&llm_asr::_task_pause, this, std::placeholders::_1, std::placeholders::_2)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string tmp_msg1; const std::string *next_data = &data; if (finish) { @@ -304,7 +304,8 @@ class llm_asr : public StackFlow { } } - int decode_wav(const std::string &in, std::string &out) { + int decode_wav(const std::string &in, std::string &out) + { int post = 0; if (in.length() > 10) for (int i = 0; i < in.length() - 4; i++) { @@ -322,25 +323,49 @@ class llm_asr : public StackFlow { return 0; } - int decode_mp3(const std::string &in, std::string &out) { + int decode_mp3(const std::string &in, std::string &out) + { return 0; } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } std::string tmp_msg1; const std::string *next_data = &data; int ret; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; @@ -364,7 +389,8 @@ class llm_asr : public StackFlow { llm_task_obj->sys_pcm_on_data((*next_data)); } - void _task_pause(const std::string &work_id, const std::string &data) { + void _task_pause(const std::string &work_id, const std::string &data) + { int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { return; @@ -377,26 +403,44 @@ class llm_asr : public StackFlow { } } - void task_pause(const std::string &work_id, const std::string &data) { + void task_pause(const std::string &work_id, const std::string &data) + { event_queue_.enqueue(EVENT_TASK_PAUSE, work_id, ""); } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_work(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->kws_awake(); if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_)); task_work(llm_task_obj, llm_channel); } - void work(const std::string &work_id, const std::string &object, const std::string &data) override { + void work(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -411,7 +455,8 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void pause(const std::string &work_id, const std::string &object, const std::string &data) override { + void pause(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -426,7 +471,8 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -454,26 +500,29 @@ class llm_asr : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); llm_task_obj->pause = std::bind(&llm_asr::task_pause, this, work_id, ""); - SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); - SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_asr::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_asr::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { - audio_url_ = unit_call("audio", "cap", input); - llm_channel->subscriber( - audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + audio_url_ = unit_call("audio", "cap", input); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } else if (input.find("asr") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_asr::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_asr::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_task_obj->ensleep_ = true; task_pause(work_id, ""); llm_channel->subscriber_work_id( - input, std::bind(&llm_asr::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_asr::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -490,7 +539,8 @@ class llm_asr : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::link:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -505,14 +555,18 @@ class llm_asr : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("sys") != std::string::npos) { if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data); - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { llm_task_obj->ensleep_ = true; - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_asr::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_asr::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { @@ -525,7 +579,8 @@ class llm_asr : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -549,9 +604,10 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::taskinfo:%s", data.c_str()); - + nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { @@ -571,12 +627,13 @@ class llm_asr : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("asr.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::exit:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -596,24 +653,25 @@ class llm_asr : public StackFlow { return 0; } - ~llm_asr() { + ~llm_asr() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); if (iteam->second->audio_flage_) { unit_call("audio", "cap_stop", "None"); } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 8e6721e6..afbad26d 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -27,6 +27,7 @@ REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_audio', 'ax_audio_3a', 'ax_fdk' REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] STATIC_FILES += [AFile('audio.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_audio', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_audio/audio.json b/projects/llm_framework/main_audio/audio.json index de7fb6dc..eab11c98 100644 --- a/projects/llm_framework/main_audio/audio.json +++ b/projects/llm_framework/main_audio/audio.json @@ -15,7 +15,7 @@ "play_param": { "card": 0, "device": 1, - "volume": 1.0, + "volume": 0.5, "channel": 1, "rate": 16000, "bit": 16, diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 8b646145..e95c1334 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -15,7 +15,8 @@ #include "../../../../SDK/components/utilities/include/sample_log.h" int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -31,7 +32,7 @@ static void __sigint(int iSigNo) { using namespace StackFlows; class llm_audio : public StackFlow { - private: +private: enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_QUEUE_PLAY }; std::atomic_bool audio_clear_flage_; std::string sys_pcm_cap_channel = "ipc:///tmp/llm/pcm.cap.socket"; @@ -42,11 +43,13 @@ class llm_audio : public StackFlow { std::unique_ptr audio_cap_thread_; std::mutex ax_play_mtx; - static void on_cap_sample(const char *data, int size) { + static void on_cap_sample(const char *data, int size) + { self->pub_ctx_->send_data((const char *)data, size); } - void hw_queue_play(const std::string &audio_data, const std::string &None) { + void hw_queue_play(const std::string &audio_data, const std::string &None) + { if (audio_clear_flage_) { return; } @@ -55,25 +58,29 @@ class llm_audio : public StackFlow { play_config.bit, audio_data.c_str(), audio_data.length()); } - void hw_play(const std::string &audio_data) { + void hw_play(const std::string &audio_data) + { std::lock_guard guard(ax_play_mtx); ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, play_config.bit, audio_data.c_str(), audio_data.length()); } - void hw_cap() { + void hw_cap() + { ax_cap_start(cap_config.card, cap_config.device, cap_config.volume, cap_config.channel, cap_config.rate, cap_config.bit, llm_audio::on_cap_sample); } - void _play(const std::string &audio_data) { + void _play(const std::string &audio_data) + { if (audio_play_thread_) { _play_stop(); } audio_play_thread_ = std::make_unique(std::bind(&llm_audio::hw_play, this, audio_data)); } - void _play_stop() { + void _play_stop() + { ax_close_play(); if (audio_play_thread_) { audio_play_thread_->join(); @@ -81,14 +88,16 @@ class llm_audio : public StackFlow { } } - void _cap() { + void _cap() + { if (!audio_cap_thread_) { pub_ctx_ = std::make_unique(sys_pcm_cap_channel, ZMQ_PUB); audio_cap_thread_ = std::make_unique(std::bind(&llm_audio::hw_cap, this)); } } - void _cap_stop() { + void _cap_stop() + { ax_close_cap(); if (audio_cap_thread_) { audio_cap_thread_->join(); @@ -97,34 +106,43 @@ class llm_audio : public StackFlow { } } - public: - llm_audio() : StackFlow("audio") { +public: + llm_audio() : StackFlow("audio") + { event_queue_.appendListener( EVENT_QUEUE_PLAY, std::bind(&llm_audio::hw_queue_play, this, std::placeholders::_1, std::placeholders::_2)); - setup("", "{\"object\":\"audio.play\",\"data\":{\"None\":\"None\"}}"); - setup("", "{\"object\":\"audio.cap\",\"data\":{\"None\":\"None\"}}"); + setup("", "audio.play", "{\"None\":\"None\"}"); + setup("", "audio.cap", "{\"None\":\"None\"}"); self = this; cap_status_ = 0; - rpc_ctx_->register_rpc_action("play", std::bind(&llm_audio::play, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("queue_play", std::bind(&llm_audio::enqueue_play, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("play_stop", std::bind(&llm_audio::play_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("queue_play_stop", - std::bind(&llm_audio::queue_play_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("audio_status", std::bind(&llm_audio::audio_status, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap", std::bind(&llm_audio::cap, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap_stop", std::bind(&llm_audio::cap_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("play", + std::bind(&llm_audio::play, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "play_raw", std::bind(&llm_audio::play_raw, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "queue_play", std::bind(&llm_audio::enqueue_play, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "play_stop", std::bind(&llm_audio::play_stop, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("queue_play_stop", std::bind(&llm_audio::queue_play_stop, this, + std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "audio_status", std::bind(&llm_audio::audio_status, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("cap", + std::bind(&llm_audio::cap, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "cap_stop", std::bind(&llm_audio::cap_stop, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1, std::placeholders::_2)); } - - int setup(const std::string &zmq_url, const std::string &raw) override { - std::string object = sample_json_str_get(raw, "object"); - std::string data = sample_json_str_get(raw, "data"); + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json config_body; nlohmann::json file_body; nlohmann::json error_body; - std::list config_file_paths; - config_file_paths.push_back("./audio.json"); - config_file_paths.push_back("/opt/m5stack/share/audio.json"); + std::string base_model_path; + std::string base_model_config_path; + std::list config_file_paths = + get_config_file_paths(base_model_path, base_model_config_path, "audio"); try { config_body = nlohmann::json::parse(data); for (auto file_name : config_file_paths) { @@ -288,7 +306,8 @@ class llm_audio : public StackFlow { return -1; } - bool destream(const std::string &indata, std::string &outdata) { + bool destream(const std::string &indata, std::string &outdata) + { static std::unordered_map _stream_data_buff; int index = std::stoi(sample_json_str_get(indata, "index")); std::string finish = sample_json_str_get(indata, "finish"); @@ -308,7 +327,8 @@ class llm_audio : public StackFlow { return true; } - std::string dewav(const std::string &rawdata) { + std::string dewav(const std::string &rawdata) + { int post = 0; if (rawdata.length() > 10) for (int i = 0; i < rawdata.length() - 4; i++) { @@ -326,15 +346,18 @@ class llm_audio : public StackFlow { return std::string("playwav"); } - std::string demp3(const std::string &rawdata) { + std::string demp3(const std::string &rawdata) + { return LLM_NONE; } - std::string depcm(const std::string &rawdata) { + std::string depcm(const std::string &rawdata) + { return LLM_NONE; } - std::string parse_data(const std::string &object, const std::string &data) { + std::string parse_data(const std::string &object, const std::string &data) + { std::string _data; bool s = (object.find("stream") == std::string::npos) ? false : true; bool wav = (object.find("wav") == std::string::npos) ? false : true; @@ -354,24 +377,33 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string play(const std::string &rawdata) { - if (rawdata.size() < 3) return LLM_NONE; - if ((rawdata[0] == '{') && (rawdata[rawdata.size() - 1] == '}')) { - std::string src_data = sample_unescapeString(sample_json_str_get(rawdata, "raw_data")); - return parse_data(sample_json_str_get(src_data, "object"), sample_json_str_get(src_data, "data")); - } else { - _play(rawdata); - } + std::string play(pzmq *_pzmq, const std::string &rawdata) + { + std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); + std::string audio_json = RPC_PARSE_TO_SECOND(rawdata); + std::string ret_val = + parse_data(sample_json_str_get(audio_json, "object"), sample_json_str_get(audio_json, "data")); + request_id_ = sample_json_str_get(audio_json, "request_id"); + send(LLM_NONE, LLM_NONE, LLM_NO_ERROR, sample_json_str_get(audio_json, "work_id"), zmq_url); + return ret_val; + } + + std::string play_raw(pzmq *_pzmq, const std::string &rawdata) + { + if (rawdata.empty()) return std::string("rawdata empty"); + _play(rawdata); return LLM_NONE; } - std::string enqueue_play(const std::string &rawdata) { + std::string enqueue_play(pzmq *_pzmq, const std::string &rawdata) + { audio_clear_flage_ = false; event_queue_.enqueue(EVENT_QUEUE_PLAY, rawdata, ""); return LLM_NONE; } - std::string audio_status(const std::string &rawdata) { + std::string audio_status(pzmq *_pzmq, const std::string &rawdata) + { if (rawdata == "play") { if (ax_play_status()) { return std::string("None"); @@ -403,17 +435,20 @@ class llm_audio : public StackFlow { } } - std::string play_stop(const std::string &rawdata) { + std::string play_stop(pzmq *_pzmq, const std::string &rawdata) + { _play_stop(); return LLM_NONE; } - std::string queue_play_stop(const std::string &rawdata) { + std::string queue_play_stop(pzmq *_pzmq, const std::string &rawdata) + { audio_clear_flage_ = true; return LLM_NONE; } - std::string cap(const std::string &rawdata) { + std::string cap(pzmq *_pzmq, const std::string &rawdata) + { if (cap_status_ == 0) { _cap(); } @@ -421,7 +456,8 @@ class llm_audio : public StackFlow { return sys_pcm_cap_channel; } - std::string cap_stop(const std::string &rawdata) { + std::string cap_stop(pzmq *_pzmq, const std::string &rawdata) + { if (cap_status_ > 0) { cap_status_--; if (cap_status_ == 0) { @@ -431,20 +467,23 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string cap_stop_all(const std::string &rawdata) { + std::string cap_stop_all(pzmq *_pzmq, const std::string &rawdata) + { cap_status_ = 0; _cap_stop(); return LLM_NONE; } - ~llm_audio() { + ~llm_audio() + { _play_stop(); _cap_stop(); } }; llm_audio *llm_audio::self; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_audio/src/sample_audio.c b/projects/llm_framework/main_audio/src/sample_audio.c index 1a7902ae..400d46be 100644 --- a/projects/llm_framework/main_audio/src/sample_audio.c +++ b/projects/llm_framework/main_audio/src/sample_audio.c @@ -41,7 +41,8 @@ static AX_S32 gplayLoopExit = 0; static AX_S32 gcapLoopExit = 0; -int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) { +int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) +{ switch (bits) { case 32: *format = AX_AUDIO_BIT_WIDTH_32; @@ -60,20 +61,23 @@ int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) { return 0; } -int IsUpTalkVqeEnabled(const AX_AP_UPTALKVQE_ATTR_T *pstVqeAttr) { +int IsUpTalkVqeEnabled(const AX_AP_UPTALKVQE_ATTR_T *pstVqeAttr) +{ int ret = ((pstVqeAttr->stAecCfg.enAecMode != AX_AEC_MODE_DISABLE) || (pstVqeAttr->stNsCfg.bNsEnable != AX_FALSE) || (pstVqeAttr->stAgcCfg.bAgcEnable != AX_FALSE)); return ret; } -int IsDnVqeEnabled(const AX_AP_DNVQE_ATTR_T *pstVqeAttr) { +int IsDnVqeEnabled(const AX_AP_DNVQE_ATTR_T *pstVqeAttr) +{ int ret = ((pstVqeAttr->stNsCfg.bNsEnable != AX_FALSE) || (pstVqeAttr->stAgcCfg.bAgcEnable != AX_FALSE)); return ret; } AX_AUDIO_SAMPLE_CONFIG_t play_config; void ax_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data, - int size) { + int size) +{ gplayLoopExit = 0; int ret = 0; AX_AUDIO_BIT_WIDTH_E format = AX_AUDIO_BIT_WIDTH_16; @@ -264,17 +268,20 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, return; } -void ax_close_play() { +void ax_close_play() +{ gplayLoopExit = 1; } -int ax_play_status() { +int ax_play_status() +{ return gplayLoopExit; } AX_AUDIO_SAMPLE_CONFIG_t cap_config; void ax_cap_start(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, - AUDIOCallback callback) { + AUDIOCallback callback) +{ int ret; AX_AUDIO_BIT_WIDTH_E format; unsigned int totalFrames = 0; @@ -458,10 +465,12 @@ void ax_cap_start(unsigned int card, unsigned int device, float Volume, int chan gcapLoopExit = 2; } -void ax_close_cap() { +void ax_close_cap() +{ gcapLoopExit = 1; } -int ax_cap_status() { +int ax_cap_status() +{ return gcapLoopExit; } \ No newline at end of file diff --git a/projects/llm_framework/main_camera/Kconfig b/projects/llm_framework/main_camera/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_camera/SConstruct b/projects/llm_framework/main_camera/SConstruct new file mode 100644 index 00000000..8de268d2 --- /dev/null +++ b/projects/llm_framework/main_camera/SConstruct @@ -0,0 +1,61 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +# SRCS = append_srcs_dir(ADir('src')) +SRCS = Glob('src/*.c*') +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +# INCLUDE += [ADir('../include'), ADir('../include/cppzmq'), ADir('../include/libzmq')] +LINK_SEARCH_PATH += [ADir('../static_lib')] + +REQUIREMENTS += ['ax_sys', 'ax_interpreter'] + +INCLUDE += [ADir('../include/opencv4')] +static_file = [] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] +STATIC_LIB += static_file * 4 + +STATIC_FILES += [AFile('camera.json')] +STATIC_FILES += Glob('mode_*.json') + +env['COMPONENTS'].append({'target':'llm_camera', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_camera/camera.json b/projects/llm_framework/main_camera/camera.json new file mode 100644 index 00000000..608b163c --- /dev/null +++ b/projects/llm_framework/main_camera/camera.json @@ -0,0 +1,18 @@ +{ + "mode": "None", + "type": "camera", + "capabilities": [ + "play", + "cap" + ], + "input_type": [ + "rpc.camera.wav.base64", + "rpc.camera.pcm.base64" + ], + "output_type": [ + "camera.pcm.stream" + ], + "cap_param": { + "None": "None" + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/camera.c b/projects/llm_framework/main_camera/src/camera.c new file mode 100644 index 00000000..c9efbf71 --- /dev/null +++ b/projects/llm_framework/main_camera/src/camera.c @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "camera.h" + +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" +#include +#include +#include +#include +#include +#include + +static int camera_capture_callback_set(struct camera_t* camera, vcamera_frame_get pcallback) +{ + if (camera->state_ == CAMERA_SATTE_CAP) { + SLOGW("Set capture callback failed"); + return -1; + } + camera->pcallback_ = pcallback; + return 0; +} + +static void* camera_capture_thread(void* param) +{ + int Ret = -1; + camera_t* camera = (camera_t*)param; + struct v4l2_buffer EnQueueBuf; + struct v4l2_buffer DeQueueBuf; + + SLOGI("Start capture"); + + while (camera->state_ == CAMERA_SATTE_CAP) { + memset(&DeQueueBuf, 0, sizeof(struct v4l2_buffer)); + DeQueueBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + DeQueueBuf.memory = V4L2_MEMORY_MMAP; + + memset(&EnQueueBuf, 0, sizeof(struct v4l2_buffer)); + EnQueueBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + EnQueueBuf.memory = V4L2_MEMORY_MMAP; + + // Retrieve data from the queue + Ret = ioctl(camera->dev_fd_, VIDIOC_DQBUF, &DeQueueBuf); + if (Ret) { + perror("Dequeue fail"); + break; + } + camera->pcallback_(camera->pcamera_buffer_[DeQueueBuf.index].pstart, camera->width_, camera->height_, + camera->pcamera_buffer_[DeQueueBuf.index].length, camera->ctx_); + + // Re-queue the data + EnQueueBuf.index = DeQueueBuf.index; + + if (ioctl(camera->dev_fd_, VIDIOC_QBUF, &EnQueueBuf)) { + perror("Enqueue fail"); + break; + } + } + + SLOGI("Stop capture"); + + return NULL; +} + +static int camera_capture_start(struct camera_t* camera) +{ + SLOGI("Start capture thread"); + if (!camera->pcallback_) { + SLOGW("Capture callback not set, start faild"); + return -1; + } + if (camera->state_ != CAMERA_SATTE_OPEN) { + SLOGW("Camera not open, start faild"); + return -1; + } + camera->state_ = CAMERA_SATTE_CAP; + + pthread_create(&camera->capture_thread_id_, NULL, camera_capture_thread, camera); + + return 0; +} + +static int camera_capture_stop(struct camera_t* camera) +{ + SLOGI("Stop capture thread"); + camera->state_ = CAMERA_SATTE_OPEN; + pthread_join(camera->capture_thread_id_, NULL); + SLOGI("Capture thread stop"); + return 0; +} + +static void camera_set_ctx(struct camera_t* camera, void* ctx) +{ + camera->ctx_ = ctx; +} + +int camera_open_from(camera_t* camera) +{ + int Ret = -1; + if (camera == NULL) return -1; + /* Check whether the camera is already open or in an error state */ + SLOGI("Open camera %s...", camera->dev_name_); + if (camera->state_ != CAMERA_STATE_CLOSE) { + SLOGE("Error: camera was open or meet error, now state is: %d", camera->state_); + goto ErrorHandle; + } + + /* Open the camera device */ + if ((camera->dev_fd_ = open(camera->dev_name_, O_RDWR, 0)) == -1) { + SLOGE("Open camera %s faild.", camera->dev_name_); + goto ErrorHandle; + } + + SLOGI("Open device successful."); + + /* Set the capture source */ + struct v4l2_input Input; + Input.index = 0; + + if (ioctl(camera->dev_fd_, VIDIOC_S_INPUT, &Input)) { + perror("VIDIOC_S_INPUT"); + } + + /* Retrieve camera information */ + struct v4l2_capability Cap; + + Ret = ioctl(camera->dev_fd_, VIDIOC_QUERYCAP, &Cap); + + if (Ret) { + perror("Get device info error"); + goto ErrorHandle; + } + + SLOGI("Get device info successful."); + + SLOGI("Driver name: %s", Cap.driver); + SLOGI("Card name: %s", Cap.card); + SLOGI("Bus info: %s", Cap.bus_info); + + struct v4l2_format TvFmt; + struct v4l2_fmtdesc FmtDesc; + struct v4l2_streamparm StreamParam; + + memset(&FmtDesc, 0, sizeof(FmtDesc)); + FmtDesc.index = 0; + FmtDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + /* Enumerate supported frame formats */ + SLOGI("Support format:"); + while (ioctl(camera->dev_fd_, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { + FmtDesc.index++; + SLOGI("Pixel format = %c%c%c%c", (FmtDesc.pixelformat & 0xFF), ((FmtDesc.pixelformat >> 8) & 0xFF), + ((FmtDesc.pixelformat >> 16) & 0xFF), (FmtDesc.pixelformat >> 24)); + SLOGI("Description = %s ", FmtDesc.description); + } + + /* Determine if the device is a device that can capture video */ + if (!(Cap.capabilities & V4L2_BUF_TYPE_VIDEO_CAPTURE)) { + SLOGE("The Current device is not a video capture device"); + goto ErrorHandle; + } + + /* Determine if video streaming is supported */ + if (!(Cap.capabilities & V4L2_CAP_STREAMING)) { + SLOGE("The Current device does not support streaming i/o"); + goto ErrorHandle; + } + + /* Get video stream information */ + memset(&StreamParam, 0, sizeof(struct v4l2_streamparm)); + StreamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Ret = ioctl(camera->dev_fd_, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("Before setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set frame rate denominator and numerator */ + StreamParam.parm.capture.timeperframe.denominator = camera->capture_fps_; + StreamParam.parm.capture.timeperframe.numerator = 1; + + if (ioctl(camera->dev_fd_, VIDIOC_S_PARM, &StreamParam) == -1) { + perror("Unable to set fps"); + goto ErrorHandle; + } + + /* Get video stream information */ + Ret = ioctl(camera->dev_fd_, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("After setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set automatic exposure */ + struct v4l2_control Ctrl; + Ctrl.id = V4L2_CID_EXPOSURE_ABSOLUTE; + Ret = ioctl(camera->dev_fd_, VIDIOC_G_CTRL, &Ctrl); + + if (Ret) { + perror("Set exposure failed"); + // goto ErrorHandle; + } + + /* Set capture format */ + TvFmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + TvFmt.fmt.pix.width = camera->width_; + TvFmt.fmt.pix.height = camera->height_; + TvFmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + TvFmt.fmt.pix.field = V4L2_FIELD_INTERLACED; + + if (ioctl(camera->dev_fd_, VIDIOC_S_FMT, &TvFmt) < 0) { + perror("Unable set VIDIOC_S_FMT"); + goto ErrorHandle; + } + + if (ioctl(camera->dev_fd_, VIDIOC_G_FMT, &TvFmt) < 0) { + perror("Get VIDIOC_S_FMT error"); + goto ErrorHandle; + } + + SLOGI("After setting frame params:"); + SLOGI("Width: %d, height: %d", TvFmt.fmt.pix.width, TvFmt.fmt.pix.height); + SLOGI("Pixel format = %c%c%c%c", (TvFmt.fmt.pix.pixelformat & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 8) & 0xFF), + ((TvFmt.fmt.pix.pixelformat >> 16) & 0xFF), (TvFmt.fmt.pix.pixelformat >> 24)); + + camera->width_ = TvFmt.fmt.pix.width; + camera->height_ = TvFmt.fmt.pix.height; + + /* Request V4L2 driver to allocate memory */ + struct v4l2_requestbuffers ReqBuffer; + memset(&ReqBuffer, 0, sizeof(ReqBuffer)); + ReqBuffer.count = camera->buffer_cnt_; + ReqBuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + ReqBuffer.memory = V4L2_MEMORY_MMAP; + + if (-1 == ioctl(camera->dev_fd_, VIDIOC_REQBUFS, &ReqBuffer)) { + perror("Fail to ioctl 'VIDIOC_REQBUFS'"); + goto ErrorHandle; + } + + camera->buffer_cnt_ = ReqBuffer.count; + SLOGI("Buffer count: %d", camera->buffer_cnt_); + + camera->pcamera_buffer_ = (CAMERA_Buffer_t*)calloc(camera->buffer_cnt_, sizeof(CAMERA_Buffer_t)); + if (camera->pcamera_buffer_ == NULL) { + SLOGE("Malloc buffer failed"); + goto ErrorHandle; + } + + /* Map kernel buffer to user space */ + struct v4l2_buffer Buf; + for (int i = 0; i < camera->buffer_cnt_; i++) { + memset(&Buf, 0, sizeof(Buf)); + Buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Buf.memory = V4L2_MEMORY_MMAP; + Buf.index = i; + + if (-1 == ioctl(camera->dev_fd_, VIDIOC_QUERYBUF, &Buf)) { + perror("Fail to ioctl : VIDIOC_QUERYBUF"); + goto ErrorHandle; + } + + camera->pcamera_buffer_[i].length = Buf.length; + camera->pcamera_buffer_[i].pstart = + mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, camera->dev_fd_, Buf.m.offset); + + if (MAP_FAILED == camera->pcamera_buffer_[i].pstart) { + perror("Fail to mmap"); + goto ErrorHandle; + } + + if (ioctl(camera->dev_fd_, VIDIOC_QBUF, &Buf)) { + perror("Unable to queue buffer"); + goto ErrorHandle; + } + } + + /* Start capturing */ + enum v4l2_buf_type BufType; + BufType = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (ioctl(camera->dev_fd_, VIDIOC_STREAMON, &BufType) < 0) { + perror("VIDIOC_STREAMON error"); + goto ErrorHandle; + } + camera->state_ = CAMERA_SATTE_OPEN; + camera->camera_capture_callback_set = camera_capture_callback_set; + camera->camera_capture_start = camera_capture_start; + camera->camera_capture_stop = camera_capture_stop; + camera->camera_set_ctx = camera_set_ctx; + camera->is_alloc_ = 0; + return 0; + +ErrorHandle: + SLOGE("Camera open meet error, now handle it"); + close(camera->dev_fd_); + free(camera->pcamera_buffer_); + return -1; +} + +camera_t* camera_open(const char* pdev_name, int width, int height, int fps) +{ + int Ret = -1; + camera_t* camera = (camera_t*)malloc(sizeof(camera_t)); + if (camera == NULL) return NULL; + memset(camera, 0, sizeof(camera_t)); + camera->buffer_cnt_ = CONFIG_CAPTURE_BUF_CNT; + + int CopyLen = strlen(pdev_name); + if (CopyLen > CONFIG_DEVNAME_LEN - 1) { + SLOGE("Error: device name length over limit: %d", CopyLen); + goto ErrorHandle; + } + memset(camera->dev_name_, 0, CONFIG_DEVNAME_LEN); + memcpy(camera->dev_name_, pdev_name, CopyLen); + + camera->width_ = width; + camera->height_ = height; + camera->capture_fps_ = fps; + + Ret = camera_open_from(camera); + if (Ret) { + goto ErrorHandle; + } + camera->is_alloc_ = 1; + return camera; + +ErrorHandle: + SLOGE("Camera open meet error, now handle it"); + free(camera); + return NULL; +} + +int camera_close(camera_t* camera) +{ + if (camera == NULL) return -1; + if (camera->state_ == CAMERA_SATTE_OPEN || camera->state_ == CAMERA_SATTE_CAP) { + SLOGI("Close camera %s", camera->dev_name_); + } else { + SLOGW("Skip close camera progress"); + return 0; + } + + for (int i = 0; i < CONFIG_CAPTURE_BUF_CNT; i++) { + munmap(camera->pcamera_buffer_[i].pstart, camera->pcamera_buffer_[i].length); + camera->pcamera_buffer_[i].length = 0; + } + + free(camera->pcamera_buffer_); + camera->pcamera_buffer_ = NULL; + + int Off = 1; + + if (ioctl(camera->dev_fd_, VIDIOC_STREAMOFF, &Off)) { + perror("Stop camera fail"); + return -1; + } + + close(camera->dev_fd_); + if (camera->is_alloc_) free(camera); + + SLOGI("camera closed"); + + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/camera.h b/projects/llm_framework/main_camera/src/camera.h new file mode 100644 index 00000000..71df9803 --- /dev/null +++ b/projects/llm_framework/main_camera/src/camera.h @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#ifndef CAMERA_H +#define CAMERA_H + +#include +#include +#define CONFIG_CAPTURE_BUF_CNT 10 +#define CONFIG_DEVNAME_LEN 32 + +#if __cplusplus +extern "C" { +#endif +typedef void (*vcamera_frame_get)(uint8_t* pdata, uint32_t width, uint32_t height, uint32_t length, void* ctx); +typedef enum { CAMERA_STATE_CLOSE, CAMERA_SATTE_OPEN, CAMERA_SATTE_CAP, CAMERA_STATE_ERR } CAMERA_STATE_t; + +typedef struct { + void* pstart; + int length; +} CAMERA_Buffer_t; + +typedef struct camera_t { + char dev_name_[CONFIG_DEVNAME_LEN]; + int dev_fd_; + int buffer_cnt_; + int capture_fps_; + int is_alloc_; + uint32_t width_; + uint32_t height_; + CAMERA_Buffer_t* pcamera_buffer_; + vcamera_frame_get pcallback_; + CAMERA_STATE_t state_; /* 0 Not open, 1 Turn on the camera, 2 Start capturing, + 3 Error */ + pthread_t capture_thread_id_; + void* ctx_; + /** + * Set capture frame callback + * Return value: 0 for success, -1 for failure + */ + int (*camera_capture_callback_set)(struct camera_t*, vcamera_frame_get); + /** + * Start capturing + * Return value: 0 for success, -1 for failure + */ + int (*camera_capture_start)(struct camera_t*); + /** + * Stop capturing + * Return value: 0 for success, -1 for failure + */ + int (*camera_capture_stop)(struct camera_t*); + void (*camera_set_ctx)(struct camera_t*, void*); +} camera_t; + +/** + * Open the camera + * @pdev_name Device node + * Return value: NULL for failure + */ +camera_t* camera_open(const char* pdev_name, int width, int height, int fps); + +/** + * Open the camera from config + * @pDevName Device node + * Return value: 0 for success, -1 for failure + */ +int camera_open_from(camera_t* camera); + +/** + * Close the camera + * Return value: 0 for success, -1 for failure + */ +int camera_close(camera_t* camera); + +#if __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/main.cpp b/projects/llm_framework/main_camera/src/main.cpp new file mode 100644 index 00000000..2eb7082d --- /dev/null +++ b/projects/llm_framework/main_camera/src/main.cpp @@ -0,0 +1,305 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" +#include "camera.h" +#include +#include + +using namespace StackFlows; +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + main_exit_flage = 1; +} + +typedef std::function task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: + camera_t *cam; + +public: + std::string response_format_; + task_callback_t out_callback_; + bool enoutput_; + bool enstream_; + std::atomic_int cap_status_; + std::unique_ptr camera_cap_thread_; + std::atomic_bool camera_clear_flage_; + + std::string devname_; + int frame_width_; + int frame_height_; + cv::Mat yuv_dist_; + + static void on_cap_fream(uint8_t *pData, uint32_t width, uint32_t height, uint32_t Length, void *ctx) + { + llm_task *self = static_cast(ctx); + if ((self->frame_height_ == height) && (self->frame_width_ == width)) { + if (self->out_callback_) self->out_callback_(pData, Length); + } else { + cv::Mat yuv_src(height, width, CV_8UC2, pData); + if ((self->frame_height_ >= height) && (self->frame_width_ >= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src.copyTo(self->yuv_dist_(cv::Rect(offsetX, offsetY, width, height))); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else if ((self->frame_height_ <= height) && (self->frame_width_ <= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (width - self->frame_width_) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (height - self->frame_height_) / 2; + yuv_src(cv::Rect(offsetX, offsetY, self->frame_width_, self->frame_height_)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else if ((self->frame_height_ >= height) && (self->frame_width_ <= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (width - self->frame_width_) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src(cv::Rect(offsetX, offsetY, self->frame_width_, height)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else { + int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src(cv::Rect(offsetX, offsetY, width, self->frame_height_)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } + } + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + bool parse_config(const nlohmann::json &config_body) + { + try { + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + devname_ = config_body.at("input"); + frame_width_ = config_body.at("frame_width"); + frame_height_ = config_body.at("frame_height"); + + } catch (...) { + return true; + } + enstream_ = (response_format_.find("stream") != std::string::npos); + yuv_dist_ = cv::Mat(frame_height_, frame_width_, CV_8UC2); + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + try { + cam = camera_open(devname_.c_str(), frame_width_, frame_height_, 30); + if (cam == NULL) { + printf("Camera open failed \n"); + return -1; + } + cam->ctx_ = static_cast(this); + cam->camera_capture_callback_set(cam, on_cap_fream); + cam->camera_capture_start(cam); + } catch (...) { + SLOGE("config file read false"); + return -3; + } + return 0; + } + + void inference(const std::string &msg) + { + // std::cout << msg << std::endl; + if (out_callback_) out_callback_("None", 4); + } + + llm_task(const std::string &workid) + { + cam = NULL; + } + + ~llm_task() + { + if (cam) { + cam->camera_capture_stop(cam); + camera_close(cam); + cam = NULL; + } + } +}; + +class llm_camera : public StackFlow { +private: + std::unordered_map> llm_task_; + +public: + llm_camera() : StackFlow("camera") + { + rpc_ctx_->register_rpc_action( + "list_camera", std::bind(&llm_camera::list_camera, this, std::placeholders::_1, std::placeholders::_2)); + } + + std::string list_camera(pzmq *_pzmq, const std::string &rawdata) + { + nlohmann::json req_body; + std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); + std::string param_json = RPC_PARSE_TO_SECOND(rawdata); + std::vector devices; + glob_t glob_result; + glob("/dev/video*", GLOB_TILDE, NULL, &glob_result); + for (size_t i = 0; i < glob_result.gl_pathc; ++i) { + devices.push_back(std::string(glob_result.gl_pathv[i])); + } + globfree(&glob_result); + send("camera.devices", devices, LLM_NO_ERROR, sample_json_str_get(param_json, "work_id"), zmq_url); + return LLM_NONE; + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const void *data, int size) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + std::string out_data((char *)data, size); + llm_channel->send_raw_to_pub(out_data); + if (llm_task_obj->enoutput_) { + std::string base64_data; + int ret = StackFlows::encode_base64(out_data, base64_data); + std::string out_json_str; + out_json_str += R"({"request_id":")"; + out_json_str += llm_channel->request_id_; + out_json_str += R"(","work_id":")"; + out_json_str += llm_channel->work_id_; + out_json_str += R"(","object":"image.yuvraw.base64","error":{"code":0, "message":""},"data":")"; + out_json_str += base64_data; + out_json_str += R"("}\n)"; + llm_channel->send_raw_to_usr(out_json_str); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + llm_task_obj->set_output(std::bind(&llm_camera::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + llm_task_[work_id_num] = llm_task_obj; + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("camera.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["input"] = llm_task_obj->devname_; + req_body["frame_width"] = llm_task_obj->frame_width_; + req_body["frame_height"] = llm_task_obj->frame_height_; + send("camera.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_camera() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main(int argc, char *argv[]) +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_camera llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 95d3dafa..62910304 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -31,8 +31,7 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] -STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), - AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_kws', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json b/projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json similarity index 100% rename from projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json rename to projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json diff --git a/projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json b/projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json similarity index 100% rename from projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json rename to projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 8e53258b..930a8bfc 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -25,7 +25,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_kws will be exit!"); main_exit_flage = 1; } @@ -40,12 +41,12 @@ static std::string base_model_config_path_; mode_config_.key = obj[#key]; class llm_task { - private: +private: sherpa_onnx::KeywordSpotterConfig mode_config_; std::unique_ptr spotter_; std::unique_ptr spotter_stream_; - public: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -61,7 +62,8 @@ class llm_task { std::function out_callback_; std::function play_awake_wav; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -89,15 +91,15 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -195,11 +197,13 @@ class llm_task { return 0; } - void set_output(std::function out_callback) { + void set_output(std::function out_callback) + { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) { + void sys_pcm_on_data(const std::string &raw) + { static int count = 0; if (count < delay_audio_frame_) { buffer_write_char(pcmdata, raw.c_str(), raw.length()); @@ -234,16 +238,19 @@ class llm_task { } } - bool delete_model() { + bool delete_model() + { spotter_.reset(); return true; } - llm_task(const std::string &workid) : audio_flage_(false) { + llm_task(const std::string &workid) : audio_flage_(false) + { pcmdata = buffer_create(); } - ~llm_task() { + ~llm_task() + { if (spotter_stream_) { spotter_stream_.reset(); } @@ -253,32 +260,19 @@ class llm_task { #undef CONFIG_AUTO_SET class llm_kws : public StackFlow { - private: +private: int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_kws::_load_config success"); - return 0; - } - } - public: - llm_kws() : StackFlow("kws") { +public: + llm_kws() : StackFlow("kws") + { task_count_ = 1; - repeat_event(1000, std::bind(&llm_kws::_load_config, this)); } - void play_awake_wav(const std::string &wav_file) { + void play_awake_wav(const std::string &wav_file) + { FILE *fp = fopen(wav_file.c_str(), "rb"); if (!fp) { printf("Open %s failed!\n", wav_file.c_str()); @@ -299,26 +293,43 @@ class llm_kws : public StackFlow { } } if (post != 0) { - unit_call("audio", "play", std::string((char *)(wav_data.data() + post), size - post)); + unit_call("audio", "play_raw", std::string((char *)(wav_data.data() + post), size - post)); } } - void task_pause(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_pause(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->audio_flage_) { if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_); llm_task_obj->audio_flage_ = false; } } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_work(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } } - void work(const std::string &work_id, const std::string &object, const std::string &data) override { + void work(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -333,7 +344,8 @@ class llm_kws : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void pause(const std::string &work_id, const std::string &object, const std::string &data) override { + void pause(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -348,21 +360,44 @@ class llm_kws : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } std::string tmp_msg1; const std::string *next_data = &data; int ret; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; @@ -370,7 +405,8 @@ class llm_kws : public StackFlow { llm_task_obj->sys_pcm_on_data((*next_data)); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -403,13 +439,16 @@ class llm_kws : public StackFlow { for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { - audio_url_ = unit_call("audio", "cap", "None"); - llm_channel->subscriber( - audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + audio_url_ = unit_call("audio", "cap", "None"); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_kws::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_kws::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -426,7 +465,8 @@ class llm_kws : public StackFlow { } } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_kws::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -447,12 +487,13 @@ class llm_kws : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("kws.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_kws::exit:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -462,20 +503,35 @@ class llm_kws : public StackFlow { send("None", "None", error_body, work_id); return -1; } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); if (llm_task_[work_id_num]->audio_flage_) { unit_call("audio", "cap_stop", "None"); } - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; } - ~llm_kws() { + ~llm_kws() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + if (iteam->second->audio_flage_) { + unit_call("audio", "cap_stop", "None"); + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index c95c1458..d20973e7 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -34,14 +34,11 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib/module-llm/libsentencepiece.a'), AFile('../static_lib/module-llm/libsentencepiece_train.a')] STATIC_LIB += static_file * 4 -STATIC_FILES += [AFile('qwen2.5-0.5B-prefill-20e.json'), - AFile('llama3.2-1B-prefill-ax630c.json'), - AFile('openbuddy-llama3.2-1b-ax630c.json'), - AFile('qwen2.5-coder-0.5B-ax630c.json'), - AFile('llama3.2-1B-prefill-ax630c_tokenizer.py'), - AFile('openbuddy-llama3.2-1b-ax630c_tokenizer.py'), +STATIC_FILES += [AFile('llama3.2-1B-prefill-ax630c_tokenizer.py'), + AFile('openbuddy-llama3.2-1B-ax630c_tokenizer.py'), AFile('qwen2.5-coder-0.5B-ax630c_tokenizer.py') ] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py b/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py index 0bf9ec51..712212fc 100644 --- a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py @@ -122,8 +122,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c.json b/projects/llm_framework/main_llm/mode_llama3.2-1B-prefill-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c.json rename to projects/llm_framework/main_llm/mode_llama3.2-1B-prefill-ax630c.json diff --git a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c.json b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json similarity index 95% rename from projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c.json rename to projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json index c435231b..a3229399 100644 --- a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c.json +++ b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json @@ -1,5 +1,5 @@ { - "mode":"openbuddy-llama3.2-1b-ax630c", + "mode":"openbuddy-llama3.2-1B-ax630c", "type":"llm", "capabilities":[ "text_generation", diff --git a/projects/llm_framework/main_llm/qwen2.5-0.5B-prefill-20e.json b/projects/llm_framework/main_llm/mode_qwen2.5-0.5B-prefill-20e.json similarity index 100% rename from projects/llm_framework/main_llm/qwen2.5-0.5B-prefill-20e.json rename to projects/llm_framework/main_llm/mode_qwen2.5-0.5B-prefill-20e.json diff --git a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c.json b/projects/llm_framework/main_llm/mode_qwen2.5-coder-0.5B-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c.json rename to projects/llm_framework/main_llm/mode_qwen2.5-coder-0.5B-ax630c.json diff --git a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c_tokenizer.py b/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py similarity index 96% rename from projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c_tokenizer.py rename to projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py index aae47921..09d7b9b3 100644 --- a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py @@ -123,8 +123,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py b/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py index 11f98b40..4fded69c 100644 --- a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py @@ -122,8 +122,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 568611a6..b89e94ba 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -17,7 +17,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -34,8 +35,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: LLMAttrType mode_config_; std::unique_ptr lLaMa_; std::string model_; @@ -48,11 +49,13 @@ class llm_task { std::atomic_bool tokenizer_server_flage_; unsigned int port_ = 8080; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -76,14 +79,14 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -122,21 +125,19 @@ class llm_task { pid_t pid = fork(); if (pid == 0) { execl("/usr/bin/python3", "python3", - ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), - "--host", "localhost", - "--port", std::to_string(port_).c_str(), - "--model_id", (base_model + "tokenizer").c_str(), - "--content", ("'" + prompt_ + "'").c_str(), - nullptr); + ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), "--host", "localhost", + "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), + "--content", ("'" + prompt_ + "'").c_str(), nullptr); perror("execl failed"); exit(1); } tokenizer_server_flage_ = true; - SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); + SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), + (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); std::this_thread::sleep_for(std::chrono::seconds(10)); } } else { - mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; + mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; } SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str()); mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; @@ -150,7 +151,11 @@ class llm_task { } }; lLaMa_ = std::make_unique(); - if (!lLaMa_->Init(mode_config_)) return -2; + if (!lLaMa_->Init(mode_config_)) { + lLaMa_->Deinit(); + lLaMa_.reset(); + return -2; + } } catch (...) { SLOGE("config false"); @@ -159,7 +164,8 @@ class llm_task { return 0; } - std::string prompt_complete(const std::string &input) { + std::string prompt_complete(const std::string &input) + { std::ostringstream oss_prompt; switch (mode_config_.tokenizer_type) { case TKT_LLaMa: @@ -180,11 +186,12 @@ class llm_task { oss_prompt << input; break; } - SLOGI("prompt_complete:%s",oss_prompt.str().c_str()); + SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); return oss_prompt.str(); } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { try { std::string out = lLaMa_->Run(prompt_complete(msg)); if (out_callback_) out_callback_(out, true); @@ -193,53 +200,52 @@ class llm_task { } } - bool pause() { + bool pause() + { lLaMa_->Stop(); return true; } - bool delete_model() { + bool delete_model() + { lLaMa_->Deinit(); lLaMa_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { + if (lLaMa_) { + lLaMa_->Deinit(); + } } }; #undef CONFIG_AUTO_SET class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_llm::_load_config success"); - return 0; - } - } - public: - llm_llm() : StackFlow("llm") { +public: + llm_llm() : StackFlow("llm") + { task_count_ = 2; - repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -260,21 +266,44 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; @@ -282,8 +311,15 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void task_asr_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -293,12 +329,20 @@ class llm_llm : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->lLaMa_->Stop(); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -326,21 +370,25 @@ class llm_llm : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("llm") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("asr") != std::string::npos) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -357,7 +405,8 @@ class llm_llm : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -372,12 +421,15 @@ class llm_llm : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("asr") != std::string::npos) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { @@ -390,7 +442,8 @@ class llm_llm : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -414,7 +467,8 @@ class llm_llm : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::taskinfo:%s", data.c_str()); // int ret = 0; nlohmann::json req_body; @@ -436,12 +490,13 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::exit:%s", data.c_str()); nlohmann::json error_body; @@ -459,19 +514,22 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index 21838316..e8c09a6b 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -23,10 +23,10 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] REQUIREMENTS += ['onnxruntime', 'samplerate'] - +INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] -STATIC_FILES += [AFile('melotts_zh-cn.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_melotts', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_melotts/melotts_zh-cn.json b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json similarity index 100% rename from projects/llm_framework/main_melotts/melotts_zh-cn.json rename to projects/llm_framework/main_melotts/mode_melotts-zh-cn.json diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index c346e7ad..c5573632 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -27,7 +27,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -51,7 +52,8 @@ typedef struct { float noise_scale_w = 0.6f; float sdp_ratio = 0.2f; - float get_length_scale() { + float get_length_scale() + { return (float)(length_scale / spacker_speed); } } melotts_config; @@ -65,8 +67,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: melotts_config mode_config_; std::unique_ptr encoder_; std::unique_ptr decoder_; @@ -83,8 +85,10 @@ class llm_task { task_callback_t out_callback_; bool enaudio_; int awake_delay_ = 1000; + std::string tts_string_stream_buff; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -108,7 +112,8 @@ class llm_task { return false; } - std::vector intersperse(const std::vector &lst, int item) { + std::vector intersperse(const std::vector &lst, int item) + { std::vector result(lst.size() * 2 + 1, item); for (size_t i = 1; i < result.size(); i += 2) { result[i] = lst[i / 2]; @@ -116,14 +121,18 @@ class llm_task { return result; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + // Compatible operation + if (model_ == "melotts_zh-cn") + config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, "melotts-zh-cn"); + try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -192,12 +201,14 @@ class llm_task { return 0; } - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } void resample_audio(float *input_buffer, int input_length, float *output_buffer, int *output_length, - double src_ratio) { + double src_ratio) + { SRC_STATE *src_state; int error; src_state = src_new(SRC_SINC_FASTEST, 1, &error); @@ -222,7 +233,8 @@ class llm_task { src_delete(src_state); } - bool TTS(const std::string &msg_str) { + bool TTS(const std::string &msg_str) + { try { std::vector phones_bef, tones_bef; lexicon_->convert(msg_str, phones_bef, tones_bef); @@ -283,7 +295,8 @@ class llm_task { return false; } - std::vector split(const std::string &s, char delim) { + std::vector split(const std::string &s, char delim) + { std::vector result; std::stringstream ss(s); std::string item; @@ -293,7 +306,8 @@ class llm_task { return result; } - void _ax_init() { + void _ax_init() + { if (!ax_init_flage_) { int ret = AX_SYS_Init(); if (0 != ret) { @@ -309,7 +323,8 @@ class llm_task { ax_init_flage_++; } - void _ax_deinit() { + void _ax_deinit() + { if (ax_init_flage_ > 0) { --ax_init_flage_; if (!ax_init_flage_) { @@ -319,12 +334,14 @@ class llm_task { } } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { enaudio_ = true; _ax_init(); } - ~llm_task() { + ~llm_task() + { _ax_deinit(); } }; @@ -332,32 +349,24 @@ int llm_task::ax_init_flage_ = 0; #undef CONFIG_AUTO_SET class llm_tts : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_melotts::_load_config success"); - return 0; - } - } - public: - llm_tts() : StackFlow("melotts") { +public: + llm_tts() : StackFlow("melotts") + { task_count_ = 1; - repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -379,11 +388,25 @@ class llm_tts : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + bool is_breakpoint(const std::string &cutf8) + { + if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || + cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") + return true; + else + return false; + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (data.empty() || (data == "None")) return; - static std::string faster_stream_buff; nlohmann::json error_body; const std::string *next_data = &data; bool enbase64 = (object.find("base64") == std::string::npos) ? false : true; @@ -400,32 +423,32 @@ class llm_tts : public StackFlow { std::string tmp_msg2; if (enbase64) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { return; } next_data = &tmp_msg2; } - std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar((*next_data)); + std::string user_msg = sample_unescapeString(*next_data); + std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar(user_msg); for (auto cutf8 : tmp_data) { - if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || - cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") { - faster_stream_buff += cutf8; - ret = llm_task_obj->TTS(faster_stream_buff); - faster_stream_buff.clear(); + if (is_breakpoint(cutf8)) { + llm_task_obj->tts_string_stream_buff += cutf8; + ret = llm_task_obj->TTS(llm_task_obj->tts_string_stream_buff); + llm_task_obj->tts_string_stream_buff.clear(); if (ret) { error_body["code"] = -11; error_body["message"] = "Model run failed."; llm_channel->send("None", "None", error_body, llm_channel->work_id_); } } else { - faster_stream_buff += cutf8; + llm_task_obj->tts_string_stream_buff += cutf8; } } if (finish_flage) { - if (!faster_stream_buff.empty()) { - faster_stream_buff.push_back('.'); - ret = llm_task_obj->TTS(faster_stream_buff); - faster_stream_buff.clear(); + if (!llm_task_obj->tts_string_stream_buff.empty()) { + llm_task_obj->tts_string_stream_buff.push_back('.'); + ret = llm_task_obj->TTS(llm_task_obj->tts_string_stream_buff); + llm_task_obj->tts_string_stream_buff.clear(); if (ret) { error_body["code"] = -11; error_body["message"] = "Model run failed."; @@ -435,10 +458,18 @@ class llm_tts : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); + llm_task_obj->tts_string_stream_buff.clear(); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "queue_play_stop", data); } @@ -446,13 +477,15 @@ class llm_tts : public StackFlow { if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "play_stop", data); } - llm_channel->subscriber_work_id(llm_task_obj->superior_id_, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + llm_task_obj->superior_id_, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -479,22 +512,26 @@ class llm_tts : public StackFlow { llm_channel->set_stream(llm_task_obj->enstream_); SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("tts") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if ((input.find("llm") != std::string::npos) || (input.find("vlm") != std::string::npos)) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); llm_task_obj->superior_id_ = input; llm_task_obj->superior_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -511,7 +548,8 @@ class llm_tts : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -526,14 +564,17 @@ class llm_tts : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if ((data.find("llm") != std::string::npos) || (data.find("vlm") != std::string::npos)) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->superior_id_ = data; llm_task_obj->superior_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { @@ -546,7 +587,8 @@ class llm_tts : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -573,7 +615,8 @@ class llm_tts : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -594,12 +637,13 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("melotts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::exit:%s", data.c_str()); nlohmann::json error_body; @@ -617,21 +661,22 @@ class llm_tts : public StackFlow { return 0; } - ~llm_tts() { + ~llm_tts() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp index 67d8c1ab..8b6255d2 100644 --- a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp +++ b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp @@ -92,9 +92,45 @@ class Lexicon { return words; } + bool is_english(std::string s) { + if (s.size() == 1) + return (s[0] >= 'A' && s[0] <= 'Z') || (s[0] >= 'a' && s[0] <= 'z'); + else + return false; + } + + std::vector merge_english(const std::vector& splitted_text) { + std::vector words; + int i = 0; + while (i < splitted_text.size()) { + std::string s; + if (is_english(splitted_text[i])) { + while (i < splitted_text.size()) { + if (!is_english(splitted_text[i])) { + break; + } + s += splitted_text[i]; + i++; + } + // to lowercase + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c){ return std::tolower(c); }); + words.push_back(s); + if (i >= splitted_text.size()) + break; + } + else { + words.push_back(splitted_text[i]); + i++; + } + } + return words; + } + void convert(const std::string& text, std::vector& phones, std::vector& tones) { auto splitted_text = splitEachChar(text); - for (auto c : splitted_text) { + auto zh_mix_en = merge_english(splitted_text); + for (auto c : zh_mix_en) { std::string s{c}; if (s == "，") s = ","; diff --git a/projects/llm_framework/main_skel/Kconfig b/projects/llm_framework/main_skel/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_skel/SConstruct b/projects/llm_framework/main_skel/SConstruct new file mode 100644 index 00000000..f40e42d3 --- /dev/null +++ b/projects/llm_framework/main_skel/SConstruct @@ -0,0 +1,47 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = append_srcs_dir(ADir('src')) +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'ax-samples'] + +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] +REQUIREMENTS += ['ax_skel', 'ax_engine', 'ax_venc', 'ax_vdec', 'ax_ivps', 'ax_proton'] + +INCLUDE += [ADir('../include'), ADir('src/runner'), ADir('../include/opencv4')] + +static_file = Glob('../static_lib/module-llm/libabsl_*') +static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') +STATIC_LIB += static_file * 2 + +STATIC_FILES += Glob('mode_*.json') + +env['COMPONENTS'].append({'target':'llm_skel', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_skel/src/main.c b/projects/llm_framework/main_skel/src/main.c new file mode 100644 index 00000000..144d3a46 --- /dev/null +++ b/projects/llm_framework/main_skel/src/main.c @@ -0,0 +1,2253 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "YuvHandler.h" +#include "ax_skel_api.h" +#include "ax_sys_api.h" +#include "ax_ivps_api.h" +#include "ax_venc_api.h" +#include "ax_buffer_tool.h" +#include "skel_log.h" +#include "ax_engine_api.h" +#include "frameMgr.h" +#include "attrParser.h" +#include "statMgr.h" +#include "common_vdec_api.h" + +#ifndef ALIGN_UP +#define ALIGN_UP(x, align) ((((x) + ((align) - 1)) / (align)) * (align)) +#endif + +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, align) (((x) / (align)) * (align)) +#endif + +#ifndef AX_MAX +#define AX_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef AX_MIN +#define AX_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" +#define SKEL_SAMPLE_OUTPUT_VEHICLE_PATH "vehicle" +#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" +#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" +#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" +#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" + +#define SAMPLE_SKEL_SHIFT_LEFT_ALIGN(a) (1 << (a)) +#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) +#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) + +#define RUN_COMMAND(format, ...) \ + do { \ + char cmd[512]; \ + snprintf(cmd, 512, format, ##__VA_ARGS__); \ + system(cmd); \ + } while (0); + +AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) +{ + if (file) { + va_list args; + char szLog[1024] = {0}; + + va_start(args, fmt); + + AX_U32 len = vsnprintf(szLog, sizeof(szLog), (char *)fmt, args); + if (len < (AX_U32)sizeof(szLog)) { + szLog[len] = '\0'; + } else { + szLog[sizeof(szLog) - 1] = '\0'; + } + + fwrite((AX_U8 *)szLog, 1, strlen(szLog), file); + + va_end(args); + } +} +#define OUTPUT_LOG_SAVE(fmt, ...) LogSaveToFile(fpResultFile, fmt "\n", ##__VA_ARGS__) +#define BLACK_VIDEO_FRAME_COUNT 60 +#define SKEL_SAMPLE_OBJECT_SIZE 512 + +struct skeleton { + int connection[2]; + int left_right_neutral; +}; + +struct skeleton pairs[] = {{{15, 13}, 0}, {{13, 11}, 0}, {{16, 14}, 0}, {{14, 12}, 0}, {{11, 12}, 0}, + {{5, 11}, 0}, {{6, 12}, 0}, {{5, 6}, 0}, {{5, 7}, 0}, {{6, 8}, 0}, + {{7, 9}, 0}, {{8, 10}, 0}, {{1, 2}, 0}, {{0, 1}, 0}, {{0, 2}, 0}, + {{1, 3}, 0}, {{2, 4}, 0}, {{0, 5}, 0}, {{0, 6}, 0}}; + +#define DETECT_SKEL_POINT_COUNT 256 + +typedef struct _AI_Detection_Box_t { + AX_F32 fX, fY, fW, fH; +} AI_Detection_Box_t; + +typedef struct _AI_Detection_Point_t { + AX_F32 fX, fY; +} AI_Detection_Point_t; + +typedef struct _AI_Detection_SkelResult_t { + const AX_CHAR *pstrObjectCategory; + AX_U8 nPointNum; + AI_Detection_Point_t tPoint[DETECT_SKEL_POINT_COUNT]; + AI_Detection_Box_t tBox; +} AI_Detection_SkelResult_t; + +static AX_U64 get_tick_count(AX_VOID) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +// static AX_U64 get_tick_count_us(AX_VOID) { +// struct timespec ts; +// clock_gettime(CLOCK_MONOTONIC, &ts); +// return (ts.tv_sec * 1000000 + ts.tv_nsec / 1000); +// } + +static AX_VOID ShowUsage(AX_VOID) +{ + // printf("usage: ./%s ...\n", SAMPLE_SKEL_NAME); + printf("options:\n"); + printf("-i, \tInput File(yuv)\n"); + printf("-r, \tInput File Resolution(wxh)(yuv: should be input)\n"); + printf("-w, \tWrite result image to new jpg file((unsigned int), default=0)\n"); + printf("-o, \tSave result to file(file name)\n"); + printf("-m, \tModels deployment path(path name)\n"); + printf("-t, \tRepeat times((unsigned int), default=1)\n"); + printf("-I, \tInterval repeat time((unsigned int)ms, default=0)\n"); + printf("-c, \tConfidence((float: 0-1), default=0)\n"); + printf("-H, \tHuman track size limit((unsigned int), default=0)\n"); + printf("-V, \tVehicle track size limit((unsigned int), default=0)\n"); + printf("-C, \tCylcle track size limit((unsigned int), default=0)\n"); + printf( + "-d, \tSkel detect type((unsigned int), default=2)\n" + "\t\t0: detect only\n" + "\t\t1: detect + track\n" + "\t\t2: detect + track + push\n"); + printf( + "-u, \tSkel push strategy((unsigned int), default=3)\n" + "\t\t1: fast push strategy\n" + "\t\t2: push strategy\n" + "\t\t3: best push strategy\n"); + printf( + "-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" + "\t\t0: VNPU Disable\n" + "\t\t1: STD-VNPU Default\n" + "\t\t2: STD-VNPU1\n" + "\t\t3: STD-VNPU2\n"); + printf( + "-p, \tSkel PPL((unsigned int), default=1)\n" + "\t\t1: AX_SKEL_PPL_HVCP\n" + "\t\t2: AX_SKEL_PPL_FACE\n"); + printf( + "-v, \tLog level((unsigned int), default=5)\n" + "\t\t0: LOG_EMERGENCY_LEVEL\n" + "\t\t1: LOG_ALERT_LEVEL\n" + "\t\t2: LOG_CRITICAL_LEVEL\n" + "\t\t3: LOG_ERROR_LEVEL\n" + "\t\t4: LOG_WARN_LEVEL\n" + "\t\t5: LOG_NOTICE_LEVEL\n" + "\t\t6: LOG_INFO_LEVEL\n" + "\t\t7: LOG_DEBUG_LEVEL\n"); + printf("-h, \tprint this message\n"); +} + +AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) +{ + if (pstConfig->nSize > 0 && pstConfig->pstItems) { + for (size_t i = 0; i < pstConfig->nSize; i++) { + if (pstConfig->pstItems[i].pstrType && pstConfig->pstItems[i].pstrValue) { + // cmd: "body_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + if (strcmp(pstConfig->pstItems[i].pstrType, "body_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "crop_encoder_qpLevel", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "crop_encoder_qpLevel") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "detect_roi_polygon") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T)) { + AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = + (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, + pstConf->nPointNum); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_strategy", value_type: AX_SKEL_PUSH_STRATEGY_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_strategy") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_STRATEGY_T)) { + AX_SKEL_PUSH_STRATEGY_T *pstConf = (AX_SKEL_PUSH_STRATEGY_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, + pstConf->ePushMode, pstConf->nIntervalTimes, pstConf->nPushCounts, + pstConf->bPushSameFrame); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_panorama", value_type: AX_SKEL_PUSH_PANORAMA_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_panorama") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T)) { + AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = + (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Enable: %d, Quality: %d]", pstConfig->pstItems[i].pstrType, + pstConf->bEnable, pstConf->nQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_body", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_body") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_vehicle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_vehicle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_cycle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_cycle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_face", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_face") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [W: %d, H: %d, P: %f, Y: %f, R: %f, B: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, + pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, + pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, + pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, + pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_plate", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_plate") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_bind_enable", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_bind_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "track_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "target_config", value_type: AX_SKEL_TARGET_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, + pstConf->pstItems[j].pstrObjectCategory); + } + } + } else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_target_config", value_type: AX_SKEL_TARGET_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, + pstConf->pstItems[j].pstrObjectCategory); + } + } + } else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "analyzer_attr_config", value_type: AX_SKEL_ANALYZER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "analyzer_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ANALYZER_CONFIG_T)) { + AX_SKEL_ANALYZER_CONFIG_T *pstConf = + (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->peItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { + ALOGI("SKEL set %s: [%d]%d", pstConfig->pstItems[i].pstrType, j, pstConf->peItems[j]); + } + } else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "venc_attr_config", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "venc_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, (AX_U32)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_attr_always", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_attr_always") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); + } + } else { + ALOGE("SKEL cmd: %s not support", pstConfig->pstItems[i].pstrType); + } + } + } + } + + return 0; +} + +FILE *LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + + if (f_in) { + if (pLen) { + fseeko(f_in, 0L, SEEK_END); + AX_U64 nFileSize = ftello(f_in); + rewind(f_in); + + *pLen = (AX_U64)nFileSize; + } + } + + return f_in; +} + +int ReLoadFile(FILE *pFile) +{ + if (pFile) { + rewind(pFile); + return 0; + } + + return -1; +} + +int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, AX_VOID *pSrcVirAddr, AX_U32 nSrcSize, AX_U32 nSrcStride, AX_U32 nSrcHeight, + AX_U64 nDstPhyAddr, AX_VOID *pDstVirAddr, AX_U32 nDstSize, AX_U32 nDstStride, AX_U32 nDstHeight) +{ + AX_U64 nStartTime = get_tick_count(); + + // src check + if (!pSrcVirAddr) { + return -1; + } + + if (nSrcSize != nSrcStride * nSrcHeight * 3 / 2) { + return -1; + } + + // dst check + if (!pDstVirAddr) { + return -1; + } + + if (nDstSize != nDstStride * nDstHeight * 3 / 2) { + return -1; + } + + if (nDstSize < nSrcSize) { + return -1; + } + + if (nDstStride < nSrcStride) { + return -1; + } + + if (nDstHeight != nSrcHeight) { + return -1; + } + + AX_VOID *src = pSrcVirAddr; + AX_VOID *dst = pDstVirAddr; + for (AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { + memcpy(dst, src, nSrcStride); + src += nSrcStride; + dst += nDstStride; + } + + AX_U64 nProcessElasped = get_tick_count() - nStartTime; + + ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, + nProcessElasped); + + return 0; +} + +int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) +{ + AX_S32 nReadSize = 0; + + if (pFile) { + if (nPhyAddr != 0 && pVirAddr) { + memset((AX_U8 *)pVirAddr, 0x00, nSize); + nReadSize = fread((AX_U8 *)pVirAddr, 1, nSize, pFile); + } else { + fseeko(pFile, nSize, SEEK_CUR); + } + } + + return nReadSize; +} + +int DecodeJpeg(FILE *InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID *OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) +{ + SAMPLE_INPUT_FILE_INFO_T stStreamInfo; + SAMPLE_STREAM_BUF_T stStreamBuf; + size_t nReadLen = 0; + AX_U64 streamPhyAddr = 0; + AX_VOID *pStreamVirAddr = NULL; + AX_VDEC_DEC_ONE_FRM_T decOneFrmParam; + size_t nFileSize = 0; + AX_U32 heightAlign = 0; + AX_U32 frmStride = 0; + frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; + heightAlign = ALIGN_UP(nHeight, 16); + AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; + + memset(&stStreamInfo, 0, sizeof(SAMPLE_INPUT_FILE_INFO_T)); + memset(&stStreamBuf, 0, sizeof(SAMPLE_STREAM_BUF_T)); + memset(&decOneFrmParam, 0, sizeof(AX_VDEC_DEC_ONE_FRM_T)); + + rewind(InputFileHandle); + fseek(InputFileHandle, 0, SEEK_END); + nFileSize = (size_t)ftello(InputFileHandle); + + stStreamInfo.fInput = InputFileHandle; + stStreamInfo.sFileSize = nFileSize; + + stStreamBuf.uBufSize = ((AX_U32)nFileSize) > STREAM_BUFFER_MAX_SIZE ? STREAM_BUFFER_MAX_SIZE : nFileSize; + int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, stStreamBuf.uBufSize, 0x100, + (AX_S8 *)"vdec_input_stream"); + if (ret != AX_SUCCESS) { + ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", stStreamBuf.uBufSize, ret); + return -1; + } + + stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; + stStreamBuf.tBufAddr.u64PhyAddr = streamPhyAddr; + + ret = StreamParserReadFrameJpeg(&stStreamInfo, &stStreamBuf, &nReadLen); + if (0 != ret) { + ALOGE("StreamParserReadFrameJpeg failed! ret = 0x%x", ret); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return -1; + } + + if (!nReadLen) { + ALOGE("read jpeg frame FAILED!\n"); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + ret = AX_ERR_VDEC_STRM_ERROR; + return ret; + } + + decOneFrmParam.stStream.pu8Addr = (AX_U8 *)stStreamBuf.tBufAddr.pVirAddr; + decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; + decOneFrmParam.stStream.u32StreamPackLen = (AX_U32)nReadLen; + + decOneFrmParam.stFrame.u64VirAddr[0] = (AX_ULONG)OneYUVDataVir; + decOneFrmParam.stFrame.u64VirAddr[1] = (AX_ULONG)OneYUVDataVir + frmStride * heightAlign; + decOneFrmParam.stFrame.u64PhyAddr[0] = OneYUVDataPhy; + decOneFrmParam.stFrame.u64PhyAddr[1] = OneYUVDataPhy + frmStride * heightAlign; + decOneFrmParam.stFrame.u32FrameSize = nFrameSize; + + ret = AX_VDEC_JpegDecodeOneFrame(&decOneFrmParam); + if (ret != AX_SUCCESS) { + ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", ret, SampleVdecRetStr(ret)); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return ret; + } + + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + + return 0; +} + +int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr, AX_U32 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + if (f_in == NULL) { + ALOGE("Unable to open input file\n"); + return -1; + } + + /* file i/o pointer to full */ + fseek(f_in, 0L, SEEK_END); + AX_U32 nFileSize = ftell(f_in); + rewind(f_in); + + if (pPhyAddr && ppVirAddr) { + AX_U64 nPhyAddr = 0; + AX_VOID *pVirAddr = NULL; + + // pVirAddr = (AX_VOID *)malloc(nFileSize); + AX_S32 nRet = AX_SYS_MemAlloc(&nPhyAddr, (AX_VOID **)&pVirAddr, nFileSize, 128, (AX_S8 *)"SKEL_TEST"); + + if (nRet != 0) { + fclose(f_in); + ALOGE("AX_SYS_MemAlloc failed"); + return -1; + } + + fread((AX_U8 *)pVirAddr, nFileSize, 1, f_in); + + *pPhyAddr = nPhyAddr; + *ppVirAddr = pVirAddr; + } + + if (pLen) { + *pLen = nFileSize; + } + + fclose(f_in); + + return 0; +} + +int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, + AX_VOID *pVirAddr, AX_U32 nLen) +{ + AX_S32 s32Ret = AX_SUCCESS; + + AX_U32 frameSize = nLen; + AX_JPEG_ENCODE_ONCE_PARAMS_T stJpegEncodeOnceParam; + memset(&stJpegEncodeOnceParam, 0, sizeof(stJpegEncodeOnceParam)); + + AX_U64 outPhyAddr = 0; + AX_VOID *outVirAddr = NULL; + + s32Ret = AX_SYS_MemAlloc(&outPhyAddr, &outVirAddr, frameSize, 128, (AX_S8 *)"SKEL-VENC"); + if (s32Ret) { + ALOGE("alloc mem err, size(%d).", frameSize); + return -1; + } + + // output + stJpegEncodeOnceParam.u32OutBufSize = frameSize; + stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; + stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; + + // input + stJpegEncodeOnceParam.stJpegParam.u32Qfactor = 90; + stJpegEncodeOnceParam.u32Width = nWidth; + stJpegEncodeOnceParam.u32Height = nHeight; + stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stJpegEncodeOnceParam.u32PicStride[0] = nStride; + stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; + stJpegEncodeOnceParam.u32PicStride[2] = 0; + + stJpegEncodeOnceParam.u64PhyAddr[0] = nPhyAddr; + stJpegEncodeOnceParam.u64PhyAddr[1] = + stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64PhyAddr[2] = 0; + + stJpegEncodeOnceParam.u64VirAddr[0] = (AX_ULONG)pVirAddr; + stJpegEncodeOnceParam.u64VirAddr[1] = + stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64VirAddr[2] = 0; + + s32Ret = AX_VENC_JpegEncodeOneFrame(&stJpegEncodeOnceParam); + if (AX_SUCCESS != s32Ret) { + ALOGE("AX_VENC_JpegEncodeOneFrame fail, ret=0x%x", s32Ret); + s32Ret = -1; + goto JENC_EXIT; + } + +JENC_EXIT: + if (0 == s32Ret && dstFile) { + FILE *fp_w = fopen(dstFile, "wb"); + + if (fp_w) { + ALOGN("\tWrite new JPG result image to file: %s", dstFile); + fwrite((AX_U8 *)stJpegEncodeOnceParam.pu8Addr, stJpegEncodeOnceParam.u32Len, 1, fp_w); + fclose(fp_w); + } + } + + if (outPhyAddr && (NULL != outVirAddr)) { + AX_SYS_MemFree(outPhyAddr, outVirAddr); + } + + return s32Ret; +} + +AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) +{ + if (nFrameSeqNum < 1) { + nFrameSeqNum = 1; + } + + if (nSrcFrameRate == nDstFrameRate) { + return AX_FALSE; + } + + if (nDstFrameRate > nSrcFrameRate) { + return AX_FALSE; + } + + if ((nFrameSeqNum * nDstFrameRate / (nSrcFrameRate)) > ((nFrameSeqNum - 1) * nDstFrameRate / (nSrcFrameRate))) { + return AX_FALSE; + } else { + return AX_TRUE; + } +} + +AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) +{ + while (argc == 1) { + sleep(10); + } + + AX_S32 nRet = 0; + AX_S32 c; + AX_S32 isExit = 0; + AX_BOOL bJpgFile = AX_FALSE; + FILE *InputFileHandle = NULL; + const AX_CHAR *InputFile = NULL; + const AX_CHAR *InputResolution = NULL; + AX_S32 InputFileFrameCnt = 1; + const AX_CHAR *ReWritePath = NULL; + const AX_CHAR *SaveResultPath = NULL; + const AX_CHAR *ModelsPath = NULL; + FILE *fpResultFile = NULL; + AX_S32 nRepeatTimes = 1; + AX_S32 nPPL = AX_SKEL_PPL_HVCP; + AX_S32 nDetectType = 2; + AX_S32 nPushStrategy = 3; + AX_S32 nNpuType = 0; + AX_S32 nInterval = 0; + AX_U32 nStride = 1920; + AX_U32 nWidth = 1920; + AX_U32 nHeight = 1080; + AX_U32 nJencBufSize = 0; + AX_U32 nFrameDepth = 1; + AX_U32 nOneSize = 0; + AX_U64 OneYUVDataPhy = 0; + AX_VOID *OneYUVDataVir = NULL; + AX_U32 nFrameSize = nStride * nHeight * 3 / 2; + AX_U32 nCacheListDepth = 1; + AX_S32 nSrcFrameRate = 25; + AX_S32 nDstFrameRate = 25; + AX_VOID *YUVDataVir = NULL; + AX_U64 YUVDataPhy = 0; + AX_VOID *YUVDataWrVir = NULL; + AX_U64 YUVDataWrPhy = 0; + AX_SKEL_HANDLE pHandle = NULL; + AX_SKEL_FRAME_T stFrame = {0}; + AX_SKEL_RESULT_T *pstResult = NULL; + AX_U64 nFileSize = 0; + AX_U64 nStartTime = 0; + AX_U64 nInitElasped = 0; + AX_U64 nCreateElasped = 0; + AX_U64 nProcessElasped = 0; + AX_U64 nResultElasped = 0; + AX_U64 nResultElaspedMin = (AX_U64)-1; + AX_U64 nResultElaspedMax = 0; + AX_U64 nResultElaspedTotal = 0; + AX_F32 fConfidence = 0.0; + // AX_U32 nHumantracksize = 0; + // AX_U32 nVehicletracksize = 0; + // AX_U32 nCycletracksize = 0; + AX_U64 nSkelFrameId = 1; + +#if defined(SAMPLE_SKEL_BUILD_VERSION) + printf("SKEL sample: %s build: %s %s\n", SAMPLE_SKEL_BUILD_VERSION, __DATE__, __TIME__); +#endif + + while ((c = getopt(argc, argv, "i:r:I:w:o:m:t:d:u:p:j:v:c:N:H:V:C:h::")) != -1) { + isExit = 0; + switch (c) { + case 'i': + InputFile = (const AX_CHAR *)optarg; + break; + case 'r': + InputResolution = (const AX_CHAR *)optarg; + break; + case 'I': + nInterval = atoi(optarg); + break; + case 'w': + ReWritePath = (const AX_CHAR *)optarg; + break; + case 'o': + SaveResultPath = (const AX_CHAR *)optarg; + break; + case 'm': + ModelsPath = (const AX_CHAR *)optarg; + break; + case 't': + nRepeatTimes = atoi(optarg); + break; + case 'd': + nDetectType = atoi(optarg); + break; + case 'u': + nPushStrategy = atoi(optarg); + break; + case 'N': + nNpuType = atoi(optarg); + break; + case 'p': + nPPL = atoi(optarg); + break; + case 'j': + nJencBufSize = atoi(optarg); + break; + case 'c': + fConfidence = atof(optarg); + break; + // case 'H': + // nHumantracksize = atoi(optarg); + // break; + // case 'V': + // nVehicletracksize = atoi(optarg); + // break; + // case 'C': + // nCycletracksize = atoi(optarg); + // break; + case 'v': + log_level = atoi(optarg); + break; + case 'h': + isExit = 1; + break; + case 'D': + nCacheListDepth = atoi(optarg); + break; + default: + isExit = 1; + break; + } + } + + if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) || + (log_level < 0 || log_level >= SKEL_LOG_MAX) || (fConfidence < 0 || fConfidence > 1) || + (nDetectType < 0 || nDetectType > 2) || (nPushStrategy < 1 || nPushStrategy > 3) || (nNpuType > 3)) { + ShowUsage(); + exit(0); + } + + if (InputFile) { + AX_U32 nInputFileLen = strlen(InputFile); + + if (nInputFileLen > 4 && (strcasecmp(&InputFile[nInputFileLen - 4], ".jpg") == 0)) { + bJpgFile = AX_TRUE; + } + } + + if (nRepeatTimes <= 0) { + nRepeatTimes = 1; + } + + if (nInterval < 0) { + nInterval = 0; + } + if (nCacheListDepth == 0) { + nCacheListDepth = 1; + } + nInterval = nInterval * 1000; + + if (access(InputFile, 0) != 0) { + ALOGE("%s not exist", InputFile); + exit(0); + } + + // clear output + if (SaveResultPath) { + RUN_COMMAND("rm -rf %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_BODY_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_VEHICLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_CYCLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_FACE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_PLATE_PATH); + } + + if (ReWritePath) { + RUN_COMMAND("rm -rf %s", ReWritePath); + RUN_COMMAND("mkdir -p %s", ReWritePath); + } + + if (InputResolution) { + AX_CHAR *temp_p = strstr(InputResolution, "x"); + + if (!temp_p || strlen(temp_p) <= 1) { + ShowUsage(); + exit(0); + } + + nWidth = atoi(InputResolution); + nStride = nWidth; + nHeight = atoi(temp_p + 1); + } + + nRet = AX_SYS_Init(); + if (0 != nRet) { + ALOGE("AX_SYS_Init() fail, ret = 0x%x", nRet); + exit(0); + } + + nRet = AX_IVPS_Init(); + + ALOGI("AX_IVPS_Init nRet = 0x%x", nRet); + + AX_VENC_MOD_ATTR_T stVencModAttr; + memset(&stVencModAttr, 0x00, sizeof(stVencModAttr)); + stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; + stVencModAttr.stModThdAttr.u32TotalThreadNum = 1; + stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; + nRet = AX_VENC_Init(&stVencModAttr); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VENC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VENC_Init nRet = 0x%x", nRet); + + nRet = AX_VDEC_Init(NULL); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VDEC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VDEC_Init nRet = 0x%x", nRet); + + // init engine + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); +#if defined(CHIP_AX650) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } else if (nNpuType >= 1 && nNpuType <= 4) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_STD; + } else if (nNpuType >= 5 && nNpuType <= 7) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_BIG_LITTLE; + } +#elif defined(CHIP_AX620E) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } else { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_ENABLE; + } +#else + // #error "NO CHIP SELECTED." +#endif + + nRet = AX_ENGINE_Init(&npu_attr); + if (0 != nRet) { + ALOGE("AX_NPU_SDK_EX_Init_with_attr() fail, ret = 0x%x", nRet); + goto EXIT0; + } + + nOneSize = nWidth * nHeight * 3 / 2; + nFrameSize = nStride * nHeight * 3 / 2; + + InputFileHandle = LoadFile(InputFile, &nFileSize); + + if (bJpgFile) { + InputFileFrameCnt = 1; + } else { + if (!InputFileHandle || (nFileSize % nOneSize) != 0) { + ALOGE("%s file is not %dx%d", InputFile, nWidth, nHeight); + goto EXIT1; + } + + InputFileFrameCnt = nFileSize / nOneSize; + } + + if (nWidth % 2 == 1 || nHeight % 2 == 1) { + ALOGE("wxh(%dx%d) should be even", nWidth, nHeight); + goto EXIT1; + } + + // YUVDataWrVir = (AX_VOID *)malloc(nFileSize); + nRet = AX_SYS_MemAlloc(&YUVDataWrPhy, &YUVDataWrVir, nFrameSize, 128, (const AX_S8 *)"SKEL_TEST"); + + if (!YUVDataWrVir) { + ALOGE("malloc fail nRet=0x%x", nRet); + goto EXIT1; + } + + if (SaveResultPath) { + AX_CHAR szPath[512] = {0}; + snprintf(szPath, 511, "%s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_LOG_FILE); + fpResultFile = fopen(szPath, "a+"); + + if (!fpResultFile) { + ALOGE("%s file not exist", szPath); + goto EXIT1; + } + } + + nStartTime = get_tick_count(); + + AX_SKEL_INIT_PARAM_T stInitParam = {0}; + + if (ModelsPath) { + stInitParam.pStrModelDeploymentPath = ModelsPath; + } else { + stInitParam.pStrModelDeploymentPath = "/opt/etc/skelModels"; + } + + nRet = AX_SKEL_Init(&stInitParam); + + nInitElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL init fail, ret = 0x%x", nRet); + + goto EXIT1; + } + + // get version + { + const AX_SKEL_VERSION_INFO_T *pstVersion = NULL; + + nRet = AX_SKEL_GetVersion(&pstVersion); + + if (0 != nRet) { + ALOGI("SKEL get version fail, ret = 0x%x", nRet); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + goto EXIT1; + } + + ALOGI("SKEL version: %s", pstVersion->pstrVersion); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + } + + // get capability + { + const AX_SKEL_CAPABILITY_T *pstCapability = NULL; + + nRet = AX_SKEL_GetCapability(&pstCapability); + + if (0 != nRet) { + ALOGI("SKEL get capability fail, ret = 0x%x", nRet); + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + goto EXIT1; + } + + for (AX_U32 i = 0; i < pstCapability->nPPLConfigSize; i++) { + ALOGI("SKEL capability[%d]: (ePPL: %d, PPLConfigKey: %s)", i, pstCapability->pstPPLConfig[i].ePPL, + pstCapability->pstPPLConfig[i].pstrPPLConfigKey); + } + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + } + + AX_SKEL_HANDLE_PARAM_T stHandleParam = {0}; + + stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; + stHandleParam.nFrameDepth = nFrameDepth; + stHandleParam.nFrameCacheDepth = nCacheListDepth; + stHandleParam.nIoDepth = 0; + stHandleParam.nWidth = nWidth; + stHandleParam.nHeight = nHeight; + + // config settings (if need) + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[16] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; + + // venc_attr_config + // default will be system definition: w*h*3/8 + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVencAttrConfigThreshold = {0}; + if (nJencBufSize > 0) { + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; + stVencAttrConfigThreshold.fValue = (AX_F32)nJencBufSize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stTrackEnableThreshold = {0}; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; + + // detect only (disable track + disable push) + if (nDetectType == 0) { + // track_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + stTrackEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + // detect + track (disable push) + else if (nDetectType == 1) { + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + if (itemIndex > 0) { + stConfig.nSize = itemIndex; + stHandleParam.stConfig = stConfig; + } + + // default NPU + if (nNpuType == 0 || nNpuType == 1) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_NPU_DEFAULT; + } + // STD-NPU + else if (nNpuType == 2) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_1; + } else if (nNpuType == 3) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_2; + } + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_Create(&stHandleParam, &pHandle); + + nCreateElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Create Handle fail, ret = 0x%x", nRet); + + goto EXIT2; + } + + if (fConfidence == 0) { + fConfidence = 0.5; + } + + // set config + { + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[64] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; +#if 0 + // body_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyMaxTargetCount = {0}; + stBodyMaxTargetCount.fValue = (AX_F32)nHumantracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleMaxTargetCount = {0}; + stVehicleMaxTargetCount.fValue = (AX_F32)nVehicletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleMaxTargetCount = {0}; + stCycleMaxTargetCount.fValue = (AX_F32)nCycletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif +#if 0 + // body_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyConfidence = {0}; + stBodyConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stFaceConfidence = {0}; + stFaceConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleConfidence = {0}; + stVehicleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleConfidence = {0}; + stCycleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPlateConfidence = {0}; + stPlateConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // crop_encoder_qpLevel + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"crop_encoder_qpLevel"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCropEncoderQpLevelThreshold = {0}; + stCropEncoderQpLevelThreshold.fValue = 90; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCropEncoderQpLevelThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // body_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stBodyMinSize = {0}; + stBodyMinSize.nWidth = 0; + stBodyMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // face_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stFaceMinSize = {0}; + stFaceMinSize.nWidth = 0; + stFaceMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // vehicle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stVehicleMinSize = {0}; + stVehicleMinSize.nWidth = 0; + stVehicleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // cycle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stCycleMinSize = {0}; + stCycleMinSize.nWidth = 0; + stCycleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // plate_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stPlateMinSize = {0}; + stPlateMinSize.nWidth = 0; + stPlateMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; +#endif + + // detect_roi_polygon + AX_SKEL_ROI_POLYGON_CONFIG_T stDetectRoi = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; + stDetectRoi.bEnable = AX_FALSE; + stDetectRoi.nPointNum = 4; + AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; + stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); + itemIndex++; + + // push_strategy + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; + AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; + stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; + stPushStrategy.nIntervalTimes = 2000; + stPushStrategy.nPushCounts = 1; + stPushStrategy.bPushSameFrame = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_STRATEGY_T); + itemIndex++; + + // 4 crop_encoder +#if 0 + // body_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stBodyCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_crop_encoder"; + stBodyCropEncoderThreshold.fScaleLeft = 0; + stBodyCropEncoderThreshold.fScaleRight = 0; + stBodyCropEncoderThreshold.fScaleTop = 0; + stBodyCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stVehicleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_crop_encoder"; + stVehicleCropEncoderThreshold.fScaleLeft = 0; + stVehicleCropEncoderThreshold.fScaleRight = 0; + stVehicleCropEncoderThreshold.fScaleTop = 0; + stVehicleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stCycleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_crop_encoder"; + stCycleCropEncoderThreshold.fScaleLeft = 0; + stCycleCropEncoderThreshold.fScaleRight = 0; + stCycleCropEncoderThreshold.fScaleTop = 0; + stCycleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stFaceCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_crop_encoder"; + stFaceCropEncoderThreshold.fScaleLeft = 0; + stFaceCropEncoderThreshold.fScaleRight = 0; + stFaceCropEncoderThreshold.fScaleTop = 0; + stFaceCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stPlateCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_crop_encoder"; + stPlateCropEncoderThreshold.fScaleLeft = 0; + stPlateCropEncoderThreshold.fScaleRight = 0; + stPlateCropEncoderThreshold.fScaleTop = 0; + stPlateCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + // push_panorama + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; + AX_SKEL_PUSH_PANORAMA_CONFIG_T stPushPanoramaConfig = {0}; + stPushPanoramaConfig.bEnable = AX_FALSE; + stPushPanoramaConfig.nQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); + itemIndex++; +#if 0 + // push_quality_body + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_body"; + AX_SKEL_ATTR_FILTER_CONFIG_T stBodyAttrFilter = {0}; + stBodyAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_vehicle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_vehicle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stVehicleAttrFilter = {0}; + stVehicleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_cycle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_cycle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stCycleAttrFilter = {0}; + stCycleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_face + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_face"; + AX_SKEL_ATTR_FILTER_CONFIG_T stFaceAttrFilter = {0}; + stFaceAttrFilter.stFaceAttrFilterConfig.nWidth = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.nHeight = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fPitch = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fYaw = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fRoll = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fBlur = 1.0; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceAttrFilter; + itemIndex++; + + // push_quality_plate + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_plate"; + AX_SKEL_ATTR_FILTER_CONFIG_T stPlateAttrFilter = {0}; + stPlateAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; +#endif + // #if 0 + // track_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stTrackEnable = {0}; + stTrackEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; + + // push_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushEnable = {0}; + stPushEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +// #endif +#if 0 + //target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + //push_target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + // push_attr_always + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_attr_always"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushAttrAlways = {0}; + stPushAttrAlways.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushAttrAlways; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +#endif + // analyzer_attr_config + // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; + // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; + // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; + + // stAnalyze.nSize = 1; + // AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_FACE_ATTRIBUTE; + // stAnalyze.peItems = &analyze_attribute; + + // stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stAnalyze; + // stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ANALYZER_CONFIG_T); + // itemIndex++; + + stConfig.nSize = itemIndex; + + nRet = AX_SKEL_SetConfig(pHandle, &stConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_SetConfig, ret = 0x%x", nRet); + + goto EXIT2; + } + } + + // get config + { + const AX_SKEL_CONFIG_T *pstConfig = NULL; + + nRet = AX_SKEL_GetConfig(pHandle, &pstConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_GetConfig, ret = 0x%x", nRet); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + + goto EXIT2; + } + + ParseConfigParam(pstConfig); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + } + + ALOGN("Task infomation:"); + ALOGN("\tInput file: %s", InputFile); + ALOGN("\tInput file resolution: %dx%d", nWidth, nHeight); + ALOGN("\tRepeat times: %d", nRepeatTimes); + ALOGN("SKEL Init Elapse:"); + ALOGN("\tAX_SKEL_Init: %lld ms", nInitElasped); + ALOGN("\tAX_SKEL_Create: %lld ms", nCreateElasped); + + OUTPUT_LOG_SAVE("==============================%s process result:==============================", InputFile); + + if (InputFileFrameCnt > 1) { + InputFileFrameCnt = InputFileFrameCnt + BLACK_VIDEO_FRAME_COUNT; + } + + // create frame mgr + FrameMgrCreate(nFrameSize, nFrameDepth + nCacheListDepth); + + for (AX_U32 nRepeat = 0; nRepeat < nRepeatTimes; nRepeat++) { + ALOGN("#####SKEL Process times: %d", nRepeat + 1); + + OUTPUT_LOG_SAVE("#####SKEL Process times: %d", nRepeat + 1); + + STAT_OBJECT_NUM_T tObjectTrackNum = {0}; + STAT_OBJECT_NUM_T tObjectPushNum = {0}; + + for (AX_U32 nFrameCnt = 0; nFrameCnt < InputFileFrameCnt; nFrameCnt++) { + AX_BLK blkId = AX_INVALID_BLOCKID; + + if (InputFileFrameCnt != 1) { + if (FrameSkipCtrl(nSrcFrameRate, nDstFrameRate, nSkelFrameId)) { + nSkelFrameId++; + + LoadFileToMem(InputFileHandle, 0, NULL, nOneSize); + + if (nInterval > 0) { + usleep(nInterval); + } + continue; + } else { + if (!FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId)) { + ALOGE("FrameMgrGet fail"); + + goto EXIT1; + } + + if (nOneSize != nFrameSize) { + LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, OneYUVDataVir, nOneSize, nWidth, nHeight, YUVDataPhy, + YUVDataVir, nFrameSize, nStride, nHeight); + } else { + LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nOneSize); + } + } + } else { + FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId); + + if (nRet != 0 || YUVDataPhy == 0 || YUVDataVir == NULL) { + ALOGE("Load input file fail"); + goto EXIT1; + } + + if (bJpgFile) { + nRet = DecodeJpeg(InputFileHandle, YUVDataPhy, YUVDataVir, nWidth, nHeight); + if (0 != nRet) { + ALOGE("DecodeJpeg failed!"); + goto EXIT3; + } + } else { + if (nOneSize != nFrameSize) { + nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, + (AX_S8 *)"SKEL_TEST"); + + nRet = LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, OneYUVDataVir, nOneSize, nWidth, nHeight, YUVDataPhy, + YUVDataVir, nFrameSize, nStride, nHeight); + } else { + nRet = LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nFrameSize); + } + } + } + stFrame.stFrame.u32Width = nWidth; + stFrame.stFrame.u32Height = nHeight; + stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stFrame.stFrame.u32FrameSize = nFrameSize; + stFrame.stFrame.u32PicStride[0] = nStride; + stFrame.stFrame.u32PicStride[1] = nStride; + stFrame.stFrame.u32PicStride[2] = nStride; + stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; + stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; + stFrame.stFrame.u64PhyAddr[2] = 0; + stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); + stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); + stFrame.stFrame.u64VirAddr[2] = 0; + stFrame.stFrame.u32BlkId[0] = blkId; + stFrame.stFrame.u32BlkId[1] = 0; + stFrame.stFrame.u32BlkId[2] = 0; + ALOGN("*****SKEL Frame(%d) Process", nFrameCnt + 1); + + // OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); + + stFrame.nFrameId = nSkelFrameId++; + nStartTime = get_tick_count(); + + while (1) { + nRet = AX_SKEL_SendFrame(pHandle, &stFrame, 0); + + if (nRet == 0) { + break; + } + + usleep(1000); + } + + FrameMgrRelease(blkId); + + nProcessElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Process fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("SKEL Process Elapse:"); + ALOGN("\tAX_SKEL_SendFrame: %lld ms", nProcessElasped); + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_GetResult(pHandle, &pstResult, -1); + + nResultElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL get result fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("\tAX_SKEL_GetResult: %lld ms", nResultElasped); + + nResultElaspedTotal += nResultElasped; + + if (nProcessElasped + nResultElasped > nResultElaspedMax) { + nResultElaspedMax = nProcessElasped + nResultElasped; + } + + if (nProcessElasped + nResultElasped < nResultElaspedMin) { + nResultElaspedMin = nProcessElasped + nResultElasped; + } + + ALOGN("SKEL Process Result:"); + + ALOGI("\tFrameId: %lld", pstResult->nFrameId); + ALOGI("\tnOriginal WxH: %dx%d", pstResult->nOriginalWidth, pstResult->nOriginalHeight); + + ALOGN("\tObject Num: %d", pstResult->nObjectSize); + + AX_U32 nSkelSize = 0; + AI_Detection_SkelResult_t Skels[SKEL_SAMPLE_OBJECT_SIZE] = {0}; + for (AX_U32 i = 0; i < pstResult->nObjectSize; i++) { + AX_SKEL_OBJECT_ITEM_T *pstItems = &pstResult->pstObjectItems[i]; + + ALOGI("\t\tFrameId: %lld", pstItems->nFrameId); + ALOGI("\t\tTrackId: %lld, TrackState: %d", pstItems->nTrackId, pstItems->eTrackState); + + ALOGN("\t\tRect[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, pstItems->stRect.fY, pstItems->stRect.fW, pstItems->stRect.fH, + pstItems->fConfidence); + StatTrackMgr(pstItems, &tObjectTrackNum); + StatPushMgr(pstItems, &tObjectPushNum); + + // get detect box only new or update state + if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW || + pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) && + nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { + Skels[nSkelSize].pstrObjectCategory = pstItems->pstrObjectCategory; + Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; + Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; + Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; + Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; + + OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, + pstItems->pstrObjectCategory, pstItems->stRect.fX, pstItems->stRect.fY, + pstItems->stRect.fW, pstItems->stRect.fH, pstItems->fConfidence); + + ALOGN("\t\t[%d]Point Set Size: %d", i, pstItems->nPointSetSize); + + // point + Skels[nSkelSize].nPointNum = AX_MIN(DETECT_SKEL_POINT_COUNT, pstItems->nPointSetSize); + for (AX_U32 j = 0; j < Skels[i].nPointNum; j++) { + ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, + pstItems->pstPointSet[j].pstrObjectCategory, pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, pstItems->pstPointSet[j].fConfidence); + Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; + Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; + + OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", j, + pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + } + + // face attr + if (nPPL == AX_SKEL_PPL_FACE) { + ALOGN("\t\tFace Attribute:"); + ALOGI("\t\t\tfYaw %f,", pstItems->stFaceAttr.fYaw); + ALOGI("\t\t\tfPitch %f,", pstItems->stFaceAttr.fPitch); + ALOGI("\t\t\tfRoll %f,", pstItems->stFaceAttr.fRoll); + ALOGI("\t\t\tfMask %f,", pstItems->stFaceAttr.fMask); + ALOGI("\t\t\tage %d", pstItems->stFaceAttr.nAge); + ALOGI("\t\t\tgender %d", pstItems->stFaceAttr.nGender); + } + nSkelSize++; + } + + if (AX_SKEL_TRACK_STATUS_SELECT == pstItems->eTrackState) { + ATTRINFO_T AttrInfo; + memset(&AttrInfo, 0x00, sizeof(AttrInfo)); + AttrParser(pstItems, &AttrInfo); + + if (pstItems->bCropFrame && pstItems->stCropFrame.pFrameData && + 0 < pstItems->stCropFrame.nFrameDataSize && SaveResultPath) { + // save attribute + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[FACE ATTR] %s", arrDat); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[PLATE ATTR] %s", arrDat); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, + "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[VEHICLE ATTR] %s", arrDat); + } else { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, pstItems->nTrackId, pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", SaveResultPath, pstItems->pstrObjectCategory, arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write crop jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, + fp_w); + fclose(fp_w); + } + } + + // panora frame + if (pstItems->bPanoraFrame && pstItems->stPanoraFrame.pFrameData && + 0 < pstItems->stPanoraFrame.nFrameDataSize && SaveResultPath) { + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, + "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", SaveResultPath, pstItems->pstrObjectCategory, arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write panora jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, + pstItems->stPanoraFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + } + } + + // feature + ALOGI("\t\tFeature Size: %d", pstItems->nFeatureSize); + if (pstItems->pstFeatureItem && 0 < pstItems->nFeatureSize && SaveResultPath) { + AX_CHAR strFile[256] = {0}; + sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", SaveResultPath, pstItems->pstrObjectCategory, + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, i); + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", strFile, + pstItems->pstFeatureItem[0].nValueSize); + fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, + pstItems->pstFeatureItem[0].nValueSize, fp_w); + fclose(fp_w); + } + } + + // meta + ALOGI("\t\tMeta Size: %d", pstItems->nMetaInfoSize); + if (pstItems->pstMetaInfo && 0 < pstItems->nMetaInfoSize) { + for (AX_U32 i = 0; i < pstItems->nMetaInfoSize; i++) { + ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, + pstItems->pstMetaInfo[i].pstrValue); + } + } + + // binds + ALOGI("\t\tBind Size: %d", pstItems->nObjectBindSize); + if (pstItems->pstObjectBind && 0 < pstItems->nObjectBindSize) { + for (AX_U32 i = 0; i < pstItems->nObjectBindSize; i++) { + ALOGI("\t\t[%s] [TrackId] %lld bind to ObjectBind[%d]: %s [TrackId] %lld", + pstItems->pstrObjectCategory, pstItems->nTrackId, i, + pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); + } + } + } + + ALOGI("\tnCacheListSize: %d", pstResult->nCacheListSize); + if (pstResult->pstCacheList) { + for (AX_U32 i = 0; i < pstResult->nCacheListSize; i++) { + ALOGI("\t\tCacheList[%d] FrameId: %lld", i, pstResult->pstCacheList[i].nFrameId); + } + } + + if (ReWritePath && pstResult->nObjectSize > 0) { + memcpy(YUVDataWrVir, YUVDataVir, nFrameSize); + + YUV_IMAGE_T YUVImage = {0}; + YUVImage.pImage = YUVDataWrVir; + YUVImage.nWidth = nWidth; + YUVImage.nHeight = nHeight; + YUVImage.stride = nStride; + YUVImage.nSize = nFrameSize; + YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; + + for (size_t i = 0; i < nSkelSize; i++) { + // draw rect + AX_CHAR ObjectCategory = Skels[i].pstrObjectCategory[0]; + AX_S16 x0 = Skels[i].tBox.fX; + AX_S16 y0 = Skels[i].tBox.fY; + AX_U16 w = Skels[i].tBox.fW; + AX_U16 h = Skels[i].tBox.fH; + YUV_COLOR RectColor = YUV_WHITE; + + switch (ObjectCategory) { + case 'b': // body + RectColor = YUV_WHITE; + break; + case 'v': // vehicle + RectColor = YUV_PURPLE; + break; + case 'c': // cycle + RectColor = YUV_GREEN; + break; + case 'f': // face + RectColor = YUV_YELLOW; + break; + case 'p': // plate + RectColor = YUV_RED; + break; + default: + RectColor = YUV_WHITE; + break; + } + + DrawRect(&YUVImage, x0, y0, w, h, RectColor); + + if (Skels[i].nPointNum > 0) { + // draw point + for (size_t j = 0; j < Skels[i].nPointNum; j++) { + x0 = Skels[i].tPoint[j].fX; + y0 = Skels[i].tPoint[j].fY; + + DrawPoint(&YUVImage, x0, y0, 4, x0 * (4 - 1), y0 * (4 - 1), YUV_DARK_GREEN); + } + } + } + + AX_CHAR arrWrFile[512] = {0}; + sprintf(arrWrFile, "%s_result_%d_%d.jpg", InputFile, nFrameCnt + 1, nRepeat + 1); + + AX_CHAR *fileName = NULL; + AX_U32 fileNameLen = strlen(arrWrFile); + AX_CHAR fullFileName[256] = {0}; + AX_S32 i = fileNameLen - 1; + + if (fileNameLen > 0) { + for (; i >= 0; i--) { + if (arrWrFile[i] == '/') { + break; + } + } + } + + if (i < 0) { + i = 0; + } else if (i < fileNameLen - 1) { + i++; + } + + fileName = (AX_CHAR *)&arrWrFile[i]; + + sprintf(fullFileName, "%s/%s", ReWritePath, fileName); + + EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, + nFrameSize); + } + + if (pstResult) { + AX_SKEL_Release((AX_VOID *)pstResult); + } + + OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process End (Elasped: %lld ms)", nFrameCnt + 1, nProcessElasped); + + if (nInterval > 0) { + usleep(nInterval); + } + } + + ALOGN("SKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, tObjectTrackNum.nVehicleNum, tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, tObjectTrackNum.nPlateNum); + + OUTPUT_LOG_SAVE("\nSKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, tObjectTrackNum.nVehicleNum, tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, tObjectTrackNum.nPlateNum); + + ALOGN("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectPushNum.nBodyNum, tObjectPushNum.nVehicleNum, tObjectPushNum.nCycleNum, tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + OUTPUT_LOG_SAVE("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]\n", + tObjectPushNum.nBodyNum, tObjectPushNum.nVehicleNum, tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, tObjectPushNum.nPlateNum); + + if (InputFileHandle) { + ReLoadFile(InputFileHandle); + } + } + + // if(AX_SUCCESS != nRet){ + // ALOGE("AX_POOL_DestroyPool failed! Error Code:0x%X\n", nRet); + // } + + ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", nRepeatTimes, + nResultElaspedMin, (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, nResultElaspedMax); + + OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", nRepeatTimes, + nResultElaspedMin, (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + +EXIT3: + if (pHandle) { + AX_SKEL_Destroy(pHandle); + } + +EXIT2: + AX_SKEL_DeInit(); + +EXIT1: + AX_ENGINE_Deinit(); + +EXIT0: + // if (YUVDataVir) { + // if (YUVDataPhy == 0) { + // free(YUVDataVir); + // } else { + // AX_SYS_MemFree(YUVDataPhy, YUVDataVir); + // } + // } + if (YUVDataWrVir) { + if (YUVDataWrPhy == 0) { + free(YUVDataWrVir); + } else { + AX_SYS_MemFree(YUVDataWrPhy, YUVDataWrVir); + } + } + + if (OneYUVDataVir) { + if (OneYUVDataPhy == 0) { + free(OneYUVDataVir); + } else { + AX_SYS_MemFree(OneYUVDataPhy, OneYUVDataVir); + } + } + + FrameMgrDestroy(); + + if (fpResultFile) { + fclose(fpResultFile); + } + + AX_VDEC_Deinit(); + + AX_VENC_Deinit(); + + AX_IVPS_Deinit(); + + AX_SYS_Deinit(); + + return (0 != nRet) ? -1 : 0; +} diff --git a/projects/llm_framework/main_skel/src/main.cpp.bak b/projects/llm_framework/main_skel/src/main.cpp.bak new file mode 100644 index 00000000..ff46984d --- /dev/null +++ b/projects/llm_framework/main_skel/src/main.cpp.bak @@ -0,0 +1,525 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "common.hpp" +#include +#include +#include + +#include "../../../../SDK/components/utilities/include/sample_log.h" + +using namespace StackFlows; + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + SLOGW("llm_yolo will be exit!"); + main_exit_flage = 1; +} + +static std::string base_model_path_; +static std::string base_model_config_path_; + +typedef struct { + std::string yolo_model; + std::string model_type = "detect"; + std::vector cls_name; + int img_h = 640; + int img_w = 640; + int cls_num = 80; + float pron_threshold = 0.45f; + float nms_threshold = 0.45; +} yolo_config; + +typedef std::function &data, bool finish)> task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: +public: + yolo_config mode_config_; + std::string model_; + // std::unique_ptr yolo_; + std::string response_format_; + std::vector inputs_; + std::vector image_data_; + bool enoutput_; + bool enstream_; + static int ax_init_flage_; + task_callback_t out_callback_; + + bool parse_config(const nlohmann::json &config_body) + { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } else + throw std::string("error"); + } catch (...) { + SLOGE("setup config_body error"); + return true; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + nlohmann::json file_body; + std::list config_file_paths; + config_file_paths.push_back(std::string("./") + model_ + ".json"); + config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + CONFIG_AUTO_SET(file_body["mode_param"], yolo_model); + CONFIG_AUTO_SET(file_body["mode_param"], img_h); + CONFIG_AUTO_SET(file_body["mode_param"], img_w); + CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], cls_name); + CONFIG_AUTO_SET(file_body["mode_param"], cls_num); + CONFIG_AUTO_SET(file_body["mode_param"], model_type); + mode_config_.yolo_model = base_model + mode_config_.yolo_model; + // skel_ = std::make_unique(); + // if (0 != skel_->Init(mode_config_.yolo_model.c_str())) { + // SLOGE("Init yolo_model model failed!\n"); + // return -5; + // } + } catch (...) { + SLOGE("config false"); + return -6; + } + return 0; + } + + std::string format_float(double value, int decimal_places) + { + std::ostringstream out; + out << std::fixed << std::setprecision(decimal_places) << value; + return out.str(); + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + bool inference(const std::string &msg) + { + try { + // SLOGI("msg:%s", msg.c_str()); + std::ofstream outFile("output.bin"); + outFile << msg; + outFile.close(); + cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); + if (src.empty()) return true; + SLOGI(""); + std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); + common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); + int ret = -1; + // yolo_->SetInput(image.data(), 0); + // if (0 != yolo_->RunSync()) { + // SLOGE("Run yolo model failed!\n"); + // throw std::string("yolo_ RunSync error"); + // } + // std::vector objects; + // yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, + // mode_config_.pron_threshold, mode_config_.nms_threshold, objects, + // mode_config_.model_type); + std::vector yolo_output; + // for (size_t i = 0; i < objects.size(); i++) { + // const detection::Object &obj = objects[i]; + // nlohmann::json output; + // output["class"] = mode_config_.cls_name[obj.label]; + // output["confidence"] = format_float(obj.prob, 2); + // output["bbox"] = nlohmann::json::array(); + // output["bbox"].push_back(format_float(obj.rect.x, 0)); + // output["bbox"].push_back(format_float(obj.rect.y, 0)); + // output["bbox"].push_back(format_float(obj.rect.x + obj.rect.width, 0)); + // output["bbox"].push_back(format_float(obj.rect.y + obj.rect.height, 0)); + // if (mode_config_.model_type == "segment") output["mask"] = obj.mask_feat; + // if (mode_config_.model_type == "pose") output["kps"] = obj.kps_feat; + // if (mode_config_.model_type == "obb") output["angle"] = obj.angle; + // yolo_output.push_back(output); + // if (out_callback_) out_callback_(yolo_output, false); + // } + if (out_callback_) out_callback_(yolo_output, true); + } catch (...) { + SLOGW("yolo_->Run have error!"); + return true; + } + return false; + } + + void _ax_init() + { + if (!ax_init_flage_) { + int ret = AX_SYS_Init(); + if (0 != ret) { + fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret); + } + // AX_ENGINE_NPU_ATTR_T npu_attr; + // memset(&npu_attr, 0, sizeof(npu_attr)); + // ret = AX_ENGINE_Init(&npu_attr); + if (0 != ret) { + fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret); + } + } + ax_init_flage_++; + } + + void _ax_deinit() + { + if (ax_init_flage_ > 0) { + --ax_init_flage_; + if (!ax_init_flage_) { + // AX_ENGINE_Deinit(); + AX_SYS_Deinit(); + } + } + } + + llm_task(const std::string &workid) + { + _ax_init(); + } + + ~llm_task() + { + _ax_deinit(); + } +}; +int llm_task::ax_init_flage_ = 0; +#undef CONFIG_AUTO_SET + +class llm_yolo : public StackFlow { +private: + int task_count_; + std::unordered_map> llm_task_; + int _load_config() + { + if (base_model_path_.empty()) { + base_model_path_ = sys_sql_select("config_base_mode_path"); + } + if (base_model_config_path_.empty()) { + base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); + } + if (base_model_path_.empty() || base_model_config_path_.empty()) { + return -1; + } else { + SLOGI("llm_yolo::_load_config success"); + return 0; + } + } + +public: + llm_yolo() : StackFlow("yolo") + { + task_count_ = 1; + repeat_event(1000, std::bind(&llm_yolo::_load_config, this)); + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::vector &data, + bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + for (const auto &jsonObj : data) { + data_body["delta"].push_back(jsonObj); + } + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); + return; + } + const std::string *next_data = &data; + bool enstream = (object.find("stream") == std::string::npos) ? false : true; + int ret; + std::string tmp_msg1; + if (enstream) { + static std::unordered_map stream_buff; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg1; + } + // must encode base64 + std::string tmp_msg2; + ret = decode_base64((*next_data), tmp_msg2); + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg2; + if (llm_task_obj->inference(*next_data)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, unit_name_); + return -1; + } + + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + SLOGE("setup json format error."); + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + + llm_task_obj->set_output(std::bind(&llm_yolo::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + + for (const auto input : llm_task_obj->inputs_) { + if (input.find("yolo") != std::string::npos) { + llm_channel->subscriber_work_id( + "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + } else if (input.find("sys") != std::string::npos) { + // TODO:... + } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + return 0; + } else { + SLOGE("load_mode Failed"); + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::link:%s", data.c_str()); + int ret = 1; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + if (data.find("yolo") != std::string::npos) { + ret = llm_channel->subscriber_work_id( + "", + std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->inputs_.push_back(data); + } else if (data.find("sys") != std::string::npos) { + // TODO:... + } + if (ret) { + error_body["code"] = -20; + error_body["message"] = "link false"; + send("None", "None", error_body, work_id); + return; + } else { + send("None", "None", LLM_NO_ERROR, work_id); + } + } + + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::unlink:%s", data.c_str()); + int ret = 0; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + llm_channel->stop_subscriber_work_id(data); + auto llm_task_obj = llm_task_[work_id_num]; + for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { + if (*it == data) { + it = llm_task_obj->inputs_.erase(it); + } else { + ++it; + } + } + send("None", "None", LLM_NO_ERROR, work_id); + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::taskinfo:%s", data.c_str()); + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("yolo.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs_"] = llm_task_obj->inputs_; + send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::exit:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_yolo() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main() +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_yolo llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_skel/src/runner/YuvHandler.c b/projects/llm_framework/main_skel/src/runner/YuvHandler.c new file mode 100644 index 00000000..60d1fb46 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/YuvHandler.c @@ -0,0 +1,269 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include "YuvHandler.h" +#include +#include +#include +#include + +/// +typedef struct _YUV_COLOR_S { + AX_U8 Y; + AX_U8 U; + AX_U8 V; +} YUV_COLOR_S; + +YUV_COLOR_S g_YuvColors[YUV_COLOR_MAX] = { + {0x00, 0x00, 0x00}, // green + {0x00, 0x00, 0xff}, // red + {0x00, 0xff, 0x00}, // blue + {0x00, 0xff, 0xff}, // purple + {0xff, 0x00, 0x00}, // dark green + {0xff, 0x00, 0xff}, // yellow + {0xff, 0xff, 0x00}, // light blue + {0xff, 0xff, 0xff}, // light purple + {0x00, 0x80, 0x80}, // dark black + {0x80, 0x80, 0x80}, // gray + {0xff, 0x80, 0x80} // white +}; + +static AX_VOID DrawPointInternal(YUV_IMAGE_T *pstImage, AX_U8 *y, AX_U8 *u, AX_U8 *v, AX_U16 x0, AX_U16 y0, + YUV_COLOR eColor) { + if (!pstImage) { + return; + } + + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + AX_U32 y_offset = 0; + AX_U32 u_offset = 0; + AX_U32 v_offset = 0; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: // YUV420 I420 + /* YYYY...UUUU...VVVV */ + y_offset = (AX_U32)(y0 * m_stride + x0); + u_offset = (AX_U32)((y0 / 2) * (m_stride / 2) + x0 / 2); + v_offset = u_offset; + + y[y_offset] = g_YuvColors[eColor].Y; + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV420_SEMIPLANAR: // YUV420SP NV12 + /* YYYY...UVUV */ + y_offset = (AX_U32)(y0 * m_stride + x0); + u_offset = (AX_U32)((y0 / 2) * m_stride + x0 / 2 * 2); + v_offset = u_offset + 1; + + if (g_YuvColors[eColor].Y == 0xFF) { + y[y_offset] = g_YuvColors[eColor].Y; + } else { + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + } + break; + + case AX_FORMAT_YUV420_SEMIPLANAR_VU: // YUV420SP NV21 + /* YYYY...VUVU */ + y_offset = (AX_U32)(y0 * m_stride + x0); + v_offset = (AX_U32)((y0 / 2) * m_stride + x0 / 2 * 2); + u_offset = v_offset + 1; + + y[y_offset] = g_YuvColors[eColor].Y; + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + /* UYVYUYVY */ + u_offset = (AX_U32)(y0 * m_stride * 2 + x0 / 2 * 4); + v_offset = u_offset + 2; + y_offset = u_offset + 1; + y[y_offset] = g_YuvColors[eColor].Y; + y[y_offset + 2] = g_YuvColors[eColor].Y; + y[u_offset] = g_YuvColors[eColor].U; + y[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + /* YUYVYUYV */ + y_offset = (AX_U32)(y0 * m_stride * 2 + x0 / 2 * 4); + u_offset = y_offset + 1; + v_offset = u_offset + 2; + + y[y_offset] = g_YuvColors[eColor].Y; + y[y_offset + 2] = g_YuvColors[eColor].Y; + y[u_offset] = g_YuvColors[eColor].U; + y[v_offset] = g_YuvColors[eColor].V; + break; + + default: + break; + } +} + +AX_VOID DrawPoint(YUV_IMAGE_T *pstImage, AX_S16 x, AX_S16 y, AX_U8 nScale /* = 1*/, AX_S16 x_offset /* = 0*/, + AX_S16 y_offset /* = 0*/, YUV_COLOR eColor /* = YUV_GREEN*/) { + if (!pstImage) { + return; + } + + AX_U8 *m_pImage = pstImage->pImage; + AX_S16 m_nWidth = pstImage->nWidth; + AX_S16 m_nHeight = pstImage->nHeight; + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + AX_U8 *pY = NULL; + AX_U8 *pU = NULL; + AX_U8 *pV = NULL; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU + m_stride * m_nHeight / 4; + break; + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU; + break; + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + pY = m_pImage; + pU = pY; + pV = pY; + break; + default: + break; + } + + AX_S16 nXStart = 0; + AX_S16 nYStart = 0; + for (AX_U32 hScale = 0; hScale < nScale; hScale++) { + for (AX_U32 wScale = 0; wScale < nScale; wScale++) { + nXStart = x * nScale + hScale - x_offset; + nYStart = y * nScale + wScale - y_offset; + if (nXStart < 0 || nXStart > m_nWidth) { + break; + } + + if (nYStart < 0 || nYStart > m_nHeight) { + break; + } + + DrawPointInternal(pstImage, pY, pU, pV, nXStart, nYStart, eColor); + } + } +} + +AX_VOID DrawLine(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_S16 x1, AX_S16 y1, YUV_COLOR eColor /* = YUV_GREEN*/, + AX_U8 nScale /* = 1*/) + +{ + if (!pstImage) { + return; + } + + AX_U8 *m_pImage = pstImage->pImage; + AX_S16 m_nWidth = pstImage->nWidth; + AX_S16 m_nHeight = pstImage->nHeight; + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + x0 = (x0 < 0) ? 0 : x0; + y0 = (y0 < 0) ? 0 : y0; + x1 = (x1 < 0) ? 0 : x1; + y1 = (y1 < 0) ? 0 : y1; + + x0 = (x0 >= m_nWidth) ? m_nWidth - 1 : x0; + y0 = (y0 >= m_nHeight) ? m_nHeight - 1 : y0; + x1 = (x1 >= m_nWidth) ? m_nWidth - 1 : x1; + y1 = (y1 >= m_nHeight) ? m_nHeight - 1 : y1; + + AX_U16 dx = (x0 > x1) ? (x0 - x1) : (x1 - x0); + AX_U16 dy = (y0 > y1) ? (y0 - y1) : (y1 - y0); + + AX_S16 xstep = (x0 < x1) ? 1 : -1; + AX_S16 ystep = (y0 < y1) ? 1 : -1; + AX_S16 nstep = 0, eps = 0; + + AX_U8 *pY = NULL; + AX_U8 *pU = NULL; + AX_U8 *pV = NULL; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU + m_stride * m_nHeight / 4; + break; + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU; + break; + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + pY = m_pImage; + pU = pY; + pV = pY; + break; + default: + break; + } + + AX_U16 x = x0; + AX_U16 y = y0; + if (dx > dy) { + while (nstep <= dx) { + if (nScale == 1) { + DrawPointInternal(pstImage, pY, pU, pV, x, y, eColor); + } else { + DrawPoint(pstImage, x, y, nScale, x * (nScale - 1), y * (nScale - 1), eColor); + } + eps += dy; + if ((eps << 1) >= dx) { + y += ystep; + eps -= dx; + } + x += xstep; + nstep++; + } + } else { + while (nstep <= dy) { + if (nScale == 1) { + DrawPointInternal(pstImage, pY, pU, pV, x, y, eColor); + } else { + DrawPoint(pstImage, x, y, nScale, x * (nScale - 1), y * (nScale - 1), eColor); + } + eps += dx; + if ((eps << 1) >= dy) { + x += xstep; + eps -= dy; + } + y += ystep; + nstep++; + } + } +} + +AX_VOID DrawRect(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_U16 w, AX_U16 h, YUV_COLOR eColor /* = YUV_GREEN*/) { + if (w > 0 && h > 0) { + DrawLine(pstImage, x0, y0, x0 + w, y0, eColor, 1); + DrawLine(pstImage, x0, y0, x0, y0 + h, eColor, 1); + DrawLine(pstImage, x0 + w, y0, x0 + w, y0 + h, eColor, 1); + DrawLine(pstImage, x0, y0 + h, x0 + w, y0 + h, eColor, 1); + } +} diff --git a/projects/llm_framework/main_skel/src/runner/YuvHandler.h b/projects/llm_framework/main_skel/src/runner/YuvHandler.h new file mode 100644 index 00000000..ce4f6d4f --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/YuvHandler.h @@ -0,0 +1,46 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _YUV_HANDLER_H_ +#define _YUV_HANDLER_H_ + +#include "ax_global_type.h" +#include "ax_sys_api.h" + +typedef enum _YUV_COLOR { + YUV_GREEN = 0, + YUV_RED, + YUV_BLUE, + YUV_PURPLE, + YUV_DARK_GREEN, + YUV_YELLOW, + YUV_LIGHT_BLUE, + YUV_LIGHT_PURPLE, + YUV_DARK_BLACK, + YUV_GRAY, + YUV_WHITE, + YUV_COLOR_MAX, +} YUV_COLOR; + +typedef struct _YUV_IMAGE_T { + AX_U8 *pImage; + AX_U16 nWidth; + AX_U16 nHeight; + AX_S16 stride; + AX_U32 nSize; + AX_IMG_FORMAT_E eType; +} YUV_IMAGE_T; + +AX_VOID DrawLine(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_S16 x1, AX_S16 y1, YUV_COLOR eColor, AX_U8 nScale); +AX_VOID DrawRect(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_U16 w, AX_U16 h, YUV_COLOR eColor); +AX_VOID DrawPoint(YUV_IMAGE_T *pstImage, AX_S16 x, AX_S16 y, AX_U8 nScale, AX_S16 x_offset, AX_S16 y_offset, + YUV_COLOR eColor); + +#endif /* _YUV_HANDLER_H_ */ diff --git a/projects/llm_framework/main_skel/src/runner/attrParser.cpp b/projects/llm_framework/main_skel/src/runner/attrParser.cpp new file mode 100644 index 00000000..8e386a90 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/attrParser.cpp @@ -0,0 +1,224 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include "attrParser.h" +#include "picojson.h" + +#define PICO_OBJECT get() +#define PICO_OBJECT_SIZE PICO_OBJECT.size() +#define PICO_ARRAY get() +#define PICO_ARRAY_SIZE PICO_ARRAY.size() +#define PICO_VALUE get() +#define PICO_BOOL get() +#define PICO_STRING get() +#define PICO_ROOT obj.PICO_OBJECT + +typedef struct _AI_Face_Attr_t { + AX_F32 fYaw; + AX_F32 fPitch; + AX_F32 fRoll; + AX_F32 fMask; + AX_U8 age; + AX_U8 gender; + AX_F32 fScore; +} AI_Face_Attr_t; + +typedef struct _AI_Plate_Attr_t { + AX_BOOL bExist; + AX_BOOL bValid; + /* + string: + blue + yellow + black + white + green + small_new_energy + large_new_energy + absence + unknown + */ + std::string strPlateColor; + /* + string: + one_row + two_rows + unknown + */ + std::string strPlateType; + /* string: UTF8*/ + std::string strPlateCode; + + _AI_Plate_Attr_t() { + bExist = AX_FALSE; + bValid = AX_FALSE; + strPlateColor = ""; + strPlateType = ""; + strPlateCode = ""; + } +} AI_Plate_Attr_t; + +static AX_VOID FaceAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Face_Attr_t face_attr; + + pAttrInfo->eType = ATTR_TYPE_FACE; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "face_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + // age + face_attr.age = PICO_ROOT["age"].PICO_VALUE; + printf("ageis %d\n",face_attr.age); + // gender + face_attr.gender = PICO_ROOT["gender"].PICO_VALUE; + // yaw + face_attr.fYaw = PICO_ROOT["yaw"].PICO_VALUE; + // pitch + face_attr.fPitch = PICO_ROOT["pitch"].PICO_VALUE; + // roll + face_attr.fRoll = PICO_ROOT["roll"].PICO_VALUE; + // mask + face_attr.fMask = PICO_ROOT["mask"].PICO_VALUE; + // score + face_attr.fScore = PICO_ROOT["score"].PICO_VALUE; + printf("score is %f\n",face_attr.fScore); + } + } +} + +static AX_VOID PlateAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Plate_Attr_t plat_attr; + + pAttrInfo->eType = ATTR_TYPE_PLATE; + + plat_attr.bExist = AX_FALSE; + plat_attr.bValid = AX_FALSE; + plat_attr.strPlateColor = ""; + plat_attr.strPlateType = ""; + plat_attr.strPlateCode = ""; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "plate_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + + plat_attr.bExist = AX_TRUE; + // color + plat_attr.strPlateColor = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("color")) { + plat_attr.strPlateColor = PICO_ROOT["color"].PICO_OBJECT["name"].PICO_STRING; + } + + // style + plat_attr.strPlateType = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("style")) { + plat_attr.strPlateType = PICO_ROOT["style"].PICO_OBJECT["name"].PICO_STRING; + } + + // code + plat_attr.strPlateCode = PICO_ROOT["code_result"].PICO_STRING; + + if (PICO_ROOT["code_killed"].PICO_BOOL) { + plat_attr.bValid = AX_FALSE; + } else { + plat_attr.bValid = AX_TRUE; + } + + pAttrInfo->bExist = AX_TRUE; + + strncpy(pAttrInfo->tPlateInfo.szColor, (const AX_CHAR *)plat_attr.strPlateColor.c_str(), sizeof(pAttrInfo->tPlateInfo.szColor) - 1); + + pAttrInfo->tPlateInfo.bValid = plat_attr.bValid; + strncpy(pAttrInfo->tPlateInfo.szNum, (const AX_CHAR *)plat_attr.strPlateCode.c_str(), sizeof(pAttrInfo->tPlateInfo.szNum) - 1); + } + } +} + +static AX_VOID VehicleAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Plate_Attr_t plat_attr; + + pAttrInfo->eType = ATTR_TYPE_VEHICLE; + + plat_attr.bExist = AX_FALSE; + plat_attr.bValid = AX_FALSE; + plat_attr.strPlateColor = ""; + plat_attr.strPlateType = ""; + plat_attr.strPlateCode = ""; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "plate_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + + plat_attr.bExist = AX_TRUE; + // color + plat_attr.strPlateColor = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("color")) { + plat_attr.strPlateColor = PICO_ROOT["color"].PICO_OBJECT["name"].PICO_STRING; + } + + // style + plat_attr.strPlateType = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("style")) { + plat_attr.strPlateType = PICO_ROOT["style"].PICO_OBJECT["name"].PICO_STRING; + } + + // code + plat_attr.strPlateCode = PICO_ROOT["code_result"].PICO_STRING; + + if (PICO_ROOT["code_killed"].PICO_BOOL) { + plat_attr.bValid = AX_FALSE; + } else { + plat_attr.bValid = AX_TRUE; + } + + pAttrInfo->bExist = AX_TRUE; + + strncpy(pAttrInfo->tPlateInfo.szColor, (const AX_CHAR *)plat_attr.strPlateColor.c_str(), sizeof(pAttrInfo->tPlateInfo.szColor) - 1); + + pAttrInfo->tPlateInfo.bValid = plat_attr.bValid; + strncpy(pAttrInfo->tPlateInfo.szNum, (const AX_CHAR *)plat_attr.strPlateCode.c_str(), sizeof(pAttrInfo->tPlateInfo.szNum) - 1); + } + } +} + +AX_VOID AttrParser(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + if (!pstObjectItems || !pAttrInfo) { + return; + } + + pAttrInfo->bExist = AX_FALSE; + pAttrInfo->eType = ATTR_TYPE_BUTT; + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (strObjectCategory == "face") { + FaceAttrResult(pstObjectItems, pAttrInfo); + } + else if (strObjectCategory == "plate") { + PlateAttrResult(pstObjectItems, pAttrInfo); + } + else if (strObjectCategory == "vehicle") { + VehicleAttrResult(pstObjectItems, pAttrInfo); + } +} diff --git a/projects/llm_framework/main_skel/src/runner/attrParser.h b/projects/llm_framework/main_skel/src/runner/attrParser.h new file mode 100644 index 00000000..60646ef1 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/attrParser.h @@ -0,0 +1,60 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _ATTR_PARSER_H_ +#define _ATTR_PARSER_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum _ATTR_TYPE_E { + ATTR_TYPE_BODY = 0, + ATTR_TYPE_VEHICLE, + ATTR_TYPE_CYCLE, + ATTR_TYPE_FACE, + ATTR_TYPE_PLATE, + ATTR_TYPE_BUTT +} ATTR_TYPE_E; + +typedef struct _FACEINFO_T { + AX_U8 nGender; /* 0-female, 1-male */ + AX_U8 nAge; + AX_CHAR szMask[32]; + AX_CHAR szGender[32]; +} FACEINFO_T; + +typedef struct _PLATEINFO_T { + AX_BOOL bValid; + AX_CHAR szNum[16]; + AX_CHAR szColor[32]; +} PLATEINFO_T; + +typedef struct _ATTRINFO_T { + ATTR_TYPE_E eType; /* ATTR_TYPE_E */ + AX_BOOL bExist; + + union + { + FACEINFO_T tFaceInfo; + PLATEINFO_T tPlateInfo; + }; +} ATTRINFO_T; + +extern AX_VOID AttrParser(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c b/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c new file mode 100755 index 00000000..4fd522e2 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c @@ -0,0 +1,2233 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "common_vdec_utils.h" +#include "ax_vdec_api.h" +#include "common_arg_parse.h" +#include "common_vdec_api.h" + +#define AX_DEC_RET_STR_CASE(s32Ret) case (s32Ret): return(#s32Ret) + +const char *SampleVdecRetStr(AX_S32 s32Ret) +{ + switch (s32Ret) { + AX_DEC_RET_STR_CASE(AX_SUCCESS); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_INVALID_GRPID); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_INVALID_CHNID); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_ILLEGAL_PARAM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NULL_PTR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BAD_ADDR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_SYS_NOTREADY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUSY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_INIT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_CONFIG); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_SUPPORT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_PERM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_EXIST); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_UNEXIST); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOMEM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOBUF); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUF_EMPTY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUF_FULL); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_QUEUE_EMPTY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_QUEUE_FULL); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_TIMED_OUT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_FLOW_END); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_RUN_ERROR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_STRM_ERROR); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_UNEXIST); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_ILLEGAL_PARAM); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_NOT_SUPPORT); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_NOT_PERM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NEED_REALLOC_BUF); + default: + SAMPLE_CRIT_LOG("Unknown return code. 0x%x", s32Ret); + return("unknown code."); + } +} + +typedef struct +{ + const uint8_t * data; // sps + int count; /* in parse */ + int index; /* in parse */ +} PARSE_STATE_T; + + + +#define PARSE_INIT(data, bytes) {(data), 8*(bytes), 0} + +#define PARSE_EOF(parse) ((parse)->index >= (parse)->count) + +static inline void __parse_init(PARSE_STATE_T *parse, const uint8_t *data, int bytes) +{ + parse->data = data; + parse->count = 8 * bytes; + parse->index = 0; +} + +static inline int __parse_get_bit(PARSE_STATE_T *parse) +{ + if (parse->index >= parse->count) + return 1; /* -> no infinite colomb's ... */ + + int bit = (parse->data[parse->index >> 3] >> (7 - (parse->index & 7))) & 1; + parse->index++; + return bit; +} + +static inline uint32_t __parse_get_bits(PARSE_STATE_T * parse, uint32_t cnt) +{ + uint32_t bit = 0; + while (cnt--) + bit = bit | (__parse_get_bit(parse) << cnt); + + return bit; +} + +#define PARSE_SKIP_BIT(parse) __parse_skip_bits(parse, 1) + +static inline void __parse_skip_bits(PARSE_STATE_T *parse, int cnt) +{ + parse->index += cnt; +} + +#define PARSE_GET_U8(parse) __parse_get_bits(parse, 8) +#define PARSE_GET_U16(parse) ((__parse_get_bits(parse, 8)<<8) | __parse_get_bits(parse, 8)) + +static inline uint32_t __parse_get_ue_golomb(PARSE_STATE_T * parse) +{ + int cnt = 0; + while (!__parse_get_bit(parse) && cnt < 32) + cnt++; + return cnt ? ((1 << cnt) - 1) + __parse_get_bits(parse, cnt) : 0; +} + +static inline int32_t __parse_get_se_golomb(PARSE_STATE_T * parse) +{ + uint32_t r = __parse_get_ue_golomb(parse) + 1; + return (r & 1) ? -(r >> 1) : (r >> 1); +} + +static inline void __parse_skip_golomb(PARSE_STATE_T * parse) +{ + int cnt = 0; + while (!__parse_get_bit(parse) && cnt < 32) + cnt++; + __parse_skip_bits(parse, cnt); +} + +#define PARSE_SKIP_UE_GOLOMB(parse) __parse_skip_golomb(parse) +#define PARSE_SKIP_SE_GOLOMB(parse) __parse_skip_golomb(parse) + +static const MPEG_RATIONAL_T aspect_ratios[] = +{ +/* page 213: */ +/* 0: unknown */ + {0, 1}, +/* 1...16: */ + {1, 1}, {12, 11 }, {10, 11}, {16, 11}, + {40, 33}, {24, 11}, {20, 11}, {32, 11}, + {80, 33}, {18, 11}, {15, 11}, {64, 33}, + {160, 99}, {4, 3}, {3, 2}, {2, 1} +}; + +int h264_parse_sps(const uint8_t *buf, int len, SAMPLE_H264_SPS_DATA_T *sps) +{ + int findSPS = -1; + int pos = 0; + + if (buf == NULL) { + SAMPLE_CRIT_LOG("buf == NULL"); + return -1; + } + + if (sps == NULL) { + SAMPLE_CRIT_LOG("sps == NULL"); + return -1; + } + + if (len < 16) { + SAMPLE_CRIT_LOG("len:%d < 16", len); + return -1; + } + + for (pos = 0; pos < len - 6; pos++) { + if ((buf[pos] == 0) && (buf[pos + 1] == 0) + && (buf[pos + 2] == 0x1) && ((buf[pos + 3] & 0x1f) == 0x7)) { + len -= (pos + 4); + buf += (pos + 4); + findSPS = AX_SUCCESS; + break; + } + } + + if (findSPS != AX_SUCCESS) { + SAMPLE_LOG_TMP("findSPS failed! Failed to parse width and height.\n" + "You can configure the '--res' parameter in case the program runs out of memory and cannot decode"); + return findSPS; + } + + PARSE_STATE_T parse = PARSE_INIT(buf, len); + int profile_idc, pic_order_cnt_type; + int frame_mbs_only; + int i, j; + + profile_idc = PARSE_GET_U8(&parse); + sps->profile = profile_idc; + SAMPLE_LOG_N("H.264 SPS: profile_idc %d", profile_idc); + /* constraint_set0_flag = __parse_get_bit(parse); */ + /* constraint_set1_flag = __parse_get_bit(parse); */ + /* constraint_set2_flag = __parse_get_bit(parse); */ + /* constraint_set3_flag = __parse_get_bit(parse); */ + /* reserved = __parse_get_bits(parse,4); */ + sps->level = PARSE_GET_U8(&parse); + __parse_skip_bits(&parse, 8); + PARSE_SKIP_UE_GOLOMB(&parse); /* seq_parameter_set_id */ + if (profile_idc >= 100) { + if (__parse_get_ue_golomb(&parse) == 3) {/* chroma_format_idc */ + PARSE_SKIP_BIT(&parse); /* residual_colour_transform_flag */ + } + PARSE_SKIP_UE_GOLOMB(&parse); /* bit_depth_luma - 8 */ + PARSE_SKIP_UE_GOLOMB(&parse); /* bit_depth_chroma - 8 */ + PARSE_SKIP_BIT(&parse); /* transform_bypass */ + if (__parse_get_bit(&parse)) { /* seq_scaling_matrix_present */ + for (i = 0; i < 8; i++) { + if (__parse_get_bit(&parse)) { + /* seq_scaling_list_present */ + int last = 8, next = 8, size = (i < 6) ? 16 : 64; + for (j = 0; j < size; j++) { + if (next) { + next = (last + __parse_get_se_golomb(&parse)) & 0xff; + } + last = next ? next : last; + } + } + } + } + } + + PARSE_SKIP_UE_GOLOMB(&parse); /* log2_max_frame_num - 4 */ + pic_order_cnt_type = __parse_get_ue_golomb(&parse); + if (pic_order_cnt_type == 0) { + PARSE_SKIP_UE_GOLOMB(&parse); /* log2_max_poc_lsb - 4 */ + } + else if (pic_order_cnt_type == 1) { + PARSE_SKIP_BIT(&parse); /* delta_pic_order_always_zero */ + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_non_ref_pic */ + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_top_to_bottom_field */ + j = __parse_get_ue_golomb(&parse); /* num_ref_frames_in_pic_order_cnt_cycle */ + for (i = 0; i < j; i++) { + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_ref_frame[i] */ + } + + } + PARSE_SKIP_UE_GOLOMB(&parse); /* ref_frames */ + PARSE_SKIP_BIT(&parse); /* gaps_in_frame_num_allowed */ + sps->width /* mbs */ = __parse_get_ue_golomb(&parse) + 1; + sps->height /* mbs */ = __parse_get_ue_golomb(&parse) + 1; + frame_mbs_only = __parse_get_bit(&parse); + SAMPLE_LOG_N("H.264 SPS: pic_width: %u mbs", (unsigned)sps->width); + SAMPLE_LOG_N("H.264 SPS: pic_height: %u mbs", (unsigned)sps->height); + SAMPLE_LOG_N("H.264 SPS: frame only flag: %d", frame_mbs_only); + + sps->width *= 16; + sps->height *= 16 * (2 - frame_mbs_only); + + if (!frame_mbs_only) + if (__parse_get_bit(&parse)) /* mb_adaptive_frame_field_flag */ + SAMPLE_LOG_N("H.264 SPS: MBAFF"); + PARSE_SKIP_BIT(&parse); /* direct_8x8_inference_flag */ + if (__parse_get_bit(&parse)) { + /* frame_cropping_flag */ + uint32_t crop_left = __parse_get_ue_golomb(&parse); + uint32_t crop_right = __parse_get_ue_golomb(&parse); + uint32_t crop_top = __parse_get_ue_golomb(&parse); + uint32_t crop_bottom = __parse_get_ue_golomb(&parse); + SAMPLE_LOG_N("H.264 SPS: cropping %d %d %d %d", + crop_left, crop_top, crop_right, crop_bottom); + + sps->width -= 2 * (crop_left + crop_right); + if (frame_mbs_only) + sps->height -= 2 * (crop_top + crop_bottom); + else + sps->height -= 4 * (crop_top + crop_bottom); + } + + /* VUI parameters */ + sps->pixel_aspect.num = 0; + if (__parse_get_bit(&parse)) { + /* vui_parameters_present flag */ + if (__parse_get_bit(&parse)) { + /* aspect_ratio_info_present */ + uint32_t aspect_ratio_idc = PARSE_GET_U8(&parse); + SAMPLE_LOG_N("H.264 SPS: aspect_ratio_idc %d", aspect_ratio_idc); + + if (aspect_ratio_idc == 255 /* Extended_SAR */) { + sps->pixel_aspect.num = PARSE_GET_U16(&parse); /* sar_width */ + sps->pixel_aspect.den = PARSE_GET_U16(&parse); /* sar_height */ + SAMPLE_LOG_N("H.264 SPS: -> sar %dx%d", sps->pixel_aspect.num, sps->pixel_aspect.den); + } else { + if (aspect_ratio_idc < sizeof(aspect_ratios) / sizeof(aspect_ratios[0])) { + sps->pixel_aspect = aspect_ratios[aspect_ratio_idc]; + SAMPLE_LOG_N("H.264 SPS: -> aspect ratio %d / %d", sps->pixel_aspect.num, sps->pixel_aspect.den); + } else { + SAMPLE_LOG_N("H.264 SPS: aspect_ratio_idc out of range !"); + } + } + } + } + + SAMPLE_LOG_N("H.264 SPS: -> video size %dx%d, aspect %d:%d", + sps->width, sps->height, sps->pixel_aspect.num, sps->pixel_aspect.den); + + return findSPS; +} + + + +AX_PAYLOAD_TYPE_E DistinguishVideoType(const AX_U8 *stream, AX_U64 len) +{ + AX_U64 ReadLen = 0; + AX_U64 i = 0; + const AX_U8 *pu8Buf = stream; + + if (stream == NULL) { + SAMPLE_CRIT_LOG(" stream == NULL"); + return PT_BUTT; + } + + ReadLen = len; + + if (ReadLen <= 6) { + SAMPLE_CRIT_LOG(" len:%lld ReadLen:%lld", len, ReadLen); + return PT_BUTT; + } + + for (i = 0; i < ReadLen - 6; i++) + { + if ((pu8Buf[i+0] == 0) && (pu8Buf[i+1] == 0) && (pu8Buf[i+2] == 1)) { + if ((pu8Buf[i+3] & 0x1f) == 0x07) { + return PT_H264; + } + } + + if ((pu8Buf[i+0] == 0) && (pu8Buf[i+1] == 0) && (pu8Buf[i+2] == 0) && (pu8Buf[i+3] == 1)) { + if ((pu8Buf[i+4] == 0x40) && (pu8Buf[i+5] == 0x01)) { + return PT_H265; + } + } + } + + return PT_BUTT; +} + +static AX_S32 __MD5SumValidOnly(AX_U8 *p_lu, AX_U8 *p_ch, + AX_U32 coded_width, AX_U32 coded_height, + AX_U32 coded_width_ch, AX_U32 coded_height_ch, + AX_U32 pic_stride, AX_U32 pic_stride_ch, + AX_U32 planar, AX_U32 frame_number, char *md5_str) +{ +#if 0 + + unsigned char digest[16]; + MD5_CTX ctx; + int i = 0; + MD5_Init(&ctx); + AX_U8 *p_yuv = p_lu; + if (p_yuv) { + for (i = 0; i < coded_height; i++) { + MD5_Update(&ctx, p_yuv, coded_width); + p_yuv += pic_stride; + } + } + p_yuv = p_ch; + if (p_yuv) { + if (!planar) { + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch); + p_yuv += pic_stride; + } + } else { + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch / 2); + p_yuv += pic_stride_ch; + } + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch / 2); + p_yuv += pic_stride_ch; + } + } + } + MD5_Final(digest, &ctx); + /* fprintf(f_out, "FRAME %d: ", frame_number);*/ + for (i = 0; i < 16; i++) { + snprintf(md5_str + i * 2, 2 + 1, "%02x", digest[i]); + } +#endif + return 0; +} + +int OutputFileCheckMD5(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, char *md5_str) +{ + AX_S32 sRet; + AX_S32 s32Ret = 0; + AX_VOID *pLumaVirAddr; + AX_VOID *pChromaVirAddr; + AX_U32 lumaMapSize; + AX_U32 chromaMapSize; + + if (frameInfo == NULL) { + s32Ret = AX_ERR_VDEC_NULL_PTR; + SAMPLE_CRIT_LOG("frameInfo == NULL\n"); + goto ERR_RET; + } + + if (md5_str == NULL) { + s32Ret = AX_ERR_VDEC_NULL_PTR; + SAMPLE_CRIT_LOG("md5_str == NULL\n"); + goto ERR_RET; + } + + if ((frameInfo->stVFrame.u64PhyAddr[0] == 0) || (frameInfo->stVFrame.u32PicStride[0] == 0)) { + SAMPLE_CRIT_LOG("VdGrp=%d, stVFrame.u64PhyAddr[0]:0x%llx stVFrame.u32PicStride[0]:%d\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0], frameInfo->stVFrame.u32PicStride[0]); + s32Ret = 0; + goto ERR_RET; + } + + lumaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16); + + pLumaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[0], lumaMapSize); + if (!pLumaVirAddr) { + s32Ret = AX_ERR_VDEC_BAD_ADDR; + SAMPLE_CRIT_LOG("AX_SYS_Mmap luma FAILED\n"); + goto ERR_RET; + } + + SAMPLE_LOG("AX_SYS_Mmap luma success, pLumaVirAddr=%p,lumaMapSize=%d\n", + pLumaVirAddr, lumaMapSize); + + chromaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16) / 2; + pChromaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[1], chromaMapSize); + if (!pChromaVirAddr) { + s32Ret = AX_ERR_VDEC_BAD_ADDR; + SAMPLE_CRIT_LOG("AX_SYS_Mmap chroma FAILED\n"); + goto ERR_RET_CHROMA; + } + + SAMPLE_LOG("AX_SYS_Mmap chroma success, pChromaVirAddr=%p,chromaMapSize=%d\n", + pChromaVirAddr, chromaMapSize); + + AX_VOID *p_lu = pLumaVirAddr; + AX_VOID *p_ch = pChromaVirAddr; + AX_U32 coded_width = frameInfo->stVFrame.u32Width; + AX_U32 coded_height = frameInfo->stVFrame.u32Height; + AX_U32 pic_stride = frameInfo->stVFrame.u32PicStride[0]; + AX_U32 coded_width_ch = frameInfo->stVFrame.u32Width; + AX_U32 coded_h_ch = frameInfo->stVFrame.u32Height / 2; + AX_U32 pic_stride_ch = frameInfo->stVFrame.u32PicStride[1]; + s32Ret = __MD5SumValidOnly(p_lu, p_ch, coded_width, coded_height, coded_width_ch, coded_h_ch, + pic_stride, pic_stride_ch, 0, 0, md5_str); + if (s32Ret) { + SAMPLE_CRIT_LOG("__MD5SumValidOnly FAILED! s32Ret:0x%x", s32Ret); + goto ERR_RET_LUMA; + } + +ERR_RET_LUMA: + if (pLumaVirAddr) { + sRet = AX_SYS_Munmap(pLumaVirAddr, lumaMapSize); + if (sRet) { + s32Ret = sRet; + SAMPLE_CRIT_LOG("AX_SYS_Munmap luma FAILED,sRet=0x%x\n", sRet); + } else { + SAMPLE_LOG("AX_SYS_Munmap luma success,pLumaVirAddr=%p,lumaMapSize=%d\n", + pLumaVirAddr, lumaMapSize); + } + } +ERR_RET_CHROMA: + if (pChromaVirAddr) { + sRet = AX_SYS_Munmap(pChromaVirAddr, chromaMapSize); + if (sRet) { + s32Ret = sRet; + SAMPLE_CRIT_LOG("AX_SYS_Munmap chroma FAILED,sRet=0x%x\n", sRet); + } else { + SAMPLE_LOG("AX_SYS_Munmap chroma success,pChromaVirAddr=%p,chromaMapSize=%d\n", + pChromaVirAddr, chromaMapSize); + } + } +ERR_RET: + return s32Ret; +} + + +AX_S32 LoadOneFileToMem(const AX_CHAR *ps8File, AX_U8 **ppu8Mem, size_t *pLen) +{ + AX_S32 res = 0; + AX_U64 tmp_size = 0; + AX_U64 read_size = 0; + FILE *fInput = NULL; + + if (ps8File == NULL) { + SAMPLE_CRIT_LOG("ps8File == NULL\n"); + goto ERR_RET; + } + + if (ppu8Mem == NULL) { + SAMPLE_CRIT_LOG("ppu8Mem == NULL\n"); + goto ERR_RET; + } + + if (pLen == NULL) { + SAMPLE_CRIT_LOG("pLen == NULL\n"); + goto ERR_RET; + } + + fInput = fopen(ps8File, "rb"); + if (fInput == NULL) { + SAMPLE_CRIT_LOG("Unable to open input file:%s\n", ps8File); + res = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + /* file i/o pointer to full */ + res = fseek(fInput, 0L, SEEK_END); + if (res) { + SAMPLE_CRIT_LOG("fseek FAILED! ret:%d\n", res); + goto ERR_RET; + } + + *pLen = ftello(fInput); + rewind(fInput); + + tmp_size = sizeof(AX_U8) * (*pLen); + *ppu8Mem = (AX_U8 *)malloc(tmp_size); + if (*ppu8Mem == NULL) { + SAMPLE_CRIT_LOG("malloc tmp_size:0x%llx FAILED!", tmp_size); + res = AX_ERR_VDEC_NOMEM; + goto ERR_RET_CLOSE; + } + + read_size = *pLen; + tmp_size = fread(*ppu8Mem, sizeof(AX_U8), read_size, fInput); + /* read input stream from file to buffer and close input file */ + if (tmp_size != read_size) { + SAMPLE_CRIT_LOG("fread FAILED! tmp_size:0x%llx != read_size:0x%llx\n", tmp_size, read_size); + res = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET_FREE; + } + +ERR_RET_FREE: + free((AX_VOID *)*ppu8Mem); + *ppu8Mem = NULL; +ERR_RET_CLOSE: + res = fclose(fInput); + if (res) { + SAMPLE_CRIT_LOG("fclose FAILED! ret:%d\n", res); + } +ERR_RET: + return res; +} + +FILE *OutputFileOpen(AX_CHAR **ppOutputFile, const SAMPLE_VDEC_OUTPUT_INFO_T *pInfo) +{ + FILE *fp_out = NULL; + int ret = 0; + AX_CHAR *file_path = NULL; + AX_CHAR *file_name = NULL; + AX_U32 slen = 0; + AX_CHAR *pFileOut = NULL; + + if (ppOutputFile == NULL) { + SAMPLE_CRIT_LOG("ppOutputFile == NULL"); + return NULL; + } + + if (*ppOutputFile == NULL) { + pFileOut = calloc(1, AX_VDEC_FILE_PATH_LEN); + if (pFileOut == NULL) { + SAMPLE_CRIT_LOG("malloc %d Bytes FAILED!\n", AX_VDEC_FILE_PATH_LEN); + return NULL; + } + *ppOutputFile = pFileOut; + + if (pInfo == NULL) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "%s", "./out.yuv"); + } else { + if (pInfo->bOneShot) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "oneShot_format%d_w_%d_h_%d.yuv", + pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } else { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "group%d_format%d_w_%d_h_%d.yuv", + pInfo->VdGrp, pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } + } + + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x pFileOut:%s AX_VDEC_FILE_NAME_LEN:%d\n", + ret, pFileOut, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_OUTPUTFILE; + } + + *ppOutputFile = pFileOut; + } else { + pFileOut = *ppOutputFile; + } + + SAMPLE_LOG("output_file_path >>>> pFileOut:%s", pFileOut); + slen = strlen(pFileOut); + ret = strncmp (pFileOut + slen - 1, "/", 1); + if (ret == 0) { + file_path = malloc(AX_VDEC_FILE_PATH_LEN); + if (NULL == file_path) { + SAMPLE_CRIT_LOG("malloc size:%d FAILED!", AX_VDEC_FILE_PATH_LEN); + goto ERR_RET_OUTPUTFILE; + } + + file_name = malloc(AX_VDEC_FILE_NAME_LEN); + if (NULL == file_name) { + SAMPLE_CRIT_LOG("malloc size:%d FAILED!", AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILEPATH; + } + + if (pInfo == NULL) { + ret = snprintf(file_name, AX_VDEC_FILE_NAME_LEN, "%s", "out.yuv"); + } else { + if (pInfo->bOneShot) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "oneShot_format%d_w_%d_h_%d.yuv", + pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } else { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "group%d_format%d_w_%d_h_%d.yuv", + pInfo->VdGrp, pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } + } + + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x file_name:%s AX_VDEC_FILE_NAME_LEN:%d\n", + ret, file_name, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + + slen = strlen(file_name); + if (slen >= AX_VDEC_FILE_NAME_LEN) { + SAMPLE_CRIT_LOG("slen:%d >= AX_VDEC_FILE_NAME_LEN:%d\n", + slen, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + + ret = snprintf(file_path, AX_VDEC_FILE_PATH_LEN, "%s%s", + pFileOut, file_name); + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x, pFileOut:%s, file_name:%s, AX_VDEC_FILE_PATH_LEN:%d\n", + ret, pFileOut, file_name, AX_VDEC_FILE_PATH_LEN); + goto ERR_RET_FILENAME; + } + + free(file_name); + free(pFileOut); + *ppOutputFile = file_path; + pFileOut = *ppOutputFile; + } + + slen = strlen(pFileOut); + ret = strncmp (pFileOut + slen - 1, "/", 1); + if (ret == 0) { + SAMPLE_CRIT_LOG("Please check pFileOut:%s, the last character cannot be '/' \n", pFileOut); + goto ERR_RET_FILENAME; + } + + if (access(pFileOut, F_OK) == 0) { + char new_name[AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN]; + memset(new_name, 0, sizeof(new_name)); + + struct timeval current_tv; + gettimeofday(¤t_tv, NULL); + + ret = sprintf(new_name, "%s_%lds_%ldus.bak", pFileOut, current_tv.tv_sec, current_tv.tv_usec); + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x, pFileOut:%s, AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN:%d\n", + ret, pFileOut, AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + // SAMPLE_LOG("new_name:%s ", new_name); + if (chmod(pFileOut, 0x777) != 0) { + SAMPLE_CRIT_LOG("chmod:%s FAILED", pFileOut); + } + + if (rename(pFileOut, new_name) == 0) { + SAMPLE_LOG("rename(pFileOut:%s, new_name:%s) success", + pFileOut, new_name); + } else { + SAMPLE_CRIT_LOG("rename(pFileOut:%s, new_name:%s) FAILED)", pFileOut, new_name); + } + } + + if (access(pFileOut, F_OK) == 0) { + if (chmod(pFileOut, 0x777) != 0) { + SAMPLE_CRIT_LOG("chmod:%s ", pFileOut); + } + + if (remove(pFileOut) == 0) { + if (access(pFileOut, F_OK) == 0) { + SAMPLE_CRIT_LOG("remove(pFileOut:%s FAIL)", pFileOut); + } + } else { + SAMPLE_CRIT_LOG("remove(pFileOut:%s FAIL)", pFileOut); + } + } + + fp_out = fopen(pFileOut, "w"); + if (fp_out == NULL) { + SAMPLE_CRIT_LOG("VdGrp=%d, fopen pFileOut:%s FAILED!", + pInfo->VdGrp, pFileOut); + } + + SAMPLE_LOG("output file name:%s, open fp_out:%p\n", pFileOut, fp_out); + return fp_out; + +ERR_RET_FILENAME: + if (file_name != NULL) { + free(file_name); + } +ERR_RET_FILEPATH: + if (file_path != NULL) { + free(file_path); + } +ERR_RET_OUTPUTFILE: + if (*ppOutputFile != NULL) { + free(*ppOutputFile); + *ppOutputFile = NULL; + } + + return NULL; +} + +static AX_S32 OutputSaveYUVFile(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, + FILE *fp_out, AX_CHAR *pOutputFilePath) +{ + int ret = 0; + AX_S32 sRet = 0; + AX_U32 i; + AX_VOID *p_lu = NULL; + AX_VOID *p_ch = NULL; + + AX_S32 s32Ret = 0; + AX_VOID *pLumaVirAddr = NULL; + AX_VOID *pChromaVirAddr = NULL; + AX_U32 lumaMapSize = 0; + AX_U32 chromaMapSize = 0; + int tmp_size = 0; + + if (NULL == frameInfo || NULL == fp_out) { + SAMPLE_CRIT_LOG("VdGrp=%d, NULL == frameInfo || NULL == fp_out\n", VdGrp); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if ((frameInfo->stVFrame.u64PhyAddr[0] == 0) || (frameInfo->stVFrame.u32PicStride[0] == 0)) { + SAMPLE_LOG("VdGrp=%d, stVFrame.u64PhyAddr[0]:0x%llx stVFrame.u32PicStride[0]:%d\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0], frameInfo->stVFrame.u32PicStride[0]); + s32Ret = 0; + goto ERR_RET; + } + + lumaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16); + pLumaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[0], lumaMapSize); + if (NULL == pLumaVirAddr) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Mmap luma FAILED! .u64PhyAddr[0]:0x%llx\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0]); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + SAMPLE_LOG_N("AX_SYS_Mmap luma success, .u64PhyAddr[0]:0x%llx, pLumaVirAddr=%p, lumaMapSize=%d\n", + frameInfo->stVFrame.u64PhyAddr[0], pLumaVirAddr, lumaMapSize); + + p_lu = pLumaVirAddr; + + SAMPLE_LOG_N("p_lu: %p\n", p_lu); + SAMPLE_LOG_N("lu_buss: 0x%llx\n", frameInfo->stVFrame.u64PhyAddr[0]); + SAMPLE_LOG_N("ch_buss: 0x%llx\n", frameInfo->stVFrame.u64PhyAddr[1]); + + AX_U32 coded_width = frameInfo->stVFrame.u32Width; + AX_U32 coded_height = frameInfo->stVFrame.u32Height; + AX_U32 pic_stride = frameInfo->stVFrame.u32PicStride[0]; + AX_U32 coded_width_ch = frameInfo->stVFrame.u32Width; + AX_U32 coded_h_ch = frameInfo->stVFrame.u32Height / 2; + AX_U32 pic_stride_ch = frameInfo->stVFrame.u32PicStride[1]; + AX_U32 pic_format = frameInfo->stVFrame.enImgFormat; + + if (AX_FORMAT_YUV400 != pic_format) { + chromaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16) / 2; + pChromaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[1], chromaMapSize); + if (NULL == pChromaVirAddr) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Mmap luma FAILED! .u64PhyAddr[1]:0x%llx\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[1]); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET_MUNMAP_LUMA; + } + + p_ch = pChromaVirAddr; + + SAMPLE_LOG_N("AX_SYS_Mmap chroma success, .u64PhyAddr[1]:0x%llx, pChromaVirAddr=%p, chromaMapSize=%d\n", + frameInfo->stVFrame.u64PhyAddr[1], pChromaVirAddr, chromaMapSize); + SAMPLE_LOG_N("p_ch: %p\n", p_ch); + } + + SAMPLE_LOG_N("p_lu: %p, p_ch: %p, \n", p_lu, p_ch, ); + SAMPLE_LOG_N("coded_width: %u, coded_height: %u, pic_stride: %u, \n" + "coded_width_ch: %u, coded_h_ch: %u, pic_stride_ch: %u, pixel_bytes: %u, pic_format:%d\n", + coded_width, coded_height, pic_stride, + coded_width_ch, coded_h_ch, pic_stride_ch, 1, pic_format); + + SAMPLE_LOG_N("write Y\n"); + if (AX_FORMAT_YUV420_SEMIPLANAR_10BIT_P010 == pic_format) { + coded_width = coded_width * 2; + coded_width_ch = coded_width_ch * 2; + }else if (AX_FORMAT_YUV420_SEMIPLANAR_10BIT_P101010 == pic_format) { + coded_width = coded_width * 10 / 8; + coded_width_ch = coded_width_ch * 10 / 8; + } + + for (i = 0; i < coded_height; i++) { + tmp_size = fwrite(p_lu, 1, coded_width, fp_out); + if (tmp_size != coded_width) { + SAMPLE_CRIT_LOG("VdGrp=%d, fwrite FAILED! tmp_size:0x%x != coded_width:0x%x", + VdGrp, tmp_size, coded_width); + s32Ret = AX_ERR_VDEC_NOMEM; + } + p_lu += pic_stride; + } + + if (AX_FORMAT_YUV400 != pic_format) { + SAMPLE_LOG_N("write UV\n"); + for (i = 0; i < coded_h_ch; i++) { + tmp_size = fwrite(p_ch, 1, coded_width_ch, fp_out); + if (tmp_size != coded_width_ch) { + SAMPLE_CRIT_LOG("VdGrp=%d, fwrite FAILED! tmp_size:0x%x != coded_width_ch:0x%x", + VdGrp, tmp_size, coded_width_ch); + s32Ret = AX_ERR_VDEC_NOMEM; + } + p_ch += pic_stride_ch; + } + } + + ret = fflush(fp_out); + if (ret) { + SAMPLE_CRIT_LOG("VdGrp=%d, fflush FAILED! fp_out:%p", + VdGrp, fp_out); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + } + + SAMPLE_LOG_N("VdGrp=%d, write YUV done! %s\n", VdGrp, pOutputFilePath); + + if (pChromaVirAddr) { + sRet = AX_SYS_Munmap(pChromaVirAddr, chromaMapSize); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Munmap chroma FAILED, sRet=0x%x\n", + VdGrp, sRet); + } + } + +ERR_RET_MUNMAP_LUMA: + if (pLumaVirAddr) { + sRet = AX_SYS_Munmap(pLumaVirAddr, lumaMapSize); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Munmap luma FAILED, sRet=0x%x\n", + VdGrp, sRet); + } + } +ERR_RET: + return s32Ret || sRet; +} + +AX_S32 OutputFileSaveYUV(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, FILE *fp_out, AX_CHAR *pOutputFilePath) +{ + AX_S32 s32Ret = 0; + + if (NULL == frameInfo) { + SAMPLE_CRIT_LOG("VdGrp=%d, NULL == frameInfo\n", VdGrp); + return -1; + } + + if (AX_COMPRESS_MODE_NONE != frameInfo->stVFrame.stCompressInfo.enCompressMode) { + SAMPLE_CRIT_LOG("VdGrp=%d, invalid enCompressMode: %d\n", VdGrp, + frameInfo->stVFrame.stCompressInfo.enCompressMode); + return -1; + } + + s32Ret = OutputSaveYUVFile(VdGrp, frameInfo, fp_out, pOutputFilePath); + + return s32Ret; +} + +static off_t __FindFileNextStartCode(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, AX_U32 *puZeroCount, AX_U32 *pNalType) +{ + AX_S32 i; + off_t sStart; + off_t oFileStart; + off_t oFileOffset = 0; + AX_U32 uLeftFileSize, uFileReadLen = 0; + AX_CHAR tmp_buf[VDEC_BS_PARSER_BUF_SIZE] = {0}; + AX_U8 byte; + int ret_val; + int ret; + + oFileStart = ftello(pstBsInfo->fInput); + sStart = oFileStart; + *puZeroCount = 0; + + while (1) { + assert(sStart <= pstBsInfo->sFileSize); + + uLeftFileSize = pstBsInfo->sFileSize - sStart; + if (uLeftFileSize == 0) { + oFileOffset = pstBsInfo->sFileSize - 1; + break; + } + + uFileReadLen = VDEC_BS_PARSER_BUF_SIZE < uLeftFileSize ? VDEC_BS_PARSER_BUF_SIZE : uLeftFileSize; + fread(tmp_buf, 1, uFileReadLen, pstBsInfo->fInput); + /* Scan for the beginning of the packet. */ + for (i = 0; i < uFileReadLen; i++) { + ret_val = tmp_buf[i]; + oFileOffset = sStart + i; + if (ret_val == EOF) { + sStart = oFileOffset - 1; + SAMPLE_LOG_N("sStart:%ld, i:%d, oFileOffset:%d, *puZeroCount:%d", + sStart, i, oFileOffset, *puZeroCount); + + ret = fseeko(pstBsInfo->fInput, 0, SEEK_END); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + + return sStart; + } + + byte = (unsigned char)ret_val; + switch (byte) { + case 0: + *puZeroCount = *puZeroCount + 1; + break; + case 1: + /* If there's more than three leading zeros, consider only three + * of them to be part of this packet and the rest to be part of + * the previous packet. */ + if (*puZeroCount > 3) { + *puZeroCount = 3; + } + + if (*puZeroCount >= 2) { + SAMPLE_LOG_N("sStart:%ld, i:%d, oFileOffset:%d, *puZeroCount:%d", + sStart, i, oFileOffset, *puZeroCount); + if (i < uFileReadLen - 1) { + ret_val = (tmp_buf[i + 1] & 0x1f); + if (ret_val == 0x1 || ret_val == 0x7 || ret_val == 0x5) { + sStart = oFileOffset - *puZeroCount; + *pNalType = ret_val; + goto FUNC_RET; + } + } + } + *puZeroCount = 0; + break; + default: + *puZeroCount = 0; + break; + } + } + + ret = fseeko(pstBsInfo->fInput, oFileOffset, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + sStart = oFileOffset + 1; + } + +FUNC_RET: + + ret = fseeko(pstBsInfo->fInput, oFileOffset + 1, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + SAMPLE_LOG_N("func end, ftello(pstBsInfo->fInput):%d uFileReadLen:%d", + ftello(pstBsInfo->fInput), uFileReadLen); + return sStart; +} + +static int __CheckFileAccessUnitBoundary(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, off_t oNalBegin, SAMPLE_BSBOUNDARY_TYPE_E *penBoundary) +{ + int ret = 0; + int iNalType, iVal; + SAMPLE_BSBOUNDARY_TYPE_E enBoundary = BSPARSER_NO_BOUNDARY; + + FILE *fInput = pstBsInfo->fInput; + off_t sStart = ftello(fInput); + + off_t tmp_offset = oNalBegin + 1; + ret = fseeko(fInput, tmp_offset, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + return ret; + } + + AX_PAYLOAD_TYPE_E enDecType = pstBsInfo->enDecType; + + if (enDecType == PT_H264) { + iNalType = (getc(fInput) & 0x1F); + + if (iNalType > NAL_CODED_SLICE_IDR) { + enBoundary = BSPARSER_BOUNDARY_NON_SLICE_NAL; + } else { + iVal = getc(fInput); + /* Check if first mb in slice is 0(ue(v)). */ + if (iVal & 0x80) { + enBoundary = BSPARSER_BOUNDARY; + } + } + } else if (enDecType == PT_H265) { + iNalType = (getc(fInput) & 0x7E) >> 1; + + if (iNalType > NAL_CODED_SLICE_CRA) { + enBoundary = BSPARSER_BOUNDARY_NON_SLICE_NAL; + } else { + iVal = getc(fInput); // nothing interesting here... + iVal = getc(fInput); + /* Check if first slice segment in picture */ + if (iVal & 0x80) { + enBoundary = BSPARSER_BOUNDARY; + } + } + } + + ret = fseeko(fInput, sStart, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + return ret; + } + + *penBoundary = enBoundary; + return 0; +} + + +static AX_S32 __StreamReadFrameInRingBuf(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, + size_t oStreamLen, size_t *pReadLen) +{ + off_t oOffset = 0; + AX_U8 *pBufRd = NULL; + AX_U8 *pBufStart = NULL; + AX_U32 uBufSize = 0; + AX_S32 sRet = 0; + AX_U32 sReadLen = 0; + AX_U32 tmp_len = 0, right_len = 0, left_len = 0; + + if (pstBsInfo == NULL) { + SAMPLE_CRIT_LOG("pstBsInfo == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pBufRd = pstStreamBuf->pBufAfterFill; + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + uBufSize = pstStreamBuf->uBufSize; + if (pBufStart == NULL) { + SAMPLE_CRIT_LOG("pBufStart == NULL\n"); + sRet = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (uBufSize < oStreamLen) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < oStreamLen:0x%x", uBufSize, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_NOMEM; + goto ERR_RET; + } + + *pReadLen = 0; + oOffset = (off_t)(pBufRd - pBufStart); + pstStreamBuf->pBufBeforeFill = pstStreamBuf->pBufAfterFill; + if ((oOffset + oStreamLen) < uBufSize) { + sReadLen = fread(pBufRd, 1, oStreamLen, pstBsInfo->fInput); + if (sReadLen != oStreamLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != oStreamLen:0x%x", sReadLen, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + pstStreamBuf->pBufAfterFill = pBufRd + sReadLen; + } else { + /* turnaround */ + right_len = uBufSize - oOffset; + sReadLen = fread(pBufRd, 1, right_len, pstBsInfo->fInput); + if (sReadLen != uBufSize - oOffset) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != (uBufSize:0x%x - oOffset:0x%llx):0x%x", + sReadLen, uBufSize, (AX_U64)oOffset, right_len); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + left_len = oStreamLen - (uBufSize - oOffset); + tmp_len = fread(pBufStart, 1, left_len, pstBsInfo->fInput); + if (tmp_len != left_len) { + SAMPLE_CRIT_LOG("fread FAILED! tmp_len:0x%x != left_len:0x%x, oStreamLen:0x%x uBufSize:0x%x oOffset:0x%llx", + tmp_len, left_len, (AX_U32)oStreamLen, uBufSize, (AX_U64)oOffset); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + sReadLen += tmp_len; + pstStreamBuf->pBufAfterFill = pBufStart + tmp_len; + } + + *pReadLen = sReadLen; +ERR_RET: + return sRet; +} + +AX_S32 StreamFileParserReadFrame(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + int ret = 0; + AX_S32 sRet = 0; + AX_U32 uZeroCount = 0; + SAMPLE_BSBOUNDARY_TYPE_E enBoundary = BSPARSER_NO_BOUNDARY; + off_t oNalBegin, oStreamLen; + off_t oBegin, oEnd, oTmpEnd; + AX_U32 sReadLen = 0; + AX_U32 uBufSize; + AX_U8 *pBufStart = NULL; + AX_U32 u32NalType = 0; + + if (pstBsInfo == NULL) { + SAMPLE_CRIT_LOG("pstBsInfo == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf->tBufAddr.pVirAddr == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf->tBufAddr.pVirAddr == NULL\n"); + sRet = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (pReadLen == NULL) { + SAMPLE_CRIT_LOG("pReadLen == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + oBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + /* Check for non-slice type in current NAL. non slice NALs are + * decoded one-by-one */ + oNalBegin = oBegin + uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (u32NalType == 0x7) { /* get i frm video nal */ + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + /* Check for non-slice type in current NAL. non slice NALs are + * decoded one-by-one */ + oNalBegin = oNalBegin + uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + } + + oEnd = oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + if (oEnd == oBegin) { + *pReadLen = 0; + return 0; /* End of stream */ + } + + /* if there is more stream and a slice type NAL */ + if (enBoundary != BSPARSER_BOUNDARY_NON_SLICE_NAL) { + while (1) { + oEnd = oNalBegin; + oNalBegin += uZeroCount; + + /* Check access unit boundary for next NAL */ + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (enBoundary == BSPARSER_NO_BOUNDARY) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + } + else if (enBoundary == BSPARSER_BOUNDARY_NON_SLICE_NAL) { + while (1) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + oTmpEnd = oNalBegin; + oNalBegin += uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (enBoundary != BSPARSER_BOUNDARY_NON_SLICE_NAL) { + break; + } + + if (oTmpEnd == oNalBegin) { + break; + } + } + + if (oTmpEnd == oNalBegin) { + break; + } + + if (enBoundary == BSPARSER_NO_BOUNDARY) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + } + } + + if (enBoundary == BSPARSER_BOUNDARY) { + break; + } + + if (oEnd == oNalBegin) { + break; + } + } + } + + ret = fseeko(pstBsInfo->fInput, oBegin, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + oStreamLen = oEnd - oBegin; + uBufSize = pstStreamBuf->uBufSize; + if (uBufSize < oStreamLen) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < oStreamLen:0x%x. bufSize is not enough, please increase STREAM_BUFFER_MAX_SIZE", + uBufSize, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_NOMEM; + goto ERR_RET; + } + + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + + sReadLen = fread(pBufStart, 1, oStreamLen, pstBsInfo->fInput); + if (sReadLen != oStreamLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != oStreamLen:0x%x", sReadLen, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + pstStreamBuf->pBufBeforeFill = pBufStart; + pstStreamBuf->pBufAfterFill = pBufStart + sReadLen; + + SAMPLE_LOG("sReadLen:0x%lx, uBufSize:0x%x", sReadLen, uBufSize); + *pReadLen = sReadLen; + return 0; +ERR_RET: + return sRet; +} + +static AX_U8 imgGetBytes(FILE *fInput, size_t pos) +{ + AX_U8 data = 0; + AX_U8 readLen = 0; + + fseeko(fInput, pos, SEEK_SET); + readLen = fread(&data, 1, 1, fInput); + if(1 != readLen) + SAMPLE_CRIT_LOG(" read file failed\n"); + + return data; +} + +AX_S32 StreamParserReadFrameJpeg(SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + size_t i,j; + AX_U32 jpeg_thumb_in_stream = 0; + AX_U64 tmp, tmp1, tmp_total = 0; + size_t curPos = 0; + AX_U32 imgLen = 0; + AX_S32 s32Ret = 0; + size_t stream_length = 0; + AX_U8 *pBufStart = NULL; + size_t sReadLen = 0; + + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + if (pBufStart == NULL) { + SAMPLE_CRIT_LOG("pBufStart == NULL\n"); + return -1; + } + + stream_length = pstBsInfo->sFileSize; + for (i = pstBsInfo->curPos; i < stream_length; ++i) { + if (0xFF == imgGetBytes(pstBsInfo->fInput, i)) { + /* if 0xFFE1 to 0xFFFD ==> skip */ + if ((((i + 1) < stream_length) && + 0xE1 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE2 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE3 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE4 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE5 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE6 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE7 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE8 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE9 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEA == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEB == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEC == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xED == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEE == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEF == imgGetBytes(pstBsInfo->fInput, i + 1))) { + /* increase counter */ + i += 2; + + /* check length vs. data */ + if ((i + 1) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + + /* get length */ + tmp = imgGetBytes(pstBsInfo->fInput, i); + tmp1 = imgGetBytes(pstBsInfo->fInput, i + 1); + tmp_total = (tmp << 8) | tmp1; + + /* check length vs. data */ + if ((tmp_total + i) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + /* update */ + i += tmp_total-1; + continue; + } + + /* if 0xFFC2 to 0xFFCB ==> skip */ + if ((((i + 1) < stream_length) && + 0xC1 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC2 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC3 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC5 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC6 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC7 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC8 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC9 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xCA == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xCB == imgGetBytes(pstBsInfo->fInput, i + 1)) ) { + /* increase counter */ + i += 2; + + /* check length vs. data */ + if ((i + 1) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + + /* get length */ + tmp = imgGetBytes(pstBsInfo->fInput, i); + tmp1 = imgGetBytes(pstBsInfo->fInput, i + 1); + tmp_total = (tmp << 8) | tmp1; + + /* check length vs. data */ + if ((tmp_total + i) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + /* update */ + i += tmp_total-1; + + /* look for EOI */ + for(j = i; j < stream_length; ++j) { + if (0xFF == imgGetBytes(pstBsInfo->fInput, j)) { + /* EOI */ + if (((j + 1) < stream_length) && + 0xD9 == imgGetBytes(pstBsInfo->fInput, j + 1)) { + /* check length vs. data */ + if ((j + 2) >= (stream_length)) { + curPos = j + 2; + s32Ret = 0; + goto ret; + } + /* update */ + i = j; + /* stil data left ==> continue */ + continue; + } + } + } + } + + /* check if thumbnails in stream */ + if (((i + 1) < stream_length) && + 0xE0 == imgGetBytes(pstBsInfo->fInput, i + 1)) { + if (((i + 9) < stream_length) && + 0x4A == imgGetBytes(pstBsInfo->fInput, i + 4) && + 0x46 == imgGetBytes(pstBsInfo->fInput, i + 5) && + 0x58 == imgGetBytes(pstBsInfo->fInput, i + 6) && + 0x58 == imgGetBytes(pstBsInfo->fInput, i + 7) && + 0x00 == imgGetBytes(pstBsInfo->fInput, i + 8) && + 0x10 == imgGetBytes(pstBsInfo->fInput, i + 9)) { + jpeg_thumb_in_stream = 1; + } + } + + /* EOI */ + if (((i + 1) < stream_length) && + 0xD9 == imgGetBytes(pstBsInfo->fInput, i + 1)) { + curPos = i + 2; + /* update amount of thumbnail or full resolution image */ + if (jpeg_thumb_in_stream) { + jpeg_thumb_in_stream = 0; + } else { + s32Ret = 0; + goto ret; + } + } + } + } + +ret: + imgLen = curPos > pstBsInfo->curPos ? curPos - pstBsInfo->curPos : 0; + if(0 == s32Ret) { + fseeko(pstBsInfo->fInput, pstBsInfo->curPos, SEEK_SET); + if (pstStreamBuf->bRingbuf == AX_TRUE) { + s32Ret = __StreamReadFrameInRingBuf(pstBsInfo, pstStreamBuf, imgLen, &sReadLen); + if (s32Ret) { + SAMPLE_CRIT_LOG("__StreamReadFrameInRingBuf FAILED! ret:0x%x", s32Ret); + return -1; + } + } else { + if (imgLen > pstStreamBuf->uBufSize) { + SAMPLE_CRIT_LOG("bufSize is not enough(imgLen %d > bufSize %d), please increase STREAM_BUFFER_MAX_SIZE", + imgLen, pstStreamBuf->uBufSize); + return -1; + } + sReadLen = fread(pBufStart, 1, imgLen, pstBsInfo->fInput); + if (sReadLen != imgLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != imgLen:0x%x", (AX_U32)sReadLen, imgLen); + return -1; + } + + pstStreamBuf->pBufBeforeFill = pBufStart; + pstStreamBuf->pBufAfterFill = pBufStart + sReadLen; + } + pstBsInfo->curPos += sReadLen; + } + + *pReadLen = imgLen; + return s32Ret; +} + +static AX_S32 __VdecUsrPicRead(FILE *pFileIn, AX_VIDEO_FRAME_T *pstFrame) +{ + AX_S32 sRet = 0; + AX_U32 i = 0, rows = 0, realRead = 0; + AX_S32 widthSrc = 0, strideSrc = 0, heightSrc = 0; + AX_IMG_FORMAT_E eFmt; + AX_VOID *pVaddr = NULL; + + if (pFileIn == NULL) { + SAMPLE_CRIT_LOG("pFileIn == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstFrame == NULL) { + SAMPLE_CRIT_LOG("pstFrame == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + widthSrc = pstFrame->u32Width; + heightSrc = pstFrame->u32Height; + strideSrc = pstFrame->u32PicStride[0]; + eFmt = pstFrame->enImgFormat; + pVaddr = (AX_VOID *)(AX_ULONG)pstFrame->u64VirAddr[0]; + switch (eFmt) { + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + rows = heightSrc * 3 / 2; + for (i = 0; i < rows; i++) { + realRead = fread(pVaddr, 1, widthSrc, pFileIn); + if (realRead < widthSrc) { + SAMPLE_CRIT_LOG("fread failed! line %d realRead=%d < widthSrc=%d\n", i, realRead, widthSrc); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + pVaddr += strideSrc; + } + break; + default: + SAMPLE_CRIT_LOG("Invalid format, eFmt = %d\n", eFmt); + } + +ERR_RET: + return sRet; +} + +static AX_S32 __VdecUsrPicInfoFill(SAMPLE_VDEC_USRPIC_ARGS_T *pstUsrPicArgs, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_POOL_CONFIG_T stPoolConfig; + AX_U32 FrameSize = 0; + AX_BLK blkId = AX_INVALID_BLOCKID; + AX_VDEC_USRPIC_T *pstUserPic = NULL; + FILE *fpUserYUV = NULL; + AX_S32 sRet = 0; + AX_S32 ret = 0; + AX_VDEC_GRP VdGrp = 0; + AX_U32 uWidth = 0; + AX_U32 u32FrameStride = 0; + + if (pstUsrPicArgs == NULL) { + SAMPLE_CRIT_LOG("pstUsrPicArgs == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstUsrPicArgs->VdGrp; + SAMPLE_LOG("VdGrp=%d begin\n", VdGrp); + + memset(&stPoolConfig, 0x0, sizeof(AX_POOL_CONFIG_T)); + + if (pstUsrPicArgs->tPicParam.bUserPicEnable) { + if ((AX_FORMAT_YUV420_SEMIPLANAR != pstUsrPicArgs->tPicParam.enImgFormat) && + (AX_FORMAT_YUV420_SEMIPLANAR_VU != pstUsrPicArgs->tPicParam.enImgFormat)) { + SAMPLE_CRIT_LOG("VdGrp=%d, unsupport enImgFormat:%d\n", + VdGrp, pstUsrPicArgs->tPicParam.enImgFormat); + sRet = AX_ERR_VDEC_NOT_SUPPORT; + goto ERR_RET; + } + uWidth = pstUsrPicArgs->tPicParam.u32PicWidth; + u32FrameStride = AX_COMM_ALIGN(uWidth, AX_VDEC_WIDTH_ALIGN); + pstVdecUserPic->pUsrPicFilePath = pstUsrPicArgs->tPicParam.pUsrPicFilePath; + FrameSize = AX_VDEC_GetPicBufferSize(u32FrameStride, + pstUsrPicArgs->tPicParam.u32PicHeight, + pstUsrPicArgs->enDecType); + + stPoolConfig.MetaSize = 512; + stPoolConfig.BlkCnt = 1; + + stPoolConfig.BlkSize = FrameSize; + stPoolConfig.CacheMode = AX_POOL_CACHE_MODE_NONCACHE; + snprintf((AX_CHAR *)stPoolConfig.PartitionName, AX_MAX_PARTITION_NAME_LEN, "anonymous"); + pstVdecUserPic->PoolId = AX_POOL_CreatePool(&stPoolConfig); + if (AX_INVALID_POOLID == pstVdecUserPic->PoolId) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_POOL_CreatePool FAILED! BlkCnt:%d, BlkSize:0x%llx\n", + VdGrp, stPoolConfig.BlkCnt, stPoolConfig.BlkSize); + sRet = AX_ERR_VDEC_NOBUF; + goto ERR_RET_DESTROY; + } + + blkId = AX_POOL_GetBlock(pstVdecUserPic->PoolId, FrameSize, NULL); + if (AX_INVALID_BLOCKID == blkId) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_POOL_GetBlock FAILED! PoolId:%d, BlkSize:0x%llx\n", + VdGrp, pstVdecUserPic->PoolId, stPoolConfig.BlkSize); + sRet = AX_ERR_VDEC_NOBUF; + goto ERR_RET_DESTROY; + } + + pstVdecUserPic->BlkId = blkId; + pstUserPic = &pstVdecUserPic->stUserPic; + pstUserPic->bEnable = AX_TRUE; + pstUserPic->bInstant = pstUsrPicArgs->bUsrInstant; + pstUserPic->stFrmInfo.bEndOfStream = AX_TRUE; + pstUserPic->stFrmInfo.enModId = AX_ID_VDEC; + pstUserPic->stFrmInfo.stVFrame.u32BlkId[0] = blkId; + pstUserPic->stFrmInfo.stVFrame.u32FrameSize = FrameSize; + pstUserPic->stFrmInfo.stVFrame.u32Width = pstUsrPicArgs->tPicParam.u32PicWidth; + pstUserPic->stFrmInfo.stVFrame.u32Height = pstUsrPicArgs->tPicParam.u32PicHeight; + pstUserPic->stFrmInfo.stVFrame.enImgFormat = pstUsrPicArgs->tPicParam.enImgFormat; + pstUserPic->stFrmInfo.stVFrame.enVscanFormat = AX_VSCAN_FORMAT_RASTER; + pstUserPic->stFrmInfo.stVFrame.stCompressInfo.enCompressMode = AX_COMPRESS_MODE_NONE; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0] = AX_POOL_Handle2PhysAddr(blkId); + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0] = (AX_ULONG)AX_POOL_GetBlockVirAddr(blkId); + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] = u32FrameStride; + pstUserPic->stFrmInfo.stVFrame.u32PicStride[1] = u32FrameStride; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[1] = pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0] + + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] * + pstUserPic->stFrmInfo.stVFrame.u32Height; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[2] = 0; + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[1] = pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0] + + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] * + pstUserPic->stFrmInfo.stVFrame.u32Height; + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[2] = 0; + pstUserPic->stFrmInfo.stVFrame.u64PTS = 0; + } + + if (pstUsrPicArgs->tPicParam.enImgFormat) { + uWidth = pstUserPic->stFrmInfo.stVFrame.u32Width; + pstVdecUserPic->fpUsrPic = fopen(pstVdecUserPic->pUsrPicFilePath,"rb"); + if (pstVdecUserPic->fpUsrPic == NULL) { + SAMPLE_CRIT_LOG("VdGrp=%d, can't open file %s in VDEC_PREPARE_USERPIC.\n", + VdGrp, pstVdecUserPic->pUsrPicFilePath); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET_FREE; + } + + fpUserYUV = pstVdecUserPic->fpUsrPic; + + sRet = __VdecUsrPicRead(fpUserYUV, &pstUserPic->stFrmInfo.stVFrame); + if (sRet) { + SAMPLE_CRIT_LOG("__VdecUsrPicRead FAILED! ret:%d\n", sRet); + goto ERR_RET_FREE; + } + + sRet = fclose(fpUserYUV); + if (sRet) { + SAMPLE_CRIT_LOG("fclose FAILED! ret:%d\n", sRet); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET_FREE; + } + + fpUserYUV = NULL; + SAMPLE_LOG("VdGrp=%d, .u64PhyAddr[0]:%llx .u64VirAddr[0]:%llx .u64PhyAddr[1]:%llx .u64VirAddr[1]:%llx\n", + VdGrp, + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0], + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0], + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[1], + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[1]); + } + + return AX_SUCCESS; + +ERR_RET_FREE: + if (fpUserYUV) { + fclose(fpUserYUV); + fpUserYUV = NULL; + } + +ERR_RET_DESTROY: + if (pstUsrPicArgs->tPicParam.bUserPicEnable) { + if (pstVdecUserPic->BlkId != AX_INVALID_BLOCKID) { + ret = AX_POOL_ReleaseBlock(pstVdecUserPic->BlkId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolI:%d, AX_POOL_ReleaseBlock FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->BlkId, ret, SampleVdecRetStr(ret)); + } + if (pstVdecUserPic->PoolId != AX_INVALID_POOLID) { + ret = AX_POOL_DestroyPool(pstVdecUserPic->PoolId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolId:%d, AX_POOL_DestroyPool FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->PoolId, ret, SampleVdecRetStr(ret)); + } + } + +ERR_RET: + return sRet; +} + +AX_S32 __VdecUsrPicCreat(SAMPLE_VDEC_USRPIC_ARGS_T *pstUsrPicArgs, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_S32 s32Ret = AX_SUCCESS; + AX_VDEC_GRP VdGrp = 0; + + if (pstVdecUserPic == NULL) { + SAMPLE_CRIT_LOG("null pointer\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstUsrPicArgs == NULL) { + SAMPLE_CRIT_LOG("pstUsrPicArgs == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstUsrPicArgs->VdGrp; + SAMPLE_LOG("VdGrp=%d begin\n", VdGrp); + + if (pstUsrPicArgs->tPicParam.bUserPicEnable == AX_FALSE) { + pstVdecUserPic->PoolId = AX_INVALID_POOLID; + goto ERR_RET; + } + + s32Ret = __VdecUsrPicInfoFill(pstUsrPicArgs, pstVdecUserPic); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, __VdecUsrPicInfoFill FAILED!\n",VdGrp); + goto ERR_RET; + } + + s32Ret = AX_VDEC_DisableUserPic(VdGrp); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_DisableUserPic FAILED! ret:0x%x %s\n", + VdGrp, s32Ret, SampleVdecRetStr(s32Ret)); + goto ERR_RET; + } + + s32Ret = AX_VDEC_SetUserPic(VdGrp, &pstVdecUserPic->stUserPic); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_SetUserPic FAILED! ret:0x%x %s\n", + VdGrp, s32Ret, SampleVdecRetStr(s32Ret)); + goto ERR_RET; + } + +ERR_RET: + return s32Ret; +} + +AX_S32 VdecUserPicEnable(AX_VDEC_GRP VdGrp, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic, + AX_BOOL *pContSendStm, SAMPLE_VDEC_CONTEXT_T *pstCtx) +{ + AX_S32 sRet = AX_SUCCESS; + AX_VDEC_RECV_PIC_PARAM_T tRecvParam; + + if (pstVdecUserPic->stUserPic.bEnable) { + sRet = AX_VDEC_StopRecvStream(VdGrp); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_StopRecvStream FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } else { + pstCtx->GrpStatus[VdGrp] = AX_VDEC_GRP_STOP_RECV; + } + + if (!pstVdecUserPic->stUserPic.bInstant) + sleep(1); + + sRet = AX_VDEC_EnableUserPic(VdGrp); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_EnableUserPic FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } + + if (pstVdecUserPic->recvStmAfUsrPic) { + *pContSendStm = AX_TRUE; + } else { + *pContSendStm = AX_FALSE; + } + + SAMPLE_LOG("VdGrp=%d, AX_VDEC_EnableUserPic finish!\n", VdGrp); + + memset(&tRecvParam, 0, sizeof(tRecvParam)); + tRecvParam.s32RecvPicNum = pstVdecUserPic->s32RecvPicNumBak; + sRet = AX_VDEC_StartRecvStream(VdGrp, &tRecvParam); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_StartRecvStream FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } else { + pstCtx->GrpStatus[VdGrp] = AX_VDEC_GRP_START_RECV; + } + } + +ERR_RET: + return sRet; +} + +AX_VOID VdecUserPicDestroy(AX_VDEC_GRP VdGrp, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_S32 ret = AX_SUCCESS; + + if (pstVdecUserPic->stUserPic.bEnable) { + if (pstVdecUserPic->BlkId != AX_INVALID_BLOCKID) { + ret = AX_POOL_ReleaseBlock(pstVdecUserPic->BlkId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, BlkId:%d, AX_POOL_ReleaseBlock FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->BlkId, ret, SampleVdecRetStr(ret)); + } + if (pstVdecUserPic->PoolId != AX_INVALID_POOLID) { + ret = AX_POOL_DestroyPool(pstVdecUserPic->PoolId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolId:%d, AX_POOL_DestroyPool FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->PoolId, ret, SampleVdecRetStr(ret)); + } + } + + if (pstVdecUserPic->stUserPic.bEnable) { + ret = AX_VDEC_DisableUserPic(VdGrp); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_DisableUserPic FAILED! ret:0x%x %s", + VdGrp, ret, SampleVdecRetStr(ret)); + } +} + + +int VdecCommonPoolPrintf(AX_VOID) +{ + return 0; + AX_S32 s32Ret = 0; + AX_POOL_FLOORPLAN_T PoolFloorPlan; + + memset(&PoolFloorPlan, 0, sizeof(AX_POOL_FLOORPLAN_T)); + + s32Ret = AX_POOL_GetConfig(&PoolFloorPlan); + if (s32Ret) { + SAMPLE_CRIT_LOG("AX_POOL_SetConfig FAILED! 0x%x\n", s32Ret); + goto ERR_RET; + } + + for (int pi = 0; pi < AX_MAX_COMM_POOLS; pi++) { + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].MetaSize:0x%llx", pi, PoolFloorPlan.CommPool[pi].MetaSize); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].BlkSize:0x%llx", pi, PoolFloorPlan.CommPool[pi].BlkSize); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].BlkCnt:0x%x", pi, PoolFloorPlan.CommPool[pi].BlkCnt); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].IsMergeMode:0x%x", pi, PoolFloorPlan.CommPool[pi].IsMergeMode); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].CacheMode:0x%x", pi, PoolFloorPlan.CommPool[pi].CacheMode); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].PartitionName:%s", pi, PoolFloorPlan.CommPool[pi].PartitionName); + } + +ERR_RET: + return s32Ret; +} + +#ifdef AX_VDEC_FFMPEG_ENABLE +AX_S32 SampleVdecFfmpegDeinit(SAMPLE_FFMPEG_T *pstFfmpeg, AX_VDEC_GRP VdGrp) +{ + AX_S32 s32Ret = AX_SUCCESS; + + SAMPLE_LOG("stream %d +++", VdGrp); + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstFfmpeg->pstAvPkt) { + // av_packet_unref(pstFfmpeg->pstAvPkt); + av_packet_free(&pstFfmpeg->pstAvPkt); + pstFfmpeg->pstAvPkt = NULL; + } + + if (pstFfmpeg->pstAvBSFCtx) { + av_bsf_free(&pstFfmpeg->pstAvBSFCtx); + pstFfmpeg->pstAvBSFCtx = NULL; + } + + if (pstFfmpeg->pstAvFmtCtx) { + avformat_close_input(&pstFfmpeg->pstAvFmtCtx); + pstFfmpeg->pstAvFmtCtx = NULL; + } + + SAMPLE_LOG("stream %d ---", VdGrp); + +ERR_RET: + return s32Ret; +} + +AX_S32 SampleVdecFfmpegInit(SAMPLE_FFMPEG_T *pstFfmpeg, const AX_CHAR *pcInputFilePath, + SAMPLE_BITSTREAM_INFO_T *pstBitStreamInfo) +{ + AX_S32 s32Ret = AX_SUCCESS; + int ret; + enum AVCodecID eCodecID = AV_CODEC_ID_H264; + AX_VDEC_GRP VdGrp = 0; + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pcInputFilePath == NULL) { + SAMPLE_CRIT_LOG("pcInputFilePath == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstBitStreamInfo == NULL) { + SAMPLE_CRIT_LOG("pstBitStreamInfo == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstBitStreamInfo->VdGrp; + + pstFfmpeg->s32VideoIndex = -1; + pstFfmpeg->pstAvFmtCtx = avformat_alloc_context(); + if (pstFfmpeg->pstAvFmtCtx == NULL) { + SAMPLE_CRIT_LOG("avformat_alloc_context() failed!"); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + ret = avformat_open_input(&pstFfmpeg->pstAvFmtCtx, pcInputFilePath, NULL, NULL); + if (ret < 0) { + AX_CHAR szError[64] = {0}; + av_strerror(ret, szError, 64); + SAMPLE_CRIT_LOG("open %s fail, error: %d, %s", pcInputFilePath, ret, szError); + goto ERR_RET; + } + + ret = avformat_find_stream_info(pstFfmpeg->pstAvFmtCtx, NULL); + if (ret < 0) { + SAMPLE_CRIT_LOG("avformat_find_stream_info fail, error = %d", ret); + goto ERR_RET; + } + + for (int i = 0; i < pstFfmpeg->pstAvFmtCtx->nb_streams; i++) { + if (AVMEDIA_TYPE_VIDEO == pstFfmpeg->pstAvFmtCtx->streams[i]->codecpar->codec_type) { + pstFfmpeg->s32VideoIndex = i; + break; + } + } + + if (-1 == pstFfmpeg->s32VideoIndex) { + SAMPLE_CRIT_LOG("%s has no video stream!", pcInputFilePath); + goto ERR_RET; + } else { + AVStream *pAvs = pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]; + eCodecID = pAvs->codecpar->codec_id; + switch (eCodecID) { + case AV_CODEC_ID_H264: + pstBitStreamInfo->eVideoType = PT_H264; + break; + case AV_CODEC_ID_HEVC: + pstBitStreamInfo->eVideoType = PT_H265; + break; + default: + SAMPLE_CRIT_LOG("Current Only support H264 or HEVC stream %d!", VdGrp); + goto ERR_RET; + } + + pstBitStreamInfo->nWidth = pAvs->codecpar->width; + pstBitStreamInfo->nHeight = pAvs->codecpar->height; + pstBitStreamInfo->nFps = av_q2d(pAvs->r_frame_rate); + if (0 == pstBitStreamInfo->nFps) { + pstBitStreamInfo->nFps = 30; + SAMPLE_LOG("stream %d fps is 0, set to %d fps", VdGrp, pstBitStreamInfo->nFps); + } + + SAMPLE_LOG("stream %d: vcodec %d, %dx%d, fps %d", + VdGrp, pstBitStreamInfo->eVideoType, pstBitStreamInfo->nWidth, pstBitStreamInfo->nHeight, + pstBitStreamInfo->nFps); + } + + pstFfmpeg->pstAvPkt = av_packet_alloc(); + if (!pstFfmpeg->pstAvPkt) { + SAMPLE_CRIT_LOG("Create packet(stream %d) fail!", VdGrp); + goto ERR_RET; + } + + if ((AV_CODEC_ID_H264 == eCodecID) || (AV_CODEC_ID_HEVC == eCodecID)) { + const AVBitStreamFilter *pstBSFilter = av_bsf_get_by_name((AV_CODEC_ID_H264 == eCodecID) ? + "h264_mp4toannexb" : "hevc_mp4toannexb"); + if (!pstBSFilter) { + SAMPLE_CRIT_LOG("av_bsf_get_by_name(stream %d) fail!", VdGrp); + goto ERR_RET; + } + + ret = av_bsf_alloc(pstBSFilter, &pstFfmpeg->pstAvBSFCtx); + if (ret < 0) { + SAMPLE_CRIT_LOG("av_bsf_alloc(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } + + ret = avcodec_parameters_copy(pstFfmpeg->pstAvBSFCtx->par_in, + pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]->codecpar); + if (ret < 0) { + SAMPLE_CRIT_LOG("avcodec_parameters_copy(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } else { + pstFfmpeg->pstAvBSFCtx->time_base_in = pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]->time_base; + } + + ret = av_bsf_init(pstFfmpeg->pstAvBSFCtx); + if (ret < 0) { + SAMPLE_CRIT_LOG("av_bsf_init(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } + } + + return s32Ret; + +ERR_RET: + SampleVdecFfmpegDeinit(pstFfmpeg, VdGrp); + return s32Ret; +} + +AX_S32 SampleVdecFfmpegExtractOnePic(SAMPLE_FFMPEG_T *pstFfmpeg, SAMPLE_BITSTREAM_INFO_T *pstBitStreamInfo, + SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + AX_S32 s32Ret = AX_SUCCESS; + int ret; + AVFormatContext *pstAvFmtCtx = NULL; + AVBSFContext *pstAvBSFCtx = NULL; + AVPacket *pstAvPkt = NULL; + static AX_U64 u64FrameCnt = 0; + AX_VDEC_GRP VdGrp = 0; + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvFmtCtx = pstFfmpeg->pstAvFmtCtx; + if (pstAvFmtCtx == NULL) { + SAMPLE_CRIT_LOG("pstAvFmtCtx == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvBSFCtx = pstFfmpeg->pstAvBSFCtx; + if (pstAvBSFCtx == NULL) { + SAMPLE_CRIT_LOG("pstAvBSFCtx == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvPkt = pstFfmpeg->pstAvPkt; + if (pstAvPkt == NULL) { + SAMPLE_CRIT_LOG("pstAvPkt == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstBitStreamInfo == NULL) { + SAMPLE_CRIT_LOG("pstBitStreamInfo == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf->tBufAddr.pVirAddr == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf->tBufAddr.pVirAddr == NULL\n"); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (pReadLen == NULL) { + SAMPLE_CRIT_LOG("pReadLen == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstBitStreamInfo->VdGrp; + + while (1) { + ret = av_read_frame(pstAvFmtCtx, pstAvPkt); + if (ret < 0) { + if (AVERROR_EOF == ret) { + SAMPLE_LOG("reach eof of stream %d ", VdGrp); + *pReadLen = 0; + break; + } else { + SAMPLE_CRIT_LOG("av_read_frame(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_STRM_ERROR; + break; + } + } else { + if (pstAvPkt->stream_index == pstFfmpeg->s32VideoIndex) { + ret = av_bsf_send_packet(pstAvBSFCtx, pstAvPkt); + if (ret < 0) { + av_packet_unref(pstAvPkt); + SAMPLE_CRIT_LOG("av_bsf_send_packet(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + break; + } + + while (ret >= 0) { + ret = av_bsf_receive_packet(pstAvBSFCtx, pstAvPkt); + if (ret < 0) { + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + + av_packet_unref(pstAvPkt); + SAMPLE_CRIT_LOG("av_bsf_receive_packet(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + if (pstStreamBuf->uBufSize < pstAvPkt->size) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < pstAvPkt->size:0x%x. " + "bufSize is not enough, please increase STREAM_BUFFER_MAX_SIZE", + pstStreamBuf->uBufSize, pstAvPkt->size); + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + u64FrameCnt++; + *pReadLen = pstAvPkt->size; + memcpy(pstStreamBuf->tBufAddr.pVirAddr, pstAvPkt->data, pstAvPkt->size); + + SAMPLE_LOG_N("u64FrameCnt:%lld, pstAvPkt->size:%d", u64FrameCnt, pstAvPkt->size); + } + } + + av_packet_unref(pstAvPkt); + } + + break; + } + +ERR_RET: + return s32Ret; +} +#endif + +AX_U32 SampleVdecSearchStartCode(AX_U8 *uStrAddr, AX_U32 uLen, AX_U32 *puReadBytes, AX_U32 *puZeroCount) +{ + AX_U32 i = 0; + + if (AX_NULL == uStrAddr) { + return AX_FALSE; + } + + if (uLen < 4) { + return AX_FALSE; + } + + if (AX_NULL == puReadBytes) { + return AX_FALSE; + } + + if (AX_NULL == puZeroCount) { + return AX_FALSE; + } + + *puReadBytes = 0; + *puZeroCount = 0; + + for (i = 0; i < uLen - 4; i++) { + if ((uStrAddr[i] == 0) + && (uStrAddr[i + 1] == 0) + && (uStrAddr[i + 2] == 0) + && (uStrAddr[i + 3] == 1)) { + *puReadBytes = i; + *puZeroCount = 3; + return AX_TRUE; + } else if ((uStrAddr[i] == 0) + && (uStrAddr[i + 1] == 0) + && (uStrAddr[i + 2] == 1)) { + *puReadBytes = i; + *puZeroCount = 2; + return AX_TRUE; + } + } + + return AX_FALSE; +} + +AX_VOID SampelVdecSetThreadName(const char *nameFmt, ...) +{ + AX_CHAR name[16]; + va_list args; + + va_start(args, nameFmt); + vsnprintf(name, sizeof(name), nameFmt, args); + va_end(args); + + prctl(PR_SET_NAME, name, NULL, NULL, NULL); +} + +AX_U64 SampleGetFileSize(char * pFileName) +{ + AX_CHAR cmd[256] = {0}; + FILE *pInfo = NULL; + AX_U64 fileSize = 0; + AX_S32 ret = 0; + AX_U32 i = 0; + + SAMPLE_LOG_TMP("calu file: %s\n", pFileName); + sprintf(cmd, "stat %s > fileInfo.txt", pFileName); + + ret = system(cmd); + if (ret) + return -1; + + pInfo = fopen("fileInfo.txt", "rb"); + if (NULL == pInfo) + return -1; + + fgets(cmd, 256, pInfo); + SAMPLE_LOG_TMP("file: %s\n", cmd); + fgets(cmd, 256, pInfo); + SAMPLE_LOG_TMP("info: %s\n", cmd); + fclose(pInfo); + + if (strlen(cmd) < 10) { + return -1; + } + + while(cmd[i]) { + if (('0' <= cmd[i]) && ('9' >= cmd[i])) + break; + + SAMPLE_LOG_TMP("pos %d char %c\n", i, cmd[i]); + i++; + } + + while(cmd[i]) { + SAMPLE_LOG_TMP("pos %d char %c\n", i, cmd[i]); + if (('0' > cmd[i]) || ('9' < cmd[i])) + break; + + fileSize = fileSize * 10 + (cmd[i] - '0'); + i++; + } + + return fileSize; +} diff --git a/projects/llm_framework/main_skel/src/runner/frameMgr.cpp b/projects/llm_framework/main_skel/src/runner/frameMgr.cpp new file mode 100644 index 00000000..bb5138d9 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/frameMgr.cpp @@ -0,0 +1,72 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include "frameMgr.h" +#include + +AX_BOOL g_bSkel_frame_mgr_inited = AX_FALSE; +AX_POOL g_skel_frame_poolId = AX_INVALID_POOLID; + +AX_VOID FrameMgrCreate(AX_U32 nFrameSize, AX_U32 nDepth) { + AX_POOL_CONFIG_T stPoolConfig; + + memset(&stPoolConfig, 0, sizeof(AX_POOL_CONFIG_T)); + stPoolConfig.MetaSize = 4096; + stPoolConfig.BlkCnt = nDepth; + stPoolConfig.BlkSize = nFrameSize; + stPoolConfig.CacheMode = AX_POOL_CACHE_MODE_NONCACHE; + memset(stPoolConfig.PartitionName, 0, sizeof(stPoolConfig.PartitionName)); + strcpy((AX_CHAR *)stPoolConfig.PartitionName, "anonymous"); + + g_skel_frame_poolId = AX_POOL_CreatePool(&stPoolConfig); + g_bSkel_frame_mgr_inited = AX_TRUE; +} + +AX_BOOL FrameMgrGet(AX_U64 *YUVDataPhy, AX_VOID **YUVDataVir, AX_U32 nFrameSize, AX_U64 nFrameId, AX_BLK *nBlkId) { + if (!g_bSkel_frame_mgr_inited + || g_skel_frame_poolId == AX_INVALID_POOLID) { + return AX_FALSE; + } + + AX_BLK blkId = AX_POOL_GetBlock(g_skel_frame_poolId, nFrameSize, NULL); + + if (blkId != AX_INVALID_BLOCKID) { + *YUVDataPhy = AX_POOL_Handle2PhysAddr(blkId); + *YUVDataVir = AX_POOL_GetBlockVirAddr(blkId); + *nBlkId = blkId; + + return AX_TRUE; + } + + return AX_FALSE; +} + +AX_VOID FrameMgrRelease(AX_BLK blkId) { + if (blkId != AX_INVALID_BLOCKID) { + blkId = AX_POOL_ReleaseBlock(blkId); + } +} + +AX_VOID FrameMgrDestroy(AX_VOID) { + if (!g_bSkel_frame_mgr_inited + || g_skel_frame_poolId == AX_INVALID_POOLID) { + return; + } + + AX_POOL_DestroyPool(g_skel_frame_poolId); + + g_skel_frame_poolId = AX_INVALID_POOLID; + g_bSkel_frame_mgr_inited = AX_FALSE; +} diff --git a/projects/llm_framework/main_skel/src/runner/frameMgr.h b/projects/llm_framework/main_skel/src/runner/frameMgr.h new file mode 100644 index 00000000..ce3fbd42 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/frameMgr.h @@ -0,0 +1,33 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _FRAME_MGR_H_ +#define _FRAME_MGR_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SKEL_FRAME_BUF_DEFAULT_DEPTH 2 + +AX_VOID FrameMgrCreate(AX_U32 nFrameSize, AX_U32 nDepth); +AX_BOOL FrameMgrGet(AX_U64 *YUVDataPhy, AX_VOID **YUVDataVir, AX_U32 nFrameSize, AX_U64 nFrameId, AX_BLK *nBlkId); +AX_VOID FrameMgrRelease(AX_BLK nBlkId); +AX_VOID FrameMgrDestroy(AX_VOID); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/picojson.h b/projects/llm_framework/main_skel/src/runner/picojson.h new file mode 100644 index 00000000..04dab8ba --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/picojson.h @@ -0,0 +1,1151 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef picojson_h +#define picojson_h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// for isnan/isinf +#if __cplusplus >= 201103L +#include +#else +extern "C" { +#ifdef _MSC_VER +#include +#elif defined(__INTEL_COMPILER) +#include +#else +#include +#endif +} +#endif + +#ifndef PICOJSON_USE_RVALUE_REFERENCE +#if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || (defined(_MSC_VER) && _MSC_VER >= 1600) +#define PICOJSON_USE_RVALUE_REFERENCE 1 +#else +#define PICOJSON_USE_RVALUE_REFERENCE 0 +#endif +#endif // PICOJSON_USE_RVALUE_REFERENCE + +#ifndef PICOJSON_NOEXCEPT +#if PICOJSON_USE_RVALUE_REFERENCE +#define PICOJSON_NOEXCEPT noexcept +#else +#define PICOJSON_NOEXCEPT throw() +#endif +#endif + +// experimental support for int64_t (see README.mkdn for detail) +#ifdef PICOJSON_USE_INT64 +#define __STDC_FORMAT_MACROS +#include +#include +#endif + +// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 +#ifndef PICOJSON_USE_LOCALE +#define PICOJSON_USE_LOCALE 1 +#endif +#if PICOJSON_USE_LOCALE +extern "C" { +#include +} +#endif + +#ifndef PICOJSON_ASSERT +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) \ + throw std::runtime_error(#e); \ + } while (0) +#endif + +#ifdef _MSC_VER +#define SNPRINTF _snprintf_s +#pragma warning(push) +#pragma warning(disable : 4244) // conversion from int to char +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4702) // unreachable code +#else +#define SNPRINTF snprintf +#endif + +namespace picojson { + +enum { + null_type, + boolean_type, + number_type, + string_type, + array_type, + object_type +#ifdef PICOJSON_USE_INT64 + , + int64_type +#endif +}; + +enum { INDENT_WIDTH = 2 }; + +struct null {}; + +class value { +public: + typedef std::vector array; + typedef std::map object; + union _storage { + bool boolean_; + double number_; +#ifdef PICOJSON_USE_INT64 + int64_t int64_; +#endif + std::string *string_; + array *array_; + object *object_; + }; + +protected: + int type_; + _storage u_; + +public: + value(); + value(int type, bool); + explicit value(bool b); +#ifdef PICOJSON_USE_INT64 + explicit value(int64_t i); +#endif + explicit value(double n); + explicit value(const std::string &s); + explicit value(const array &a); + explicit value(const object &o); +#if PICOJSON_USE_RVALUE_REFERENCE + explicit value(std::string &&s); + explicit value(array &&a); + explicit value(object &&o); +#endif + explicit value(const char *s); + value(const char *s, size_t len); + ~value(); + value(const value &x); + value &operator=(const value &x); +#if PICOJSON_USE_RVALUE_REFERENCE + value(value &&x) PICOJSON_NOEXCEPT; + value &operator=(value &&x) PICOJSON_NOEXCEPT; +#endif + void swap(value &x) PICOJSON_NOEXCEPT; + template bool is() const; + template const T &get() const; + template T &get(); + template void set(const T &); +#if PICOJSON_USE_RVALUE_REFERENCE + template void set(T &&); +#endif + bool evaluate_as_boolean() const; + const value &get(const size_t idx) const; + const value &get(const std::string &key) const; + value &get(const size_t idx); + value &get(const std::string &key); + + bool contains(const size_t idx) const; + bool contains(const std::string &key) const; + std::string to_str() const; + template void serialize(Iter os, bool prettify = false) const; + std::string serialize(bool prettify = false) const; + +private: + template value(const T *); // intentionally defined to block implicit conversion of pointer to bool + template static void _indent(Iter os, int indent); + template void _serialize(Iter os, int indent) const; + std::string _serialize(int indent) const; + void clear(); +}; + +typedef value::array array; +typedef value::object object; + +inline value::value() : type_(null_type), u_() { +} + +inline value::value(int type, bool) : type_(type), u_() { + switch (type) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(boolean_, false); + INIT(number_, 0.0); +#ifdef PICOJSON_USE_INT64 + INIT(int64_, 0); +#endif + INIT(string_, new std::string()); + INIT(array_, new array()); + INIT(object_, new object()); +#undef INIT + default: + break; + } +} + +inline value::value(bool b) : type_(boolean_type), u_() { + u_.boolean_ = b; +} + +#ifdef PICOJSON_USE_INT64 +inline value::value(int64_t i) : type_(int64_type), u_() { + u_.int64_ = i; +} +#endif + +inline value::value(double n) : type_(number_type), u_() { + if ( +#ifdef _MSC_VER + !_finite(n) +#elif __cplusplus >= 201103L || !(defined(isnan) && defined(isinf)) + std::isnan(n) || std::isinf(n) +#else + isnan(n) || isinf(n) +#endif + ) { + throw std::overflow_error(""); + } + u_.number_ = n; +} + +inline value::value(const std::string &s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const array &a) : type_(array_type), u_() { + u_.array_ = new array(a); +} + +inline value::value(const object &o) : type_(object_type), u_() { + u_.object_ = new object(o); +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(std::string &&s) : type_(string_type), u_() { + u_.string_ = new std::string(std::move(s)); +} + +inline value::value(array &&a) : type_(array_type), u_() { + u_.array_ = new array(std::move(a)); +} + +inline value::value(object &&o) : type_(object_type), u_() { + u_.object_ = new object(std::move(o)); +} +#endif + +inline value::value(const char *s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const char *s, size_t len) : type_(string_type), u_() { + u_.string_ = new std::string(s, len); +} + +inline void value::clear() { + switch (type_) { +#define DEINIT(p) \ + case p##type: \ + delete u_.p; \ + break + DEINIT(string_); + DEINIT(array_); + DEINIT(object_); +#undef DEINIT + default: + break; + } +} + +inline value::~value() { + clear(); +} + +inline value::value(const value &x) : type_(x.type_), u_() { + switch (type_) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(string_, new std::string(*x.u_.string_)); + INIT(array_, new array(*x.u_.array_)); + INIT(object_, new object(*x.u_.object_)); +#undef INIT + default: + u_ = x.u_; + break; + } +} + +inline value &value::operator=(const value &x) { + if (this != &x) { + value t(x); + swap(t); + } + return *this; +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(value &&x) PICOJSON_NOEXCEPT : type_(null_type), u_() { + swap(x); +} +inline value &value::operator=(value &&x) PICOJSON_NOEXCEPT { + swap(x); + return *this; +} +#endif +inline void value::swap(value &x) PICOJSON_NOEXCEPT { + std::swap(type_, x.type_); + std::swap(u_, x.u_); +} + +#define IS(ctype, jtype) \ + template <> inline bool value::is() const { \ + return type_ == jtype##_type; \ + } +IS(null, null) +IS(bool, boolean) +#ifdef PICOJSON_USE_INT64 +IS(int64_t, int64) +#endif +IS(std::string, string) +IS(array, array) +IS(object, object) +#undef IS +template <> inline bool value::is() const { + return type_ == number_type +#ifdef PICOJSON_USE_INT64 + || type_ == int64_type +#endif + ; +} + +#define GET(ctype, var) \ + template <> inline const ctype &value::get() const { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } \ + template <> inline ctype &value::get() { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } +GET(bool, u_.boolean_) +GET(std::string, *u_.string_) +GET(array, *u_.array_) +GET(object, *u_.object_) +#ifdef PICOJSON_USE_INT64 +GET(double, + (type_ == int64_type && (const_cast(this)->type_ = number_type, const_cast(this)->u_.number_ = u_.int64_), + u_.number_)) +GET(int64_t, u_.int64_) +#else +GET(double, u_.number_) +#endif +#undef GET + +#define SET(ctype, jtype, setter) \ + template <> inline void value::set(const ctype &_val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +SET(bool, boolean, u_.boolean_ = _val;) +SET(std::string, string, u_.string_ = new std::string(_val);) +SET(array, array, u_.array_ = new array(_val);) +SET(object, object, u_.object_ = new object(_val);) +SET(double, number, u_.number_ = _val;) +#ifdef PICOJSON_USE_INT64 +SET(int64_t, int64, u_.int64_ = _val;) +#endif +#undef SET + +#if PICOJSON_USE_RVALUE_REFERENCE +#define MOVESET(ctype, jtype, setter) \ + template <> inline void value::set(ctype && _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) +MOVESET(array, array, u_.array_ = new array(std::move(_val));) +MOVESET(object, object, u_.object_ = new object(std::move(_val));) +#undef MOVESET +#endif + +inline bool value::evaluate_as_boolean() const { + switch (type_) { + case null_type: + return false; + case boolean_type: + return u_.boolean_; + case number_type: + return u_.number_ != 0; +#ifdef PICOJSON_USE_INT64 + case int64_type: + return u_.int64_ != 0; +#endif + case string_type: + return !u_.string_->empty(); + default: + return true; + } +} + +inline const value &value::get(const size_t idx) const { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline value &value::get(const size_t idx) { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline const value &value::get(const std::string &key) const { + static value s_null; + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline value &value::get(const std::string &key) { + static value s_null; + PICOJSON_ASSERT(is()); + object::iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline bool value::contains(const size_t idx) const { + PICOJSON_ASSERT(is()); + return idx < u_.array_->size(); +} + +inline bool value::contains(const std::string &key) const { + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end(); +} + +inline std::string value::to_str() const { + switch (type_) { + case null_type: + return "null"; + case boolean_type: + return u_.boolean_ ? "true" : "false"; +#ifdef PICOJSON_USE_INT64 + case int64_type: { + char buf[sizeof("-9223372036854775808")]; + SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); + return buf; + } +#endif + case number_type: { + char buf[256]; + double tmp; + SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); +#if PICOJSON_USE_LOCALE + char *decimal_point = localeconv()->decimal_point; + if (strcmp(decimal_point, ".") != 0) { + size_t decimal_point_len = strlen(decimal_point); + for (char *p = buf; *p != '\0'; ++p) { + if (strncmp(p, decimal_point, decimal_point_len) == 0) { + return std::string(buf, p) + "." + (p + decimal_point_len); + } + } + } +#endif + return buf; + } + case string_type: + return *u_.string_; + case array_type: + return "array"; + case object_type: + return "object"; + default: + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + } + return std::string(); +} + +template void copy(const std::string &s, Iter oi) { + std::copy(s.begin(), s.end(), oi); +} + +template struct serialize_str_char { + Iter oi; + void operator()(char c) { + switch (c) { +#define MAP(val, sym) \ + case val: \ + copy(sym, oi); \ + break + MAP('"', "\\\""); + MAP('\\', "\\\\"); + MAP('/', "\\/"); + MAP('\b', "\\b"); + MAP('\f', "\\f"); + MAP('\n', "\\n"); + MAP('\r', "\\r"); + MAP('\t', "\\t"); +#undef MAP + default: + if (static_cast(c) < 0x20 || c == 0x7f) { + char buf[7]; + SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); + copy(buf, buf + 6, oi); + } else { + *oi++ = c; + } + break; + } + } +}; + +template void serialize_str(const std::string &s, Iter oi) { + *oi++ = '"'; + serialize_str_char process_char = {oi}; + std::for_each(s.begin(), s.end(), process_char); + *oi++ = '"'; +} + +template void value::serialize(Iter oi, bool prettify) const { + return _serialize(oi, prettify ? 0 : -1); +} + +inline std::string value::serialize(bool prettify) const { + return _serialize(prettify ? 0 : -1); +} + +template void value::_indent(Iter oi, int indent) { + *oi++ = '\n'; + for (int i = 0; i < indent * INDENT_WIDTH; ++i) { + *oi++ = ' '; + } +} + +template void value::_serialize(Iter oi, int indent) const { + switch (type_) { + case string_type: + serialize_str(*u_.string_, oi); + break; + case array_type: { + *oi++ = '['; + if (indent != -1) { + ++indent; + } + for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { + if (i != u_.array_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + i->_serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.array_->empty()) { + _indent(oi, indent); + } + } + *oi++ = ']'; + break; + } + case object_type: { + *oi++ = '{'; + if (indent != -1) { + ++indent; + } + for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { + if (i != u_.object_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(i->first, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + i->second._serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.object_->empty()) { + _indent(oi, indent); + } + } + *oi++ = '}'; + break; + } + default: + copy(to_str(), oi); + break; + } + if (indent == 0) { + *oi++ = '\n'; + } +} + +inline std::string value::_serialize(int indent) const { + std::string s; + _serialize(std::back_inserter(s), indent); + return s; +} + +template class input { +protected: + Iter cur_, end_; + bool consumed_; + int line_; + +public: + input(const Iter &first, const Iter &last) : cur_(first), end_(last), consumed_(false), line_(1) { + } + int getc() { + if (consumed_) { + if (*cur_ == '\n') { + ++line_; + } + ++cur_; + } + if (cur_ == end_) { + consumed_ = false; + return -1; + } + consumed_ = true; + return *cur_ & 0xff; + } + void ungetc() { + consumed_ = false; + } + Iter cur() const { + if (consumed_) { + input *self = const_cast *>(this); + self->consumed_ = false; + ++self->cur_; + } + return cur_; + } + int line() const { + return line_; + } + void skip_ws() { + while (1) { + int ch = getc(); + if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { + ungetc(); + break; + } + } + } + bool expect(const int expected) { + skip_ws(); + if (getc() != expected) { + ungetc(); + return false; + } + return true; + } + bool match(const std::string &pattern) { + for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { + if (getc() != *pi) { + ungetc(); + return false; + } + } + return true; + } +}; + +template inline int _parse_quadhex(input &in) { + int uni_ch = 0, hex; + for (int i = 0; i < 4; i++) { + if ((hex = in.getc()) == -1) { + return -1; + } + if ('0' <= hex && hex <= '9') { + hex -= '0'; + } else if ('A' <= hex && hex <= 'F') { + hex -= 'A' - 0xa; + } else if ('a' <= hex && hex <= 'f') { + hex -= 'a' - 0xa; + } else { + in.ungetc(); + return -1; + } + uni_ch = uni_ch * 16 + hex; + } + return uni_ch; +} + +template inline bool _parse_codepoint(String &out, input &in) { + int uni_ch; + if ((uni_ch = _parse_quadhex(in)) == -1) { + return false; + } + if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { + if (0xdc00 <= uni_ch) { + // a second 16-bit of a surrogate pair appeared + return false; + } + // first 16-bit of surrogate pair, get the next one + if (in.getc() != '\\' || in.getc() != 'u') { + in.ungetc(); + return false; + } + int second = _parse_quadhex(in); + if (!(0xdc00 <= second && second <= 0xdfff)) { + return false; + } + uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); + uni_ch += 0x10000; + } + if (uni_ch < 0x80) { + out.push_back(static_cast(uni_ch)); + } else { + if (uni_ch < 0x800) { + out.push_back(static_cast(0xc0 | (uni_ch >> 6))); + } else { + if (uni_ch < 0x10000) { + out.push_back(static_cast(0xe0 | (uni_ch >> 12))); + } else { + out.push_back(static_cast(0xf0 | (uni_ch >> 18))); + out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); + } + out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); + } + out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); + } + return true; +} + +template inline bool _parse_string(String &out, input &in) { + while (1) { + int ch = in.getc(); + if (ch < ' ') { + in.ungetc(); + return false; + } else if (ch == '"') { + return true; + } else if (ch == '\\') { + if ((ch = in.getc()) == -1) { + return false; + } + switch (ch) { +#define MAP(sym, val) \ + case sym: \ + out.push_back(val); \ + break + MAP('"', '\"'); + MAP('\\', '\\'); + MAP('/', '/'); + MAP('b', '\b'); + MAP('f', '\f'); + MAP('n', '\n'); + MAP('r', '\r'); + MAP('t', '\t'); +#undef MAP + case 'u': + if (!_parse_codepoint(out, in)) { + return false; + } + break; + default: + return false; + } + } else { + out.push_back(static_cast(ch)); + } + } + return false; +} + +template inline bool _parse_array(Context &ctx, input &in) { + if (!ctx.parse_array_start()) { + return false; + } + size_t idx = 0; + if (in.expect(']')) { + return ctx.parse_array_stop(idx); + } + do { + if (!ctx.parse_array_item(in, idx)) { + return false; + } + idx++; + } while (in.expect(',')); + return in.expect(']') && ctx.parse_array_stop(idx); +} + +template inline bool _parse_object(Context &ctx, input &in) { + if (!ctx.parse_object_start()) { + return false; + } + if (in.expect('}')) { + return true; + } + do { + std::string key; + if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { + return false; + } + if (!ctx.parse_object_item(in, key)) { + return false; + } + } while (in.expect(',')); + return in.expect('}'); +} + +template inline std::string _parse_number(input &in) { + std::string num_str; + while (1) { + int ch = in.getc(); + if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { + num_str.push_back(static_cast(ch)); + } else if (ch == '.') { +#if PICOJSON_USE_LOCALE + num_str += localeconv()->decimal_point; +#else + num_str.push_back('.'); +#endif + } else { + in.ungetc(); + break; + } + } + return num_str; +} + +template inline bool _parse(Context &ctx, input &in) { + in.skip_ws(); + int ch = in.getc(); + switch (ch) { +#define IS(ch, text, op) \ + case ch: \ + if (in.match(text) && op) { \ + return true; \ + } else { \ + return false; \ + } + IS('n', "ull", ctx.set_null()); + IS('f', "alse", ctx.set_bool(false)); + IS('t', "rue", ctx.set_bool(true)); +#undef IS + case '"': + return ctx.parse_string(in); + case '[': + return _parse_array(ctx, in); + case '{': + return _parse_object(ctx, in); + default: + if (('0' <= ch && ch <= '9') || ch == '-') { + double f; + char *endp; + in.ungetc(); + std::string num_str(_parse_number(in)); + if (num_str.empty()) { + return false; + } +#ifdef PICOJSON_USE_INT64 + { + errno = 0; + intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); + if (errno == 0 && std::numeric_limits::min() <= ival && ival <= std::numeric_limits::max() && + endp == num_str.c_str() + num_str.size()) { + ctx.set_int64(ival); + return true; + } + } +#endif + f = strtod(num_str.c_str(), &endp); + if (endp == num_str.c_str() + num_str.size()) { + ctx.set_number(f); + return true; + } + return false; + } + break; + } + in.ungetc(); + return false; +} + +class deny_parse_context { +public: + bool set_null() { + return false; + } + bool set_bool(bool) { + return false; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { + return false; + } +#endif + bool set_number(double) { + return false; + } + template bool parse_string(input &) { + return false; + } + bool parse_array_start() { + return false; + } + template bool parse_array_item(input &, size_t) { + return false; + } + bool parse_array_stop(size_t) { + return false; + } + bool parse_object_start() { + return false; + } + template bool parse_object_item(input &, const std::string &) { + return false; + } +}; + +class default_parse_context { +protected: + value *out_; + +public: + default_parse_context(value *out) : out_(out) { + } + bool set_null() { + *out_ = value(); + return true; + } + bool set_bool(bool b) { + *out_ = value(b); + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t i) { + *out_ = value(i); + return true; + } +#endif + bool set_number(double f) { + *out_ = value(f); + return true; + } + template bool parse_string(input &in) { + *out_ = value(string_type, false); + return _parse_string(out_->get(), in); + } + bool parse_array_start() { + *out_ = value(array_type, false); + return true; + } + template bool parse_array_item(input &in, size_t) { + array &a = out_->get(); + a.push_back(value()); + default_parse_context ctx(&a.back()); + return _parse(ctx, in); + } + bool parse_array_stop(size_t) { + return true; + } + bool parse_object_start() { + *out_ = value(object_type, false); + return true; + } + template bool parse_object_item(input &in, const std::string &key) { + object &o = out_->get(); + default_parse_context ctx(&o[key]); + return _parse(ctx, in); + } + +private: + default_parse_context(const default_parse_context &); + default_parse_context &operator=(const default_parse_context &); +}; + +class null_parse_context { +public: + struct dummy_str { + void push_back(int) { + } + }; + +public: + null_parse_context() { + } + bool set_null() { + return true; + } + bool set_bool(bool) { + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { + return true; + } +#endif + bool set_number(double) { + return true; + } + template bool parse_string(input &in) { + dummy_str s; + return _parse_string(s, in); + } + bool parse_array_start() { + return true; + } + template bool parse_array_item(input &in, size_t) { + return _parse(*this, in); + } + bool parse_array_stop(size_t) { + return true; + } + bool parse_object_start() { + return true; + } + template bool parse_object_item(input &in, const std::string &) { + return _parse(*this, in); + } + +private: + null_parse_context(const null_parse_context &); + null_parse_context &operator=(const null_parse_context &); +}; + +// obsolete, use the version below +template inline std::string parse(value &out, Iter &pos, const Iter &last) { + std::string err; + pos = parse(out, pos, last, &err); + return err; +} + +template inline Iter _parse(Context &ctx, const Iter &first, const Iter &last, std::string *err) { + input in(first, last); + if (!_parse(ctx, in) && err != NULL) { + char buf[64]; + SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); + *err = buf; + while (1) { + int ch = in.getc(); + if (ch == -1 || ch == '\n') { + break; + } else if (ch >= ' ') { + err->push_back(static_cast(ch)); + } + } + } + return in.cur(); +} + +template inline Iter parse(value &out, const Iter &first, const Iter &last, std::string *err) { + default_parse_context ctx(&out); + return _parse(ctx, first, last, err); +} + +inline std::string parse(value &out, const std::string &s) { + std::string err; + parse(out, s.begin(), s.end(), &err); + return err; +} + +inline std::string parse(value &out, std::istream &is) { + std::string err; + parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); + return err; +} + +template struct last_error_t { static std::string s; }; +template std::string last_error_t::s; + +inline void set_last_error(const std::string &s) { + last_error_t::s = s; +} + +inline const std::string &get_last_error() { + return last_error_t::s; +} + +inline bool operator==(const value &x, const value &y) { + if (x.is()) + return y.is(); +#define PICOJSON_CMP(type) \ + if (x.is()) \ + return y.is() && x.get() == y.get() + PICOJSON_CMP(bool); + PICOJSON_CMP(double); + PICOJSON_CMP(std::string); + PICOJSON_CMP(array); + PICOJSON_CMP(object); +#undef PICOJSON_CMP + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + return false; +} + +inline bool operator!=(const value &x, const value &y) { + return !(x == y); +} +} + +#if !PICOJSON_USE_RVALUE_REFERENCE +namespace std { +template <> inline void swap(picojson::value &x, picojson::value &y) { + x.swap(y); +} +} +#endif + +inline std::istream &operator>>(std::istream &is, picojson::value &x) { + picojson::set_last_error(std::string()); + const std::string err(picojson::parse(x, is)); + if (!err.empty()) { + picojson::set_last_error(err); + is.setstate(std::ios::failbit); + } + return is; +} + +inline std::ostream &operator<<(std::ostream &os, const picojson::value &x) { + x.serialize(std::ostream_iterator(os)); + return os; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/sample_skel.c b/projects/llm_framework/main_skel/src/runner/sample_skel.c new file mode 100644 index 00000000..b1e9364a --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/sample_skel.c @@ -0,0 +1,2285 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "YuvHandler.h" +#include "ax_skel_api.h" +#include "ax_sys_api.h" +#include "ax_ivps_api.h" +#include "ax_venc_api.h" +#include "ax_buffer_tool.h" +#include "skel_log.h" +#include "ax_engine_api.h" +#include "frameMgr.h" +#include "attrParser.h" +#include "statMgr.h" +#include "common_vdec_api.h" + +#ifndef ALIGN_UP +#define ALIGN_UP(x, align) ((((x) + ((align) - 1)) / (align)) * (align)) +#endif + +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, align) (((x) / (align)) * (align)) +#endif + +#ifndef AX_MAX +#define AX_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef AX_MIN +#define AX_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" +#define SKEL_SAMPLE_OUTPUT_VEHICLE_PATH "vehicle" +#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" +#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" +#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" +#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" + +#define SAMPLE_SKEL_SHIFT_LEFT_ALIGN(a) (1 << (a)) +#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) +#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) + +#define RUN_COMMAND(format, ...) do { \ + char cmd[512]; \ + snprintf(cmd, 512, format, ##__VA_ARGS__); \ + system(cmd); \ + } while(0); + +AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) { + if (file) { + va_list args; + char szLog[1024] = {0}; + + va_start(args, fmt); + + AX_U32 len = vsnprintf(szLog, sizeof(szLog), (char *)fmt, args); + if (len < (AX_U32)sizeof(szLog)) { + szLog[len] = '\0'; + } else { + szLog[sizeof(szLog) - 1] = '\0'; + } + + fwrite((AX_U8 *)szLog, 1, strlen(szLog), file); + + va_end(args); + } +} +#define OUTPUT_LOG_SAVE(fmt, ...) LogSaveToFile(fpResultFile, fmt "\n", ##__VA_ARGS__) +#define BLACK_VIDEO_FRAME_COUNT 60 +#define SKEL_SAMPLE_OBJECT_SIZE 512 + +struct skeleton { + int connection[2]; + int left_right_neutral; +}; + +struct skeleton pairs[] = {{{15, 13}, 0}, {{13, 11}, 0}, {{16, 14}, 0}, {{14, 12}, 0}, {{11, 12}, 0}, + {{5, 11}, 0}, {{6, 12}, 0}, {{5, 6}, 0}, {{5, 7}, 0}, {{6, 8}, 0}, + {{7, 9}, 0}, {{8, 10}, 0}, {{1, 2}, 0}, {{0, 1}, 0}, {{0, 2}, 0}, + {{1, 3}, 0}, {{2, 4}, 0}, {{0, 5}, 0}, {{0, 6}, 0}}; + +#define DETECT_SKEL_POINT_COUNT 256 + +typedef struct _AI_Detection_Box_t { + AX_F32 fX, fY, fW, fH; +} AI_Detection_Box_t; + +typedef struct _AI_Detection_Point_t { + AX_F32 fX, fY; +} AI_Detection_Point_t; + +typedef struct _AI_Detection_SkelResult_t { + const AX_CHAR *pstrObjectCategory; + AX_U8 nPointNum; + AI_Detection_Point_t tPoint[DETECT_SKEL_POINT_COUNT]; + AI_Detection_Box_t tBox; +} AI_Detection_SkelResult_t; + +static AX_U64 get_tick_count(AX_VOID) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +// static AX_U64 get_tick_count_us(AX_VOID) { +// struct timespec ts; +// clock_gettime(CLOCK_MONOTONIC, &ts); +// return (ts.tv_sec * 1000000 + ts.tv_nsec / 1000); +// } + +static AX_VOID ShowUsage(AX_VOID) { + // printf("usage: ./%s ...\n", SAMPLE_SKEL_NAME); + printf("options:\n"); + printf("-i, \tInput File(yuv)\n"); + printf("-r, \tInput File Resolution(wxh)(yuv: should be input)\n"); + printf("-w, \tWrite result image to new jpg file((unsigned int), default=0)\n"); + printf("-o, \tSave result to file(file name)\n"); + printf("-m, \tModels deployment path(path name)\n"); + printf("-t, \tRepeat times((unsigned int), default=1)\n"); + printf("-I, \tInterval repeat time((unsigned int)ms, default=0)\n"); + printf("-c, \tConfidence((float: 0-1), default=0)\n"); + printf("-H, \tHuman track size limit((unsigned int), default=0)\n"); + printf("-V, \tVehicle track size limit((unsigned int), default=0)\n"); + printf("-C, \tCylcle track size limit((unsigned int), default=0)\n"); + printf("-d, \tSkel detect type((unsigned int), default=2)\n" + "\t\t0: detect only\n" + "\t\t1: detect + track\n" + "\t\t2: detect + track + push\n"); + printf("-u, \tSkel push strategy((unsigned int), default=3)\n" + "\t\t1: fast push strategy\n" + "\t\t2: push strategy\n" + "\t\t3: best push strategy\n"); + printf("-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" + "\t\t0: VNPU Disable\n" + "\t\t1: STD-VNPU Default\n" + "\t\t2: STD-VNPU1\n" + "\t\t3: STD-VNPU2\n"); + printf("-p, \tSkel PPL((unsigned int), default=1)\n" + "\t\t1: AX_SKEL_PPL_HVCP\n" + "\t\t2: AX_SKEL_PPL_FACE\n"); + printf("-v, \tLog level((unsigned int), default=5)\n" + "\t\t0: LOG_EMERGENCY_LEVEL\n" + "\t\t1: LOG_ALERT_LEVEL\n" + "\t\t2: LOG_CRITICAL_LEVEL\n" + "\t\t3: LOG_ERROR_LEVEL\n" + "\t\t4: LOG_WARN_LEVEL\n" + "\t\t5: LOG_NOTICE_LEVEL\n" + "\t\t6: LOG_INFO_LEVEL\n" + "\t\t7: LOG_DEBUG_LEVEL\n"); + printf("-h, \tprint this message\n"); +} + +AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { + if (pstConfig->nSize > 0 && pstConfig->pstItems) { + for (size_t i = 0; i < pstConfig->nSize; i++) { + if (pstConfig->pstItems[i].pstrType && pstConfig->pstItems[i].pstrValue) { + // cmd: "body_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + if (strcmp(pstConfig->pstItems[i].pstrType, "body_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "crop_encoder_qpLevel", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "crop_encoder_qpLevel") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "detect_roi_polygon") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T)) { + AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, pstConf->nPointNum); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_strategy", value_type: AX_SKEL_PUSH_STRATEGY_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_strategy") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_STRATEGY_T)) { + AX_SKEL_PUSH_STRATEGY_T *pstConf = (AX_SKEL_PUSH_STRATEGY_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, pstConf->ePushMode, pstConf->nIntervalTimes, + pstConf->nPushCounts, pstConf->bPushSameFrame); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_panorama", value_type: AX_SKEL_PUSH_PANORAMA_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_panorama") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T)) { + AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Enable: %d, Quality: %d]", pstConfig->pstItems[i].pstrType, + pstConf->bEnable, pstConf->nQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_body", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_body") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_vehicle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_vehicle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_cycle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_cycle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_face", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_face") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [W: %d, H: %d, P: %f, Y: %f, R: %f, B: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, + pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, + pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_plate", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_plate") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_bind_enable", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_bind_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "track_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "target_config", value_type: AX_SKEL_TARGET_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_target_config", value_type: AX_SKEL_TARGET_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "analyzer_attr_config", value_type: AX_SKEL_ANALYZER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "analyzer_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ANALYZER_CONFIG_T)) { + AX_SKEL_ANALYZER_CONFIG_T *pstConf = (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->peItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + ALOGI("SKEL set %s: [%d]%d", pstConfig->pstItems[i].pstrType, j, pstConf->peItems[j]); + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "venc_attr_config", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "venc_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, (AX_U32)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_attr_always", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_attr_always") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + else { + ALOGE("SKEL cmd: %s not support", pstConfig->pstItems[i].pstrType); + } + } + } + } + + return 0; +} + +FILE* LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + + if (f_in) { + if (pLen) { + fseeko(f_in, 0L, SEEK_END); + AX_U64 nFileSize = ftello(f_in); + rewind(f_in); + + *pLen = (AX_U64)nFileSize; + } + } + + return f_in; +} + +int ReLoadFile(FILE* pFile) +{ + if (pFile) { + rewind(pFile); + return 0; + } + + return -1; +} + +int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, + AX_VOID *pSrcVirAddr, + AX_U32 nSrcSize, + AX_U32 nSrcStride, + AX_U32 nSrcHeight, + AX_U64 nDstPhyAddr, + AX_VOID *pDstVirAddr, + AX_U32 nDstSize, + AX_U32 nDstStride, + AX_U32 nDstHeight + ) { + AX_U64 nStartTime = get_tick_count(); + + // src check + if (!pSrcVirAddr) { + return -1; + } + + if (nSrcSize != nSrcStride * nSrcHeight * 3 /2) { + return -1; + } + + // dst check + if (!pDstVirAddr) { + return -1; + } + + if (nDstSize != nDstStride * nDstHeight * 3 /2) { + return -1; + } + + if (nDstSize < nSrcSize) { + return -1; + } + + if (nDstStride < nSrcStride) { + return -1; + } + + if (nDstHeight != nSrcHeight) { + return -1; + } + + AX_VOID *src = pSrcVirAddr; + AX_VOID *dst = pDstVirAddr; + for(AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { + memcpy(dst, src, nSrcStride); + src += nSrcStride; + dst += nDstStride; + } + + AX_U64 nProcessElasped = get_tick_count() - nStartTime; + + ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, nProcessElasped); + + return 0; +} + +int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) +{ + AX_S32 nReadSize = 0; + + if (pFile) { + if (nPhyAddr != 0 && pVirAddr) { + memset((AX_U8 *)pVirAddr, 0x00, nSize); + nReadSize = fread((AX_U8 *)pVirAddr, 1, nSize, pFile); + } + else { + fseeko(pFile, nSize, SEEK_CUR); + } + } + + return nReadSize; +} + +int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) +{ + SAMPLE_INPUT_FILE_INFO_T stStreamInfo; + SAMPLE_STREAM_BUF_T stStreamBuf; + size_t nReadLen = 0; + AX_U64 streamPhyAddr = 0; + AX_VOID *pStreamVirAddr = NULL; + AX_VDEC_DEC_ONE_FRM_T decOneFrmParam; + size_t nFileSize = 0; + AX_U32 heightAlign = 0; + AX_U32 frmStride = 0; + frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; + heightAlign = ALIGN_UP(nHeight, 16); + AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; + + memset(&stStreamInfo, 0, sizeof(SAMPLE_INPUT_FILE_INFO_T)); + memset(&stStreamBuf, 0, sizeof(SAMPLE_STREAM_BUF_T)); + memset(&decOneFrmParam, 0, sizeof(AX_VDEC_DEC_ONE_FRM_T)); + + rewind(InputFileHandle); + fseek(InputFileHandle, 0, SEEK_END); + nFileSize = (size_t)ftello(InputFileHandle); + + stStreamInfo.fInput = InputFileHandle; + stStreamInfo.sFileSize = nFileSize; + + stStreamBuf.uBufSize = ((AX_U32)nFileSize) > STREAM_BUFFER_MAX_SIZE ? STREAM_BUFFER_MAX_SIZE : nFileSize; + int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, + stStreamBuf.uBufSize, 0x100, (AX_S8 *)"vdec_input_stream"); + if (ret != AX_SUCCESS) { + ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", + stStreamBuf.uBufSize, ret); + return -1; + } + + stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; + stStreamBuf.tBufAddr.u64PhyAddr = streamPhyAddr; + + ret = StreamParserReadFrameJpeg(&stStreamInfo, &stStreamBuf, &nReadLen); + if (0 != ret) { + ALOGE("StreamParserReadFrameJpeg failed! ret = 0x%x", ret); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return -1; + } + + if (!nReadLen) { + ALOGE("read jpeg frame FAILED!\n"); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + ret = AX_ERR_VDEC_STRM_ERROR; + return ret; + } + + decOneFrmParam.stStream.pu8Addr = (AX_U8*)stStreamBuf.tBufAddr.pVirAddr; + decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; + decOneFrmParam.stStream.u32StreamPackLen = (AX_U32)nReadLen; + + decOneFrmParam.stFrame.u64VirAddr[0] = (AX_ULONG)OneYUVDataVir; + decOneFrmParam.stFrame.u64VirAddr[1] = (AX_ULONG)OneYUVDataVir + frmStride * heightAlign; + decOneFrmParam.stFrame.u64PhyAddr[0] = OneYUVDataPhy; + decOneFrmParam.stFrame.u64PhyAddr[1] = OneYUVDataPhy + frmStride * heightAlign; + decOneFrmParam.stFrame.u32FrameSize = nFrameSize; + + ret = AX_VDEC_JpegDecodeOneFrame(&decOneFrmParam); + if (ret != AX_SUCCESS) { + ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", + ret, SampleVdecRetStr(ret)); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return ret; + } + + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + + return 0; +} + +int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr, AX_U32 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + if (f_in == NULL) { + ALOGE("Unable to open input file\n"); + return -1; + } + + /* file i/o pointer to full */ + fseek(f_in, 0L, SEEK_END); + AX_U32 nFileSize = ftell(f_in); + rewind(f_in); + + if (pPhyAddr && ppVirAddr) { + AX_U64 nPhyAddr = 0; + AX_VOID *pVirAddr = NULL; + + // pVirAddr = (AX_VOID *)malloc(nFileSize); + AX_S32 nRet = AX_SYS_MemAlloc(&nPhyAddr, (AX_VOID **)&pVirAddr, nFileSize, 128, (AX_S8 *)"SKEL_TEST"); + + if (nRet != 0) { + fclose(f_in); + ALOGE("AX_SYS_MemAlloc failed"); + return -1; + } + + fread((AX_U8 *)pVirAddr, nFileSize, 1, f_in); + + *pPhyAddr = nPhyAddr; + *ppVirAddr = pVirAddr; + } + + if (pLen) { + *pLen = nFileSize; + } + + fclose(f_in); + + return 0; +} + +int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_U32 nLen) { + AX_S32 s32Ret = AX_SUCCESS; + + AX_U32 frameSize = nLen; + AX_JPEG_ENCODE_ONCE_PARAMS_T stJpegEncodeOnceParam; + memset(&stJpegEncodeOnceParam, 0, sizeof(stJpegEncodeOnceParam)); + + AX_U64 outPhyAddr = 0; + AX_VOID *outVirAddr = NULL; + + s32Ret = AX_SYS_MemAlloc(&outPhyAddr, &outVirAddr, frameSize, 128, (AX_S8 *)"SKEL-VENC"); + if (s32Ret) { + ALOGE("alloc mem err, size(%d).", frameSize); + return -1; + } + + // output + stJpegEncodeOnceParam.u32OutBufSize = frameSize; + stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; + stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; + + // input + stJpegEncodeOnceParam.stJpegParam.u32Qfactor = 90; + stJpegEncodeOnceParam.u32Width = nWidth; + stJpegEncodeOnceParam.u32Height = nHeight; + stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stJpegEncodeOnceParam.u32PicStride[0] = nStride; + stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; + stJpegEncodeOnceParam.u32PicStride[2] = 0; + + stJpegEncodeOnceParam.u64PhyAddr[0] = nPhyAddr; + stJpegEncodeOnceParam.u64PhyAddr[1] = stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64PhyAddr[2] = 0; + + stJpegEncodeOnceParam.u64VirAddr[0] = (AX_ULONG)pVirAddr; + stJpegEncodeOnceParam.u64VirAddr[1] = stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64VirAddr[2] = 0; + + s32Ret = AX_VENC_JpegEncodeOneFrame(&stJpegEncodeOnceParam); + if (AX_SUCCESS != s32Ret) { + ALOGE("AX_VENC_JpegEncodeOneFrame fail, ret=0x%x", s32Ret); + s32Ret = -1; + goto JENC_EXIT; + } + +JENC_EXIT: + if (0 == s32Ret && dstFile) { + FILE *fp_w = fopen(dstFile, "wb"); + + if (fp_w) { + ALOGN("\tWrite new JPG result image to file: %s", dstFile); + fwrite((AX_U8 *)stJpegEncodeOnceParam.pu8Addr, stJpegEncodeOnceParam.u32Len, 1, fp_w); + fclose(fp_w); + } + } + + if (outPhyAddr && (NULL != outVirAddr)) { + AX_SYS_MemFree(outPhyAddr, outVirAddr); + } + + return s32Ret; +} + +AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) { + if (nFrameSeqNum < 1) { + nFrameSeqNum = 1; + } + + if (nSrcFrameRate == nDstFrameRate) { + return AX_FALSE; + } + + if (nDstFrameRate > nSrcFrameRate) { + return AX_FALSE; + } + + if ((nFrameSeqNum * nDstFrameRate / (nSrcFrameRate)) > ((nFrameSeqNum - 1) * nDstFrameRate / (nSrcFrameRate))) { + return AX_FALSE; + } else { + return AX_TRUE; + } +} + +AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { + AX_S32 nRet = 0; + AX_S32 c; + AX_S32 isExit = 0; + AX_BOOL bJpgFile = AX_FALSE; + FILE *InputFileHandle = NULL; + const AX_CHAR *InputFile = NULL; + const AX_CHAR *InputResolution = NULL; + AX_S32 InputFileFrameCnt = 1; + const AX_CHAR *ReWritePath = NULL; + const AX_CHAR *SaveResultPath = NULL; + const AX_CHAR *ModelsPath = NULL; + FILE *fpResultFile = NULL; + AX_S32 nRepeatTimes = 1; + AX_S32 nPPL = AX_SKEL_PPL_HVCP; + AX_S32 nDetectType = 2; + AX_S32 nPushStrategy = 3; + AX_S32 nNpuType = 0; + AX_S32 nInterval = 0; + AX_U32 nStride = 1920; + AX_U32 nWidth = 1920; + AX_U32 nHeight = 1080; + AX_U32 nJencBufSize = 0; + AX_U32 nFrameDepth = 1; + AX_U32 nOneSize = 0; + AX_U64 OneYUVDataPhy = 0; + AX_VOID *OneYUVDataVir = NULL; + AX_U32 nFrameSize = nStride * nHeight * 3 / 2; + AX_U32 nCacheListDepth = 1; + AX_S32 nSrcFrameRate = 25; + AX_S32 nDstFrameRate = 25; + AX_VOID *YUVDataVir = NULL; + AX_U64 YUVDataPhy = 0; + AX_VOID *YUVDataWrVir = NULL; + AX_U64 YUVDataWrPhy = 0; + AX_SKEL_HANDLE pHandle = NULL; + AX_SKEL_FRAME_T stFrame = {0}; + AX_SKEL_RESULT_T *pstResult = NULL; + AX_U64 nFileSize = 0; + AX_U64 nStartTime = 0; + AX_U64 nInitElasped = 0; + AX_U64 nCreateElasped = 0; + AX_U64 nProcessElasped = 0; + AX_U64 nResultElasped = 0; + AX_U64 nResultElaspedMin = (AX_U64)-1; + AX_U64 nResultElaspedMax = 0; + AX_U64 nResultElaspedTotal = 0; + AX_F32 fConfidence = 0.0; + // AX_U32 nHumantracksize = 0; + // AX_U32 nVehicletracksize = 0; + // AX_U32 nCycletracksize = 0; + AX_U64 nSkelFrameId = 1; + +#if defined(SAMPLE_SKEL_BUILD_VERSION) + printf("SKEL sample: %s build: %s %s\n", SAMPLE_SKEL_BUILD_VERSION, __DATE__, __TIME__); +#endif + + while ((c = getopt(argc, argv, "i:r:I:w:o:m:t:d:u:p:j:v:c:N:H:V:C:h::")) != -1) { + isExit = 0; + switch (c) { + case 'i': + InputFile = (const AX_CHAR *)optarg; + break; + case 'r': + InputResolution = (const AX_CHAR *)optarg; + break; + case 'I': + nInterval = atoi(optarg); + break; + case 'w': + ReWritePath = (const AX_CHAR *)optarg; + break; + case 'o': + SaveResultPath = (const AX_CHAR *)optarg; + break; + case 'm': + ModelsPath = (const AX_CHAR *)optarg; + break; + case 't': + nRepeatTimes = atoi(optarg); + break; + case 'd': + nDetectType = atoi(optarg); + break; + case 'u': + nPushStrategy = atoi(optarg); + break; + case 'N': + nNpuType = atoi(optarg); + break; + case 'p': + nPPL = atoi(optarg); + break; + case 'j': + nJencBufSize = atoi(optarg); + break; + case 'c': + fConfidence = atof(optarg); + break; + // case 'H': + // nHumantracksize = atoi(optarg); + // break; + // case 'V': + // nVehicletracksize = atoi(optarg); + // break; + // case 'C': + // nCycletracksize = atoi(optarg); + // break; + case 'v': + log_level = atoi(optarg); + break; + case 'h': + isExit = 1; + break; + case 'D': + nCacheListDepth = atoi(optarg); + break; + default: + isExit = 1; + break; + } + } + + if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) + || (log_level < 0 || log_level >= SKEL_LOG_MAX) + || (fConfidence < 0 || fConfidence > 1) + || (nDetectType < 0 || nDetectType > 2) + || (nPushStrategy < 1 || nPushStrategy > 3) + || (nNpuType > 3)) { + ShowUsage(); + exit(0); + } + + if (InputFile) { + AX_U32 nInputFileLen = strlen(InputFile); + + if (nInputFileLen > 4 && (strcasecmp(&InputFile[nInputFileLen - 4], ".jpg") == 0)) { + bJpgFile = AX_TRUE; + } + } + + if (nRepeatTimes <= 0) { + nRepeatTimes = 1; + } + + if (nInterval < 0) { + nInterval = 0; + } + if (nCacheListDepth == 0) { + nCacheListDepth = 1; + } + nInterval = nInterval * 1000; + + if (access(InputFile, 0) != 0) { + ALOGE("%s not exist", InputFile); + exit(0); + } + + // clear output + if (SaveResultPath) { + RUN_COMMAND("rm -rf %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_BODY_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_VEHICLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_CYCLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_FACE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_PLATE_PATH); + } + + if (ReWritePath) { + RUN_COMMAND("rm -rf %s", ReWritePath); + RUN_COMMAND("mkdir -p %s", ReWritePath); + } + + if (InputResolution) { + AX_CHAR *temp_p = strstr(InputResolution, "x"); + + if (!temp_p || strlen(temp_p) <= 1) { + ShowUsage(); + exit(0); + } + + nWidth = atoi(InputResolution); + nStride = nWidth; + nHeight = atoi(temp_p + 1); + } + + nRet = AX_SYS_Init(); + if (0 != nRet) { + ALOGE("AX_SYS_Init() fail, ret = 0x%x", nRet); + exit(0); + } + + nRet = AX_IVPS_Init(); + + ALOGI("AX_IVPS_Init nRet = 0x%x", nRet); + + AX_VENC_MOD_ATTR_T stVencModAttr; + memset(&stVencModAttr, 0x00, sizeof(stVencModAttr)); + stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; + stVencModAttr.stModThdAttr.u32TotalThreadNum = 1; + stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; + nRet = AX_VENC_Init(&stVencModAttr); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VENC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VENC_Init nRet = 0x%x", nRet); + + nRet = AX_VDEC_Init(NULL); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VDEC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VDEC_Init nRet = 0x%x", nRet); + + // init engine + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); +#if defined(CHIP_AX650) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else if (nNpuType >= 1 && nNpuType <= 4) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_STD; + } + else if (nNpuType >= 5 && nNpuType <= 7) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_BIG_LITTLE; + } +#elif defined(CHIP_AX620E) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_ENABLE; + } +#else + // #error "NO CHIP SELECTED." +#endif + + nRet = AX_ENGINE_Init(&npu_attr); + if (0 != nRet) { + ALOGE("AX_NPU_SDK_EX_Init_with_attr() fail, ret = 0x%x", nRet); + goto EXIT0; + } + + nOneSize = nWidth * nHeight * 3 / 2; + nFrameSize = nStride * nHeight * 3 / 2; + + InputFileHandle = LoadFile(InputFile, &nFileSize); + + if (bJpgFile) { + InputFileFrameCnt = 1; + } + else { + if (!InputFileHandle + || (nFileSize % nOneSize) != 0) { + ALOGE("%s file is not %dx%d", InputFile, nWidth, nHeight); + goto EXIT1; + } + + InputFileFrameCnt = nFileSize / nOneSize; + } + + + if (nWidth%2 == 1 + || nHeight%2 == 1) { + ALOGE("wxh(%dx%d) should be even", nWidth, nHeight); + goto EXIT1; + } + + // YUVDataWrVir = (AX_VOID *)malloc(nFileSize); + nRet = AX_SYS_MemAlloc(&YUVDataWrPhy, &YUVDataWrVir, nFrameSize, 128, (const AX_S8 *)"SKEL_TEST"); + + if (!YUVDataWrVir) { + ALOGE("malloc fail nRet=0x%x", nRet); + goto EXIT1; + } + + if (SaveResultPath) { + AX_CHAR szPath[512] = {0}; + snprintf(szPath, 511, "%s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_LOG_FILE); + fpResultFile = fopen(szPath, "a+"); + + if (!fpResultFile) { + ALOGE("%s file not exist", szPath); + goto EXIT1; + } + } + + nStartTime = get_tick_count(); + + AX_SKEL_INIT_PARAM_T stInitParam = {0}; + + if (ModelsPath) { + stInitParam.pStrModelDeploymentPath = ModelsPath; + } + else { + stInitParam.pStrModelDeploymentPath = "/opt/etc/skelModels"; + } + + nRet = AX_SKEL_Init(&stInitParam); + + nInitElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL init fail, ret = 0x%x", nRet); + + goto EXIT1; + } + + // get version + { + const AX_SKEL_VERSION_INFO_T *pstVersion = NULL; + + nRet = AX_SKEL_GetVersion(&pstVersion); + + if (0 != nRet) { + ALOGI("SKEL get version fail, ret = 0x%x", nRet); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + goto EXIT1; + } + + ALOGI("SKEL version: %s", pstVersion->pstrVersion); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + } + + // get capability + { + const AX_SKEL_CAPABILITY_T *pstCapability = NULL; + + nRet = AX_SKEL_GetCapability(&pstCapability); + + if (0 != nRet) { + ALOGI("SKEL get capability fail, ret = 0x%x", nRet); + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + goto EXIT1; + } + + for (AX_U32 i = 0; i < pstCapability->nPPLConfigSize; i++) { + ALOGI("SKEL capability[%d]: (ePPL: %d, PPLConfigKey: %s)", i, pstCapability->pstPPLConfig[i].ePPL, + pstCapability->pstPPLConfig[i].pstrPPLConfigKey); + } + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + } + + AX_SKEL_HANDLE_PARAM_T stHandleParam = {0}; + + stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; + stHandleParam.nFrameDepth = nFrameDepth; + stHandleParam.nFrameCacheDepth = nCacheListDepth; + stHandleParam.nIoDepth = 0; + stHandleParam.nWidth = nWidth; + stHandleParam.nHeight = nHeight; + + // config settings (if need) + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[16] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; + + // venc_attr_config + // default will be system definition: w*h*3/8 + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVencAttrConfigThreshold = {0}; + if (nJencBufSize > 0) { + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; + stVencAttrConfigThreshold.fValue = (AX_F32) nJencBufSize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stTrackEnableThreshold = {0}; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; + + // detect only (disable track + disable push) + if (nDetectType == 0) { + // track_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + stTrackEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + // detect + track (disable push) + else if (nDetectType == 1) { + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + if (itemIndex > 0) { + stConfig.nSize = itemIndex; + stHandleParam.stConfig = stConfig; + } + + // default NPU + if (nNpuType == 0 || nNpuType == 1) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_NPU_DEFAULT; + } + // STD-NPU + else if (nNpuType == 2) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_1; + } + else if (nNpuType == 3) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_2; + } + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_Create(&stHandleParam, &pHandle); + + nCreateElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Create Handle fail, ret = 0x%x", nRet); + + goto EXIT2; + } + + if (fConfidence == 0) { + fConfidence = 0.5; + } + + // set config + { + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[64] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; +#if 0 + // body_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyMaxTargetCount = {0}; + stBodyMaxTargetCount.fValue = (AX_F32)nHumantracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleMaxTargetCount = {0}; + stVehicleMaxTargetCount.fValue = (AX_F32)nVehicletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleMaxTargetCount = {0}; + stCycleMaxTargetCount.fValue = (AX_F32)nCycletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif +#if 0 + // body_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyConfidence = {0}; + stBodyConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stFaceConfidence = {0}; + stFaceConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleConfidence = {0}; + stVehicleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleConfidence = {0}; + stCycleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPlateConfidence = {0}; + stPlateConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // crop_encoder_qpLevel + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"crop_encoder_qpLevel"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCropEncoderQpLevelThreshold = {0}; + stCropEncoderQpLevelThreshold.fValue = 90; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCropEncoderQpLevelThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // body_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stBodyMinSize = {0}; + stBodyMinSize.nWidth = 0; + stBodyMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // face_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stFaceMinSize = {0}; + stFaceMinSize.nWidth = 0; + stFaceMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // vehicle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stVehicleMinSize = {0}; + stVehicleMinSize.nWidth = 0; + stVehicleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // cycle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stCycleMinSize = {0}; + stCycleMinSize.nWidth = 0; + stCycleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // plate_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stPlateMinSize = {0}; + stPlateMinSize.nWidth = 0; + stPlateMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; +#endif + + // detect_roi_polygon + AX_SKEL_ROI_POLYGON_CONFIG_T stDetectRoi = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; + stDetectRoi.bEnable = AX_FALSE; + stDetectRoi.nPointNum = 4; + AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; + stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); + itemIndex++; + + // push_strategy + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; + AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; + stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; + stPushStrategy.nIntervalTimes = 2000; + stPushStrategy.nPushCounts = 1; + stPushStrategy.bPushSameFrame = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_STRATEGY_T); + itemIndex++; + + //4 crop_encoder +#if 0 + // body_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stBodyCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_crop_encoder"; + stBodyCropEncoderThreshold.fScaleLeft = 0; + stBodyCropEncoderThreshold.fScaleRight = 0; + stBodyCropEncoderThreshold.fScaleTop = 0; + stBodyCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stVehicleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_crop_encoder"; + stVehicleCropEncoderThreshold.fScaleLeft = 0; + stVehicleCropEncoderThreshold.fScaleRight = 0; + stVehicleCropEncoderThreshold.fScaleTop = 0; + stVehicleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stCycleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_crop_encoder"; + stCycleCropEncoderThreshold.fScaleLeft = 0; + stCycleCropEncoderThreshold.fScaleRight = 0; + stCycleCropEncoderThreshold.fScaleTop = 0; + stCycleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stFaceCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_crop_encoder"; + stFaceCropEncoderThreshold.fScaleLeft = 0; + stFaceCropEncoderThreshold.fScaleRight = 0; + stFaceCropEncoderThreshold.fScaleTop = 0; + stFaceCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stPlateCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_crop_encoder"; + stPlateCropEncoderThreshold.fScaleLeft = 0; + stPlateCropEncoderThreshold.fScaleRight = 0; + stPlateCropEncoderThreshold.fScaleTop = 0; + stPlateCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + // push_panorama + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; + AX_SKEL_PUSH_PANORAMA_CONFIG_T stPushPanoramaConfig = {0}; + stPushPanoramaConfig.bEnable = AX_FALSE; + stPushPanoramaConfig.nQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); + itemIndex++; +#if 0 + // push_quality_body + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_body"; + AX_SKEL_ATTR_FILTER_CONFIG_T stBodyAttrFilter = {0}; + stBodyAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_vehicle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_vehicle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stVehicleAttrFilter = {0}; + stVehicleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_cycle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_cycle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stCycleAttrFilter = {0}; + stCycleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_face + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_face"; + AX_SKEL_ATTR_FILTER_CONFIG_T stFaceAttrFilter = {0}; + stFaceAttrFilter.stFaceAttrFilterConfig.nWidth = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.nHeight = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fPitch = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fYaw = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fRoll = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fBlur = 1.0; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceAttrFilter; + itemIndex++; + + // push_quality_plate + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_plate"; + AX_SKEL_ATTR_FILTER_CONFIG_T stPlateAttrFilter = {0}; + stPlateAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; +#endif +//#if 0 + // track_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stTrackEnable = {0}; + stTrackEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; + + //push_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushEnable = {0}; + stPushEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +//#endif +#if 0 + //target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + //push_target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + // push_attr_always + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_attr_always"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushAttrAlways = {0}; + stPushAttrAlways.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushAttrAlways; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +#endif + //analyzer_attr_config + // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; + // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; + // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; + + // stAnalyze.nSize = 1; + // AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_FACE_ATTRIBUTE; + // stAnalyze.peItems = &analyze_attribute; + + // stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stAnalyze; + // stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ANALYZER_CONFIG_T); + // itemIndex++; + + stConfig.nSize = itemIndex; + + nRet = AX_SKEL_SetConfig(pHandle, &stConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_SetConfig, ret = 0x%x", nRet); + + goto EXIT2; + } + } + + // get config + { + const AX_SKEL_CONFIG_T *pstConfig = NULL; + + nRet = AX_SKEL_GetConfig(pHandle, &pstConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_GetConfig, ret = 0x%x", nRet); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + + goto EXIT2; + } + + ParseConfigParam(pstConfig); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + } + + ALOGN("Task infomation:"); + ALOGN("\tInput file: %s", InputFile); + ALOGN("\tInput file resolution: %dx%d", nWidth, nHeight); + ALOGN("\tRepeat times: %d", nRepeatTimes); + ALOGN("SKEL Init Elapse:"); + ALOGN("\tAX_SKEL_Init: %lld ms", nInitElasped); + ALOGN("\tAX_SKEL_Create: %lld ms", nCreateElasped); + + OUTPUT_LOG_SAVE("==============================%s process result:==============================", InputFile); + + if (InputFileFrameCnt > 1) { + InputFileFrameCnt = InputFileFrameCnt + BLACK_VIDEO_FRAME_COUNT; + } + + // create frame mgr + FrameMgrCreate(nFrameSize, nFrameDepth + nCacheListDepth); + + for (AX_U32 nRepeat = 0; nRepeat < nRepeatTimes; nRepeat++) { + ALOGN("#####SKEL Process times: %d", nRepeat + 1); + + OUTPUT_LOG_SAVE("#####SKEL Process times: %d", nRepeat + 1); + + STAT_OBJECT_NUM_T tObjectTrackNum = {0}; + STAT_OBJECT_NUM_T tObjectPushNum = {0}; + + for (AX_U32 nFrameCnt = 0; nFrameCnt < InputFileFrameCnt; nFrameCnt++) { + AX_BLK blkId = AX_INVALID_BLOCKID; + + if (InputFileFrameCnt != 1) { + if (FrameSkipCtrl(nSrcFrameRate, nDstFrameRate, nSkelFrameId)) { + nSkelFrameId ++; + + LoadFileToMem(InputFileHandle, 0, NULL, nOneSize); + + if (nInterval > 0) { + usleep(nInterval); + } + continue; + } + else { + if (!FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId)) { + + ALOGE("FrameMgrGet fail"); + + goto EXIT1; + } + + if (nOneSize != nFrameSize) { + LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nOneSize); + } + } + } else { + FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId); + + if (nRet != 0 + || YUVDataPhy == 0 + || YUVDataVir == NULL) { + ALOGE("Load input file fail"); + goto EXIT1; + } + + if (bJpgFile) { + nRet = DecodeJpeg(InputFileHandle, YUVDataPhy, YUVDataVir, nWidth, nHeight); + if (0 != nRet) { + ALOGE("DecodeJpeg failed!"); + goto EXIT3; + } + } + else { + if (nOneSize != nFrameSize) { + nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, (AX_S8 *)"SKEL_TEST"); + + nRet = LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + nRet = LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nFrameSize); + } + } + } + stFrame.stFrame.u32Width = nWidth; + stFrame.stFrame.u32Height = nHeight; + stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stFrame.stFrame.u32FrameSize = nFrameSize; + stFrame.stFrame.u32PicStride[0] = nStride; + stFrame.stFrame.u32PicStride[1] = nStride; + stFrame.stFrame.u32PicStride[2] = nStride; + stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; + stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; + stFrame.stFrame.u64PhyAddr[2] = 0; + stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); + stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); + stFrame.stFrame.u64VirAddr[2] = 0; + stFrame.stFrame.u32BlkId[0] = blkId; + stFrame.stFrame.u32BlkId[1] = 0; + stFrame.stFrame.u32BlkId[2] = 0; + ALOGN("*****SKEL Frame(%d) Process", nFrameCnt + 1); + + //OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); + + stFrame.nFrameId = nSkelFrameId ++; + nStartTime = get_tick_count(); + + while (1) { + nRet = AX_SKEL_SendFrame(pHandle, &stFrame, 0); + + if (nRet == 0) { + break; + } + + usleep(1000); + } + + FrameMgrRelease(blkId); + + nProcessElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Process fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("SKEL Process Elapse:"); + ALOGN("\tAX_SKEL_SendFrame: %lld ms", nProcessElasped); + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_GetResult(pHandle, &pstResult, -1); + + nResultElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL get result fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("\tAX_SKEL_GetResult: %lld ms", nResultElasped); + + nResultElaspedTotal += nResultElasped; + + if (nProcessElasped + nResultElasped > nResultElaspedMax) { + nResultElaspedMax = nProcessElasped + nResultElasped; + } + + if (nProcessElasped + nResultElasped < nResultElaspedMin) { + nResultElaspedMin = nProcessElasped + nResultElasped; + } + + ALOGN("SKEL Process Result:"); + + ALOGI("\tFrameId: %lld", pstResult->nFrameId); + ALOGI("\tnOriginal WxH: %dx%d", pstResult->nOriginalWidth, pstResult->nOriginalHeight); + + ALOGN("\tObject Num: %d", pstResult->nObjectSize); + + AX_U32 nSkelSize = 0; + AI_Detection_SkelResult_t Skels[SKEL_SAMPLE_OBJECT_SIZE] = {0}; + for (AX_U32 i = 0; i < pstResult->nObjectSize; i++) { + AX_SKEL_OBJECT_ITEM_T *pstItems = &pstResult->pstObjectItems[i]; + + ALOGI("\t\tFrameId: %lld", pstItems->nFrameId); + ALOGI("\t\tTrackId: %lld, TrackState: %d", pstItems->nTrackId, pstItems->eTrackState); + + ALOGN("\t\tRect[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, + pstItems->stRect.fY, pstItems->stRect.fW, + pstItems->stRect.fH, pstItems->fConfidence); + StatTrackMgr(pstItems, &tObjectTrackNum); + StatPushMgr(pstItems, &tObjectPushNum); + + // get detect box only new or update state + if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW + || pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) + && nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { + Skels[nSkelSize].pstrObjectCategory = pstItems->pstrObjectCategory; + Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; + Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; + Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; + Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; + + OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", + i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, pstItems->stRect.fY, + pstItems->stRect.fW, pstItems->stRect.fH, + pstItems->fConfidence); + + ALOGN("\t\t[%d]Point Set Size: %d", i, pstItems->nPointSetSize); + + // point + Skels[nSkelSize].nPointNum = AX_MIN(DETECT_SKEL_POINT_COUNT, pstItems->nPointSetSize); + for (AX_U32 j = 0; j < Skels[i].nPointNum; j++) { + ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; + Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; + + OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", + j, + pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + } + + //face attr + if(nPPL == AX_SKEL_PPL_FACE){ + ALOGN("\t\tFace Attribute:"); + ALOGI("\t\t\tfYaw %f,",pstItems->stFaceAttr.fYaw); + ALOGI("\t\t\tfPitch %f,",pstItems->stFaceAttr.fPitch); + ALOGI("\t\t\tfRoll %f,",pstItems->stFaceAttr.fRoll); + ALOGI("\t\t\tfMask %f,",pstItems->stFaceAttr.fMask); + ALOGI("\t\t\tage %d",pstItems->stFaceAttr.nAge); + ALOGI("\t\t\tgender %d",pstItems->stFaceAttr.nGender); + } + nSkelSize ++; + } + + if (AX_SKEL_TRACK_STATUS_SELECT == pstItems->eTrackState) { + ATTRINFO_T AttrInfo; + memset(&AttrInfo, 0x00, sizeof(AttrInfo)); + AttrParser(pstItems, &AttrInfo); + + if (pstItems->bCropFrame + && pstItems->stCropFrame.pFrameData + && 0 < pstItems->stCropFrame.nFrameDataSize + && SaveResultPath) { + // save attribute + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, + AttrInfo.tFaceInfo.nAge, + AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[FACE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[PLATE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[VEHICLE ATTR] %s", arrDat); + } + else { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write crop jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + + } + + // panora frame + if (pstItems->bPanoraFrame + && pstItems->stPanoraFrame.pFrameData + && 0 < pstItems->stPanoraFrame.nFrameDataSize + && SaveResultPath) { + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write panora jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, pstItems->stPanoraFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + } + } + + // feature + ALOGI("\t\tFeature Size: %d", pstItems->nFeatureSize); + if (pstItems->pstFeatureItem + && 0 < pstItems->nFeatureSize + && SaveResultPath) { + AX_CHAR strFile[256] = {0}; + sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", + SaveResultPath, pstItems->pstrObjectCategory, + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, i); + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", + strFile, + pstItems->pstFeatureItem[0].nValueSize); + fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, pstItems->pstFeatureItem[0].nValueSize, fp_w); + fclose(fp_w); + } + } + + // meta + ALOGI("\t\tMeta Size: %d", pstItems->nMetaInfoSize); + if (pstItems->pstMetaInfo + && 0 < pstItems->nMetaInfoSize) { + for (AX_U32 i = 0; i < pstItems->nMetaInfoSize; i++) { + ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, pstItems->pstMetaInfo[i].pstrValue); + } + } + + // binds + ALOGI("\t\tBind Size: %d", pstItems->nObjectBindSize); + if (pstItems->pstObjectBind + && 0 < pstItems->nObjectBindSize) { + for (AX_U32 i = 0; i < pstItems->nObjectBindSize; i++) { + ALOGI("\t\t[%s] [TrackId] %lld bind to ObjectBind[%d]: %s [TrackId] %lld", + pstItems->pstrObjectCategory, pstItems->nTrackId, i, + pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); + } + } + } + + ALOGI("\tnCacheListSize: %d", pstResult->nCacheListSize); + if (pstResult->pstCacheList) { + for (AX_U32 i = 0; i < pstResult->nCacheListSize; i++) { + ALOGI("\t\tCacheList[%d] FrameId: %lld", i, pstResult->pstCacheList[i].nFrameId); + } + } + + if (ReWritePath && pstResult->nObjectSize > 0) { + memcpy(YUVDataWrVir, YUVDataVir, nFrameSize); + + YUV_IMAGE_T YUVImage = {0}; + YUVImage.pImage = YUVDataWrVir; + YUVImage.nWidth = nWidth; + YUVImage.nHeight = nHeight; + YUVImage.stride = nStride; + YUVImage.nSize = nFrameSize; + YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; + + for (size_t i = 0; i < nSkelSize; i++) { + // draw rect + AX_CHAR ObjectCategory = Skels[i].pstrObjectCategory[0]; + AX_S16 x0 = Skels[i].tBox.fX; + AX_S16 y0 = Skels[i].tBox.fY; + AX_U16 w = Skels[i].tBox.fW; + AX_U16 h = Skels[i].tBox.fH; + YUV_COLOR RectColor = YUV_WHITE; + + switch(ObjectCategory){ + case 'b': //body + RectColor = YUV_WHITE; + break; + case 'v': //vehicle + RectColor = YUV_PURPLE; + break; + case 'c': //cycle + RectColor = YUV_GREEN; + break; + case 'f': //face + RectColor = YUV_YELLOW; + break; + case 'p': //plate + RectColor = YUV_RED; + break; + default : + RectColor = YUV_WHITE; + break; + } + + DrawRect(&YUVImage, x0, y0, w, h, RectColor); + + if (Skels[i].nPointNum > 0) { + // draw point + for (size_t j = 0; j < Skels[i].nPointNum; j++) { + x0 = Skels[i].tPoint[j].fX; + y0 = Skels[i].tPoint[j].fY; + + DrawPoint(&YUVImage, x0, y0, 4, x0 * (4 - 1), y0 * (4 - 1), YUV_DARK_GREEN); + } + } + } + + AX_CHAR arrWrFile[512] = {0}; + sprintf(arrWrFile, "%s_result_%d_%d.jpg", InputFile, nFrameCnt + 1, nRepeat + 1); + + AX_CHAR *fileName = NULL; + AX_U32 fileNameLen = strlen(arrWrFile); + AX_CHAR fullFileName[256] = {0}; + AX_S32 i = fileNameLen - 1; + + if (fileNameLen > 0) { + for (; i >= 0; i --) { + if (arrWrFile[i] == '/') { + break; + } + } + } + + if (i < 0) { + i = 0; + } + else if (i < fileNameLen - 1) { + i ++; + } + + fileName = (AX_CHAR *)&arrWrFile[i]; + + sprintf(fullFileName, "%s/%s", ReWritePath, fileName); + + EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, nFrameSize); + } + + if (pstResult) { + AX_SKEL_Release((AX_VOID *)pstResult); + } + + OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process End (Elasped: %lld ms)", nFrameCnt + 1, nProcessElasped); + + if (nInterval > 0) { + usleep(nInterval); + } + } + + ALOGN("SKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + OUTPUT_LOG_SAVE("\nSKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + ALOGN("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + OUTPUT_LOG_SAVE("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]\n", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + if (InputFileHandle) { + ReLoadFile(InputFileHandle); + } + } + + // if(AX_SUCCESS != nRet){ + // ALOGE("AX_POOL_DestroyPool failed! Error Code:0x%X\n", nRet); + // } + + ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + + OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + +EXIT3: + if (pHandle) { + AX_SKEL_Destroy(pHandle); + } + +EXIT2: + AX_SKEL_DeInit(); + +EXIT1: + AX_ENGINE_Deinit(); + +EXIT0: + // if (YUVDataVir) { + // if (YUVDataPhy == 0) { + // free(YUVDataVir); + // } else { + // AX_SYS_MemFree(YUVDataPhy, YUVDataVir); + // } + // } + if (YUVDataWrVir) { + if (YUVDataWrPhy == 0) { + free(YUVDataWrVir); + } else { + AX_SYS_MemFree(YUVDataWrPhy, YUVDataWrVir); + } + } + + if (OneYUVDataVir) { + if (OneYUVDataPhy == 0) { + free(OneYUVDataVir); + } else { + AX_SYS_MemFree(OneYUVDataPhy, OneYUVDataVir); + } + } + + FrameMgrDestroy(); + + if (fpResultFile) { + fclose(fpResultFile); + } + + AX_VDEC_Deinit(); + + AX_VENC_Deinit(); + + AX_IVPS_Deinit(); + + AX_SYS_Deinit(); + + return (0 != nRet) ? -1 : 0; +} diff --git a/projects/llm_framework/main_skel/src/runner/skel_log.h b/projects/llm_framework/main_skel/src/runner/skel_log.h new file mode 100644 index 00000000..ffb495e6 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/skel_log.h @@ -0,0 +1,61 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _SAMPLE_SKEL_LOG_H_ +#define _SAMPLE_SKEL_LOG_H_ + +#include + +typedef enum { + SKEL_LOG_MIN = -1, + SKEL_LOG_EMERGENCY = 0, + SKEL_LOG_ALERT = 1, + SKEL_LOG_CRITICAL = 2, + SKEL_LOG_ERROR = 3, + SKEL_LOG_WARN = 4, + SKEL_LOG_NOTICE = 5, + SKEL_LOG_INFO = 6, + SKEL_LOG_DEBUG = 7, + SKEL_LOG_MAX +} SKEL_LOG_LEVEL_E; + +static SKEL_LOG_LEVEL_E log_level = SKEL_LOG_NOTICE; + +#if 1 +#define MACRO_BLACK "\033[1;30;30m" +#define MACRO_RED "\033[1;30;31m" +#define MACRO_GREEN "\033[1;30;32m" +#define MACRO_YELLOW "\033[1;30;33m" +#define MACRO_BLUE "\033[1;30;34m" +#define MACRO_PURPLE "\033[1;30;35m" +#define MACRO_WHITE "\033[1;30;37m" +#define MACRO_END "\033[0m" +#else +#define MACRO_BLACK +#define MACRO_RED +#define MACRO_GREEN +#define MACRO_YELLOW +#define MACRO_BLUE +#define MACRO_PURPLE +#define MACRO_WHITE +#define MACRO_END +#endif + +#define ALOGE(fmt, ...) printf(MACRO_RED "[E][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGW(fmt, ...) if (log_level >= SKEL_LOG_WARN) \ + printf(MACRO_YELLOW "[W][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGI(fmt, ...) if (log_level >= SKEL_LOG_INFO) \ + printf(MACRO_GREEN "[I][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGD(fmt, ...) if (log_level >= SKEL_LOG_DEBUG) \ + printf(MACRO_WHITE "[D][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGN(fmt, ...) if (log_level >= SKEL_LOG_NOTICE) \ + printf(MACRO_PURPLE "[N][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) + +#endif /* _SAMPLE_SKEL_TRACE_H_ */ diff --git a/projects/llm_framework/main_skel/src/runner/statMgr.cpp b/projects/llm_framework/main_skel/src/runner/statMgr.cpp new file mode 100644 index 00000000..c3a71010 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/statMgr.cpp @@ -0,0 +1,65 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include "statMgr.h" +#include + +AX_VOID StatTrackMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum) { + if (!pstObjectItems || !pObjectNum) { + return; + } + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (pstObjectItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW) { + if (strObjectCategory == "body") { + pObjectNum->nBodyNum ++; + } + else if (strObjectCategory == "vehicle") { + pObjectNum->nVehicleNum ++; + } + else if (strObjectCategory == "cycle") { + pObjectNum->nCycleNum ++; + } + else if (strObjectCategory == "face") { + pObjectNum->nFaceNum ++; + } + else if (strObjectCategory == "plate") { + pObjectNum->nPlateNum ++; + } + } +} + +AX_VOID StatPushMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum) { + if (!pstObjectItems || !pObjectNum) { + return; + } + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (pstObjectItems->eTrackState == AX_SKEL_TRACK_STATUS_SELECT) { + if (strObjectCategory == "body") { + pObjectNum->nBodyNum ++; + } + else if (strObjectCategory == "vehicle") { + pObjectNum->nVehicleNum ++; + } + else if (strObjectCategory == "cycle") { + pObjectNum->nCycleNum ++; + } + else if (strObjectCategory == "face") { + pObjectNum->nFaceNum ++; + } + else if (strObjectCategory == "plate") { + pObjectNum->nPlateNum ++; + } + } +} diff --git a/projects/llm_framework/main_skel/src/runner/statMgr.h b/projects/llm_framework/main_skel/src/runner/statMgr.h new file mode 100644 index 00000000..8cad1f60 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/statMgr.h @@ -0,0 +1,36 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _STAT_MGR_H_ +#define _STAT_MGR_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _STAT_OBJECT_NUM_T { + AX_U32 nBodyNum; + AX_U32 nVehicleNum; + AX_U32 nCycleNum; + AX_U32 nFaceNum; + AX_U32 nPlateNum; +} STAT_OBJECT_NUM_T; + +extern AX_VOID StatTrackMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum); +extern AX_VOID StatPushMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/llm_framework/main_sys/SConstruct b/projects/llm_framework/main_sys/SConstruct index 22513bcb..0808de47 100644 --- a/projects/llm_framework/main_sys/SConstruct +++ b/projects/llm_framework/main_sys/SConstruct @@ -8,7 +8,7 @@ with open(env['PROJECT_TOOL_S']) as f: SRCS = Glob('src/*.c*') INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] -REQUIREMENTS = ['hv', 'pthread', 'utilities', 'DeviceDriver', 'util', 'StackFlow'] +REQUIREMENTS = ['hv', 'pthread', 'utilities', 'DeviceDriver', 'util', 'StackFlow', 'simdjson_component'] STATIC_LIB = [] DYNAMIC_LIB = [] DEFINITIONS = [] @@ -32,7 +32,7 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] STATIC_FILES += [AFile('sys_config.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_sys', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_sys/include/event_loop.h b/projects/llm_framework/main_sys/include/event_loop.h index d2dbcc85..26d06101 100644 --- a/projects/llm_framework/main_sys/include/event_loop.h +++ b/projects/llm_framework/main_sys/include/event_loop.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #include #include diff --git a/projects/llm_framework/main_sys/include/remote_action.h b/projects/llm_framework/main_sys/include/remote_action.h index 3c894df9..3f4a8fe0 100644 --- a/projects/llm_framework/main_sys/include/remote_action.h +++ b/projects/llm_framework/main_sys/include/remote_action.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #if __cplusplus diff --git a/projects/llm_framework/main_sys/include/remote_server.h b/projects/llm_framework/main_sys/include/remote_server.h index f3c7b225..93f815b0 100644 --- a/projects/llm_framework/main_sys/include/remote_server.h +++ b/projects/llm_framework/main_sys/include/remote_server.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #if __cplusplus diff --git a/projects/llm_framework/main_sys/include/serial.h b/projects/llm_framework/main_sys/include/serial.h index 2ac12314..eb5a7f17 100644 --- a/projects/llm_framework/main_sys/include/serial.h +++ b/projects/llm_framework/main_sys/include/serial.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #include "all.h" diff --git a/projects/llm_framework/main_sys/include/zmq_bus.h b/projects/llm_framework/main_sys/include/zmq_bus.h index eb07db76..e32c7c63 100644 --- a/projects/llm_framework/main_sys/include/zmq_bus.h +++ b/projects/llm_framework/main_sys/include/zmq_bus.h @@ -21,10 +21,10 @@ void zmq_bus_stop_work(); using namespace StackFlows; class unit_data { - private: +private: std::unique_ptr user_inference_chennal_; - public: +public: std::string work_id; std::string input_url; std::string output_url; @@ -41,23 +41,31 @@ int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_s void zmq_com_send(int com_id, const std::string &out_str); class zmq_bus_com { - protected: +private: + int reace_event_; + int raw_msg_len_; + std::string raw_msg_buff_; + +protected: std::string _zmq_url; int exit_flage; int err_count; int _port; + std::string json_str_; + int json_str_flage_; std::unique_ptr reace_data_event_thread; - public: +public: std::unique_ptr user_chennal_; zmq_bus_com(); void work(const std::string &zmq_url_format, int port); void stop(); + void select_json_str(const std::string &json_src, std::function out_fun); virtual void on_data(const std::string &data); + virtual void on_raw_data(const std::string &data); virtual void send_data(const std::string &data); virtual void reace_data_event(); virtual void send_data_event(); ~zmq_bus_com(); }; - #endif \ No newline at end of file diff --git a/projects/llm_framework/main_sys/src/config.cpp b/projects/llm_framework/main_sys/src/config.cpp index d3ef2ee6..457eb984 100644 --- a/projects/llm_framework/main_sys/src/config.cpp +++ b/projects/llm_framework/main_sys/src/config.cpp @@ -1,15 +1,16 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #include #include #include "all.h" #include "json.hpp" -void load_default_config() { +void load_default_config() +{ std::ifstream file("./sys_config.json"); if (!file.is_open()) { return; diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 32d26dd0..0ac9c037 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -34,9 +34,12 @@ #include "subprocess.h" #include "zmq_bus.h" #include "remote_action.h" +#include +#include "hv/ifconfig.h" void usr_print_error(const std::string &request_id, const std::string &work_id, const std::string &error_msg, - int zmq_out) { + int zmq_out) +{ nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -49,7 +52,8 @@ void usr_print_error(const std::string &request_id, const std::string &work_id, } template -void usr_out(const std::string &request_id, const std::string &work_id, const T &data, int zmq_out) { +void usr_out(const std::string &request_id, const std::string &work_id, const T &data, int zmq_out) +{ nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -65,13 +69,15 @@ void usr_out(const std::string &request_id, const std::string &work_id, const T zmq_com_send(zmq_out, out); } -int sys_ping(int com_id, const nlohmann::json &json_obj) { +int sys_ping(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"\"}", com_id); return out; } -void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) { +void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) +{ SAFE_SETTING("config_serial_baud", (int)json_obj["data"]["baud"]); SAFE_SETTING("config_serial_data_bits", (int)json_obj["data"]["data_bits"]); SAFE_SETTING("config_serial_stop_bits", (int)json_obj["data"]["stop_bits"]); @@ -82,14 +88,16 @@ void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) { serial_work(); } -int sys_uartsetup(int com_id, const nlohmann::json &json_obj) { +int sys_uartsetup(int com_id, const nlohmann::json &json_obj) +{ int out = 0; std::thread t(_sys_uartsetup, com_id, json_obj); t.detach(); return out; } -void get_memory_info(unsigned long *total_memory, unsigned long *free_memory) { +void get_memory_info(unsigned long *total_memory, unsigned long *free_memory) +{ FILE *meminfo = fopen("/proc/meminfo", "r"); if (meminfo == NULL) { perror("fopen"); @@ -113,7 +121,8 @@ struct cpu_use_t { long time; }; -int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) { +int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) +{ unsigned long one; unsigned long two; unsigned long temp; @@ -148,20 +157,42 @@ int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) { get_memory_info(&one, &two); float use = ((one * 1.0f) - (two * 1.0f)) / (one * 1.0f); data_body["mem"] = (int)(use * 100); - out_body["data"] = data_body; - std::string out = out_body.dump(); + std::vector ifcs; + ifconfig(ifcs); + std::vector jifcs; + for (auto &item : ifcs) { + nlohmann::json eth_info; + eth_info["name"] = item.name; + eth_info["ip"] = item.ip; + char eth_ip_buff[128] = {0}; + sprintf(eth_ip_buff, "/sys/class/net/%s/speed", item.name); + FILE *file = fopen(eth_ip_buff, "r"); + memset(eth_ip_buff, 0, sizeof(eth_ip_buff)); + if (file != NULL) { + int size = fread(eth_ip_buff, 1, sizeof(eth_ip_buff), file); + if (size > 0) eth_ip_buff[size - 1] = '\0'; + fclose(file); + } + eth_info["speed"] = eth_ip_buff; + jifcs.push_back(eth_info); + } + data_body["eth_info"] = jifcs; + out_body["data"] = data_body; + std::string out = out_body.dump(); zmq_com_send(com_id, out); return 0; } -int sys_hwinfo(int com_id, const nlohmann::json &json_obj) { +int sys_hwinfo(int com_id, const nlohmann::json &json_obj) +{ int out = 0; std::thread t(_sys_hwinfo, com_id, json_obj); t.detach(); return out; } -int sys_lsmode(int com_id, const nlohmann::json &json_obj) { +int sys_lsmode(int com_id, const nlohmann::json &json_obj) +{ int out; nlohmann::json out_body; int stream = false; @@ -220,7 +251,17 @@ int sys_lsmode(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_lstask(int com_id, const nlohmann::json &json_obj) { +int sys_rmmode(int com_id, const nlohmann::json &json_obj) +{ + std::string rmmode_name = json_obj["data"]; + const std::string command = "dpkg -P llm-" + rmmode_name; + int out = system(command.c_str()); + usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"\"}", com_id); + return out; +} + +int sys_lstask(int com_id, const nlohmann::json &json_obj) +{ int out; sys_lstask_err_1: @@ -229,7 +270,8 @@ int sys_lstask(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_push(int com_id, const nlohmann::json &json_obj) { +int sys_push(int com_id, const nlohmann::json &json_obj) +{ int out; nlohmann::json out_body; out_body["request_id"] = json_obj["request_id"]; @@ -279,7 +321,8 @@ int sys_push(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_pull(int com_id, const nlohmann::json &json_obj) { +int sys_pull(int com_id, const nlohmann::json &json_obj) +{ int out; sys_pull_err_1: @@ -288,7 +331,8 @@ int sys_pull(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_update(int com_id, const nlohmann::json &json_obj) { +int sys_update(int com_id, const nlohmann::json &json_obj) +{ int out; const std::string command = "find /mnt -name \"llm_update_*.deb\" >> /tmp/find_update_file_out.txt 2>&1 "; out = system(command.c_str()); @@ -311,7 +355,9 @@ int sys_update(int com_id, const nlohmann::json &json_obj) { "{\"code\":-10, \"message\":\"Not available at the moment.\"}", com_id); return out; } -int sys_upgrade(int com_id, const nlohmann::json &json_obj) { + +int sys_upgrade(int com_id, const nlohmann::json &json_obj) +{ int out; std::string version; try { @@ -336,8 +382,8 @@ int sys_upgrade(int com_id, const nlohmann::json &json_obj) { return out; } -int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std::string bashcmd, int stream, - int base64) { +int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std::string bashcmd, int stream, int base64) +{ int out; int stream_size = 512; /****************************************************/ @@ -404,7 +450,8 @@ int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std:: return out; } -int sys_bashexec(int com_id, const nlohmann::json &json_obj) { +int sys_bashexec(int com_id, const nlohmann::json &json_obj) +{ std::string request_id = json_obj["request_id"]; std::string work_id = json_obj["work_id"]; std::string bashcmd; @@ -438,24 +485,27 @@ int sys_bashexec(int com_id, const nlohmann::json &json_obj) { return 0; } -int sys_reset(int com_id, const nlohmann::json &json_obj) { +int sys_reset(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"llm server restarting ...\"}", com_id); const char *cmd = - "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm/reset.lock ; sync ; sleep 1 ; systemctl restart " - "llm-* \" > /dev/null 2>&1 & "; + "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm_reset.lock ; sync ; " + "systemctl restart llm-* \" > /dev/null 2>&1 & "; system(cmd); return out; } -int sys_version(int com_id, const nlohmann::json &json_obj) { - usr_out(json_obj["request_id"], json_obj["work_id"], std::string("v1.2"), com_id); +int sys_version(int com_id, const nlohmann::json &json_obj) +{ + usr_out(json_obj["request_id"], json_obj["work_id"], std::string("v1.3"), com_id); int out = 0; return out; } -int sys_reboot(int com_id, const nlohmann::json &json_obj) { +int sys_reboot(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"rebooting ...\"}", com_id); usleep(200000); @@ -463,7 +513,8 @@ int sys_reboot(int com_id, const nlohmann::json &json_obj) { return out; } -void server_work() { +void server_work() +{ key_sql["sys.ping"] = sys_ping; key_sql["sys.lsmode"] = sys_lsmode; key_sql["sys.lstask"] = sys_lstask; @@ -476,31 +527,50 @@ void server_work() { key_sql["sys.uartsetup"] = sys_uartsetup; key_sql["sys.reset"] = sys_reset; key_sql["sys.reboot"] = sys_reboot; - key_sql["sys.version"] = sys_version; + key_sql["sys.version"] = sys_version; + key_sql["sys.rmmode"] = sys_rmmode; } -void server_stop_work() { +void server_stop_work() +{ } - +std::mutex unit_action_match_mtx; +simdjson::ondemand::parser parser; typedef int (*sys_fun_call)(int, const nlohmann::json &); -void unit_action_match(int com_id, const std::string &json_str) { - int ret; - std::string out_str; - nlohmann::json req_body; - - std::string action; +void unit_action_match(int com_id, const std::string &json_str) +{ + std::lock_guard guard(unit_action_match_mtx); + simdjson::padded_string json_string(json_str); + simdjson::ondemand::document doc; + auto error = parser.iterate(json_string).get(doc); + if (error) { + SLOGE("json format error:%s", json_str.c_str()); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } std::string request_id; + error = doc["request_id"].get_string(request_id); + if (error) { + SLOGE("miss request_id, error:%s", simdjson::error_message(error)); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } std::string work_id; - try { - req_body = nlohmann::json::parse(json_str); - action = req_body["action"]; - request_id = req_body["request_id"]; - work_id = req_body["work_id"]; - } catch (...) { - SLOGE("json format error:%s", json_str.c_str()); + error = doc["work_id"].get_string(work_id); + if (error) { + SLOGE("miss work_id, error:%s", simdjson::error_message(error)); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } + if (work_id.empty()) work_id = "sys"; + std::string action; + error = doc["action"].get_string(action); + if (error) { + SLOGE("miss action, error:%s", simdjson::error_message(error)); usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } + // SLOGI("request_id:%s work_id:%s action:%s", request_id.c_str(), work_id.c_str(), action.c_str()); std::vector work_id_fragment; std::string fragment; for (auto c : work_id) { @@ -512,23 +582,30 @@ void unit_action_match(int com_id, const std::string &json_str) { } } if (fragment.length()) work_id_fragment.push_back(fragment); - - if ((work_id_fragment.size() > 0) && (work_id_fragment[0] == "sys")) { - std::string unit_action = "sys." + action; - sys_fun_call call_fun = NULL; - SAFE_READING(call_fun, sys_fun_call, unit_action); - if (call_fun) - call_fun(com_id, req_body); - else - usr_print_error(request_id, work_id, "{\"code\":-3, \"message\":\"action match false\"}", com_id); - } else if (action == "inference") { + if (action == "inference") { char zmq_push_url[128]; - sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); - req_body["zmq_com"] = std::string(zmq_push_url); - int ret = zmq_bus_publisher_push(work_id, req_body.dump()); + int post = sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); + std::string inference_raw_data; + inference_raw_data.resize(post + json_str.length() + 13); + post = sprintf(inference_raw_data.data(), "{\"zmq_com\":\""); + post += sprintf(inference_raw_data.data() + post, "%s", zmq_push_url); + post += sprintf(inference_raw_data.data() + post, "\","); + memcpy(inference_raw_data.data() + post, json_str.data() + 1, json_str.length() - 1); + // post += json_str.length() - 1; + // SLOGI("inference_raw_data:%s size:%d %d", inference_raw_data.c_str(), inference_raw_data.length(), post); + int ret = zmq_bus_publisher_push(work_id, inference_raw_data); if (ret) { usr_print_error(request_id, work_id, "{\"code\":-4, \"message\":\"inference data push false\"}", com_id); } + } else if ((work_id_fragment.size() > 0) && (work_id_fragment[0] == "sys")) { + std::string unit_action = "sys." + action; + sys_fun_call call_fun = NULL; + SAFE_READING(call_fun, sys_fun_call, unit_action); + if (call_fun) { + call_fun(com_id, nlohmann::json::parse(json_str)); + } else { + usr_print_error(request_id, work_id, "{\"code\":-3, \"message\":\"action match false\"}", com_id); + } } else { if ((work_id_fragment[0].length() != 0) && (remote_call(com_id, json_str) != 0)) { usr_print_error(request_id, work_id, "{\"code\":-9, \"message\":\"unit call false\"}", com_id); diff --git a/projects/llm_framework/main_sys/src/main.cpp b/projects/llm_framework/main_sys/src/main.cpp index d48e9a95..7afbef6e 100644 --- a/projects/llm_framework/main_sys/src/main.cpp +++ b/projects/llm_framework/main_sys/src/main.cpp @@ -31,12 +31,14 @@ std::string zmq_s_format; std::string zmq_c_format; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ printf("llm_sys will be exit!\n"); main_exit_flage = 1; } -void get_run_config() { +void get_run_config() +{ key_sql["config_serial_dev"] = std::string("/dev/ttyS1"); key_sql["config_serial_baud"] = 115200; key_sql["config_serial_data_bits"] = 8; @@ -50,6 +52,7 @@ void get_run_config() { key_sql["config_zmq_s_format"] = std::string("ipc:///tmp/llm/%i.sock"); key_sql["config_zmq_c_format"] = std::string("ipc:///tmp/llm/%i.sock"); key_sql["config_lsmod_dir"] = std::string("/opt/m5stack/data/models/"); + key_sql["config_model_d"] = std::string("/opt/m5stack/data/models/"); key_sql["config_base_mode_path"] = std::string("/opt/m5stack/data/"); key_sql["config_base_mode_config_path"] = std::string("/opt/m5stack/etc/"); key_sql["config_tcp_server"] = 10001; @@ -72,7 +75,8 @@ void get_run_config() { reset_init = 1; \ } -void uart_reset_check() { +void uart_reset_check() +{ uart_t uart_parm; uart_parm.baud = any_cast(key_sql["config_serial_baud"]); uart_parm.data_bits = any_cast(key_sql["config_serial_data_bits"]); @@ -126,7 +130,8 @@ void tcp_work(); void tcp_stop_work(); -void all_work() { +void all_work() +{ zmq_s_format = any_cast(key_sql["config_zmq_s_format"]); zmq_c_format = any_cast(key_sql["config_zmq_c_format"]); server_work(); @@ -139,7 +144,8 @@ void all_work() { if (enable_tcp) tcp_work(); } -void all_stop_work() { +void all_stop_work() +{ int enable_tcp = 0; SAFE_READING(enable_tcp, int, "config_enable_tcp"); @@ -150,10 +156,12 @@ void all_stop_work() { zmq_bus_stop_work(); } -void all_work_check() { +void all_work_check() +{ } -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_sys/src/remote_action.cpp b/projects/llm_framework/main_sys/src/remote_action.cpp index 56854e2d..25933692 100644 --- a/projects/llm_framework/main_sys/src/remote_action.cpp +++ b/projects/llm_framework/main_sys/src/remote_action.cpp @@ -14,24 +14,25 @@ using namespace StackFlows; int unit_call_timeout; -int remote_call(int com_id, const std::string &json_str) { +int remote_call(int com_id, const std::string &json_str) +{ std::string work_id = sample_json_str_get(json_str, "work_id"); std::string work_unit = work_id.substr(0, work_id.find(".")); std::string action = sample_json_str_get(json_str, "action"); - char com_url[128]; - sprintf(com_url, zmq_c_format.c_str(), com_id); - std::string send_data = "{\"zmq_com\":\""; - send_data += std::string(com_url); - send_data += "\",\"raw_data\":\""; - send_data += sample_escapeString(json_str); - send_data += "\"}"; + char com_url[256]; + int length = snprintf(com_url, 255, zmq_c_format.c_str(), com_id); + std::string send_data; + std::string com_urls(com_url); + RPC_PUSH_PARAM(send_data, com_urls, json_str); pzmq clent(work_unit); - return clent.call_rpc_action(action, send_data, [](const std::string &val) {}); + return clent.call_rpc_action(action, send_data, [](pzmq *_pzmq, const std::string &val) {}); } -void remote_action_work() { +void remote_action_work() +{ SAFE_READING(unit_call_timeout, int, "config_unit_call_timeout"); } -void remote_action_stop_work() { +void remote_action_stop_work() +{ } diff --git a/projects/llm_framework/main_sys/src/remote_server.cpp b/projects/llm_framework/main_sys/src/remote_server.cpp index 65c201b9..5a4567f7 100644 --- a/projects/llm_framework/main_sys/src/remote_server.cpp +++ b/projects/llm_framework/main_sys/src/remote_server.cpp @@ -23,21 +23,25 @@ int port_list_start; std::vector port_list; std::unique_ptr sys_rpc_server_; -std::string sys_sql_select(const std::string &key) { +std::string sys_sql_select(const std::string &key) +{ std::string out; SAFE_READING(out, std::string, key); return out; } -void sys_sql_set(const std::string &key, const std::string &val) { +void sys_sql_set(const std::string &key, const std::string &val) +{ SAFE_SETTING(key, val); } -void sys_sql_unset(const std::string &key) { +void sys_sql_unset(const std::string &key) +{ SAFE_ERASE(key); } -unit_data *sys_allocate_unit(const std::string &unit) { +unit_data *sys_allocate_unit(const std::string &unit) +{ unit_data *unit_p = new unit_data(); { unit_p->port_ = work_id_number_counter++; @@ -112,7 +116,8 @@ unit_data *sys_allocate_unit(const std::string &unit) { return unit_p; } -int sys_release_unit(const std::string &unit) { +int sys_release_unit(const std::string &unit) +{ unit_data *unit_p = NULL; SAFE_READING(unit_p, unit_data *, unit); if (NULL == unit_p) { @@ -133,21 +138,25 @@ int sys_release_unit(const std::string &unit) { return 0; } -char const *c_sys_sql_select(char const *key) { +char const *c_sys_sql_select(char const *key) +{ static std::string val; val = sys_sql_select(key); return val.c_str(); } -void c_sys_sql_set(char const *key, char const *val) { +void c_sys_sql_set(char const *key, char const *val) +{ sys_sql_set(key, val); } -void c_sys_sql_unset(char const *key) { +void c_sys_sql_unset(char const *key) +{ sys_sql_unset(key); } -void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, int *work_id) { +void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, int *work_id) +{ unit_data *unit_p = sys_allocate_unit(unit); strcpy(input_url, unit_p->inference_url.c_str()); strcpy(output_url, unit_p->output_url.c_str()); @@ -157,32 +166,35 @@ void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, in sscanf(unit_p->work_id.c_str(), work_id_format.c_str(), work_id); } -int c_sys_release_unit(char const *unit) { +int c_sys_release_unit(char const *unit) +{ return sys_release_unit(unit); } -std::string rpc_sql_select(const std::string &raw) { - return sys_sql_select(raw); -} - -std::string rpc_allocate_unit(const std::string &raw) { +std::string rpc_allocate_unit(pzmq *_pzmq, const std::string &raw) +{ unit_data *unit_info = sys_allocate_unit(raw); - std::string retval = "{\"work_id_number\":"; - retval += std::to_string(unit_info->port_); - retval += ",\"out_port\":\""; - retval += unit_info->output_url; - retval += "\",\"inference_port\":\""; - retval += unit_info->inference_url; - retval += "\"}"; - return retval; + std::string send_data; + std::string send_data1; + std::string str_port = std::to_string(unit_info->port_); + RPC_PUSH_PARAM(send_data1, unit_info->output_url, unit_info->inference_url); + RPC_PUSH_PARAM(send_data, str_port, send_data1); + return send_data; } -std::string rpc_release_unit(const std::string &raw) { +std::string rpc_release_unit(pzmq *_pzmq, const std::string &raw) +{ sys_release_unit(raw); return "Success"; } -std::string rpc_sql_set(const std::string &raw) { +std::string rpc_sql_select(pzmq *_pzmq, const std::string &raw) +{ + return sys_sql_select(raw); +} + +std::string rpc_sql_set(pzmq *_pzmq, const std::string &raw) +{ std::string key = sample_json_str_get(raw, "key"); std::string val = sample_json_str_get(raw, "val"); if (key.empty()) return "False"; @@ -190,12 +202,14 @@ std::string rpc_sql_set(const std::string &raw) { return "Success"; } -std::string rpc_sql_unset(const std::string &raw) { +std::string rpc_sql_unset(pzmq *_pzmq, const std::string &raw) +{ sys_sql_unset(raw); return "Success"; } -void remote_server_work() { +void remote_server_work() +{ int port_list_end; SAFE_READING(work_id_number_counter, int, "config_work_id"); SAFE_READING(port_list_start, int, "config_zmq_min_port"); @@ -203,13 +217,19 @@ void remote_server_work() { port_list.resize(port_list_end - port_list_start, 0); sys_rpc_server_ = std::make_unique("sys"); - sys_rpc_server_->register_rpc_action("sql_select", std::bind(rpc_sql_select, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("register_unit", std::bind(rpc_allocate_unit, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("release_unit", std::bind(rpc_release_unit, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("sql_set", std::bind(rpc_sql_set, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("sql_unset", std::bind(rpc_sql_unset, std::placeholders::_1)); -} - -void remote_server_stop_work() { + sys_rpc_server_->register_rpc_action("sql_select", + std::bind(rpc_sql_select, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("register_unit", + std::bind(rpc_allocate_unit, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("release_unit", + std::bind(rpc_release_unit, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("sql_set", + std::bind(rpc_sql_set, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("sql_unset", + std::bind(rpc_sql_unset, std::placeholders::_1, std::placeholders::_2)); +} + +void remote_server_stop_work() +{ sys_rpc_server_.reset(); } \ No newline at end of file diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 74a4164a..6dc46086 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -26,11 +26,12 @@ #include class serial_com : public zmq_bus_com { - private: +private: int uart_fd; - public: - serial_com(uart_t *uart_parm, const std::string &dev_name) : zmq_bus_com() { +public: + serial_com(uart_t *uart_parm, const std::string &dev_name) : zmq_bus_com() + { uart_fd = linux_uart_init((char *)dev_name.c_str(), uart_parm); if (uart_fd <= 0) { SLOGE("open %s false!", dev_name.c_str()); @@ -38,57 +39,45 @@ class serial_com : public zmq_bus_com { } } - void send_data(const std::string &data) { - SLOGD("serial send:%s", data.c_str()); - linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); + void send_data(const std::string &data) + { + if (exit_flage) linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); } - void reace_data_event() { - std::string json_str; - int flage = 0; + void reace_data_event() + { fd_set readfds; + std::vector buff(1024); while (exit_flage) { - char reace_buf[128] = {0}; FD_ZERO(&readfds); FD_SET(uart_fd, &readfds); struct timeval timeout = {0, 500000}; if ((select(uart_fd + 1, &readfds, NULL, NULL, &timeout) <= 0) || (!FD_ISSET(uart_fd, &readfds))) continue; - int len = linux_uart_read(uart_fd, 128, reace_buf); + ssize_t len = linux_uart_read(uart_fd, buff.size(), buff.data()); + if (len <= 0) continue; { - char *data = reace_buf; - for (int i = 0; i < len; i++) { - json_str += data[i]; - if (data[i] == '{') flage++; - if (data[i] == '}') flage--; - if (flage == 0) { - if (json_str[0] == '{') { - SLOGD("uart reace:%s", json_str.c_str()); - on_data(json_str); - } - json_str.clear(); - } - if (flage < 0) { - flage = 0; - json_str.clear(); - { - std::string out_str; - out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; - out_str += std::to_string(time(NULL)); - out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; - send_data(out_str); - } - } + try { + select_json_str(std::string(buff.data(), len), + std::bind(&serial_com::on_data, this, std::placeholders::_1)); + } catch (...) { + std::string out_str; + out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; + out_str += std::to_string(time(NULL)); + out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; + send_data(out_str); } } } } - void stop() { + void stop() + { zmq_bus_com::stop(); linux_uart_deinit(uart_fd); } - - ~serial_com() { + + ~serial_com() + { if (exit_flage) { stop(); } @@ -96,7 +85,8 @@ class serial_com : public zmq_bus_com { }; std::unique_ptr serial_con_; -void serial_work() { +void serial_work() +{ uart_t uart_parm; std::string dev_name; int port; @@ -120,8 +110,8 @@ void serial_work() { out_str += ",\"error\":{\"code\":0, \"message\":\"upgrade over\"}}"; serial_con_->send_data(out_str); } - if (access("/tmp/llm/reset.lock", F_OK) == 0) { - remove("/tmp/llm/reset.lock"); + if (access("/tmp/llm_reset.lock", F_OK) == 0) { + remove("/tmp/llm_reset.lock"); sync(); std::string out_str; out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; @@ -132,6 +122,7 @@ void serial_work() { sync(); } -void serial_stop_work() { +void serial_stop_work() +{ serial_con_.reset(); } diff --git a/projects/llm_framework/main_sys/src/tcp_com.cpp b/projects/llm_framework/main_sys/src/tcp_com.cpp index 16115ea5..552a4a78 100644 --- a/projects/llm_framework/main_sys/src/tcp_com.cpp +++ b/projects/llm_framework/main_sys/src/tcp_com.cpp @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #include "all.h" #include "hv/TcpServer.h" #include @@ -31,72 +31,69 @@ std::atomic counter_port(8000); TcpServer srv; class tcp_com : public zmq_bus_com { - private: - public: - SocketChannelPtr channel; - std::string json_str; - int flage; +private: +public: + std::weak_ptr channel; std::mutex tcp_server_mutex; - tcp_com() : zmq_bus_com() { - flage = 0; + tcp_com() : zmq_bus_com() + { } - void send_data(const std::string& data) { - tcp_server_mutex.lock(); - if (exit_flage) channel->write(data); - tcp_server_mutex.unlock(); + void send_data(const std::string& data) + { + // tcp_server_mutex.lock(); + // if (exit_flage) + // { + // auto tcp_channel = channel.lock(); + // if(tcp_channel) + // { + // tcp_channel->write(data); + // } + // } + // tcp_server_mutex.unlock(); + auto tcp_channel = channel.lock(); + if (tcp_channel) { + tcp_channel->write(data); + } } }; -void onConnection(const SocketChannelPtr& channel) { - std::string peeraddr = channel->peeraddr(); - tcp_com* con_data; +void onConnection(const SocketChannelPtr& channel) +{ if (channel->isConnected()) { - con_data = new tcp_com(); - con_data->channel = channel; - con_data->work(zmq_s_format, counter_port.fetch_add(1)); - if(counter_port.load() > 65535) counter_port.store(8000); - channel->setContext(con_data); + auto p_com = channel->newContextPtr(); + p_com->channel = channel; + p_com->work(zmq_s_format, counter_port.fetch_add(1)); + if (counter_port.load() > 65535) counter_port.store(8000); } else { - con_data = (tcp_com*)channel->context(); - con_data->tcp_server_mutex.lock(); - con_data->stop(); - con_data->tcp_server_mutex.unlock(); - delete con_data; + auto p_com = channel->getContextPtr(); + // p_com->tcp_server_mutex.lock(); + p_com->stop(); + // p_com->tcp_server_mutex.unlock(); + channel->deleteContextPtr(); } } -void onMessage(const SocketChannelPtr& channel, Buffer* buf) { - int len = (int)buf->size(); - char* data = (char*)buf->data(); - tcp_com* con_data = (tcp_com*)channel->context(); - con_data->tcp_server_mutex.lock(); - for (size_t i = 0; i < len; i++) { - con_data->json_str += data[i]; - if (data[i] == '{') con_data->flage++; - if (data[i] == '}') con_data->flage--; - if (con_data->flage == 0) { - if (con_data->json_str[0] == '{') { - con_data->on_data(con_data->json_str); - } - con_data->json_str.clear(); - } - if (con_data->flage < 0) { - con_data->flage = 0; - con_data->json_str.clear(); - { - std::string out_str; - out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; - out_str += std::to_string(time(NULL)); - out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; - channel->write(out_str); - } - } +void onMessage(const SocketChannelPtr& channel, Buffer* buf) +{ + int len = (int)buf->size(); + char* data = (char*)buf->data(); + auto p_com = channel->getContextPtr(); + p_com->tcp_server_mutex.lock(); + try { + p_com->select_json_str(std::string(data, len), std::bind(&tcp_com::on_data, p_com, std::placeholders::_1)); + } catch (...) { + std::string out_str; + out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; + out_str += std::to_string(time(NULL)); + out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; + channel->write(out_str); } - con_data->tcp_server_mutex.unlock(); + p_com->tcp_server_mutex.unlock(); } -void tcp_work() { +void tcp_work() +{ int listenport = 0; SAFE_READING(listenport, int, "config_tcp_server"); @@ -107,9 +104,10 @@ void tcp_work() { printf("server listen on port %d, listenfd=%d ...\n", listenport, listenfd); srv.onConnection = onConnection; srv.onMessage = onMessage; - srv.setThreadNum(1); + srv.setThreadNum(2); srv.start(); } -void tcp_stop_work() { +void tcp_stop_work() +{ srv.stop(); } diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index bd29328d..a9fb7b85 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -10,17 +10,26 @@ #include #include #include +#include +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif using namespace StackFlows; void unit_action_match(int com_id, const std::string &json_str); -zmq_bus_com::zmq_bus_com() { - exit_flage = 1; - err_count = 0; +zmq_bus_com::zmq_bus_com() +{ + exit_flage = 1; + err_count = 0; + json_str_flage_ = 0; + reace_event_ = 0; + raw_msg_len_ = 0; } -void zmq_bus_com::work(const std::string &zmq_url_format, int port) { +void zmq_bus_com::work(const std::string &zmq_url_format, int port) +{ _port = port; exit_flage = 1; std::string ports = std::to_string(port); @@ -28,59 +37,105 @@ void zmq_bus_com::work(const std::string &zmq_url_format, int port) { sprintf((char *)buff.data(), zmq_url_format.c_str(), port); _zmq_url = std::string((char *)buff.data()); SAFE_SETTING("serial_zmq_url", _zmq_url); - user_chennal_ = - std::make_unique(_zmq_url, ZMQ_PULL, std::bind(&zmq_bus_com::send_data, this, std::placeholders::_1)); + user_chennal_ = std::make_unique(_zmq_url, ZMQ_PULL, + [this](pzmq *_pzmq, const std::string &data) { this->send_data(data); }); reace_data_event_thread = std::make_unique(std::bind(&zmq_bus_com::reace_data_event, this)); } -void zmq_bus_com::stop() { +void zmq_bus_com::stop() +{ exit_flage = 0; user_chennal_.reset(); reace_data_event_thread->join(); } -void zmq_bus_com::on_data(const std::string &data) { +void zmq_bus_com::on_data(const std::string &data) +{ unit_action_match(_port, data); } -void zmq_bus_com::send_data(const std::string &data) { +void zmq_bus_com::on_raw_data(const std::string &data) +{ + std::string base64_data; + int ret = StackFlows::encode_base64(data, base64_data); + ssize_t pos = json_str_.find("\"data\""); + if (pos != std::string::npos) { + ssize_t end_pos = pos + 6; + int status = 0; + for (ssize_t end_pos = pos + 6; end_pos < json_str_.length(); end_pos++) { + if ((json_str_[end_pos] == '\"') || (json_str_[end_pos] == ',') || (json_str_[end_pos] == '}')) { + status++; + } + if (status == 3) { + break; + } + } + json_str_.erase(pos, end_pos - pos); + } + std::string new_data; + new_data.resize(json_str_.length() + 10 + base64_data.length()); + memcpy((void *)new_data.data(), json_str_.c_str(), json_str_.length()); + sprintf((char *)(new_data.data() + json_str_.length() - 1), ",\"data\":\""); + memcpy((void *)(new_data.data() + json_str_.length() + 8), base64_data.c_str(), base64_data.length()); + new_data[json_str_.length() + 8 + base64_data.length()] = '"'; + new_data[json_str_.length() + 9 + base64_data.length()] = '}'; + on_data(new_data); +} + +void zmq_bus_com::send_data(const std::string &data) +{ // printf("zmq_bus_com::send_data : send:%s\n", data.c_str()); } -void zmq_bus_com::reace_data_event() { +void zmq_bus_com::reace_data_event() +{ // while (exit_flage) // { // std::this_thread::sleep_for(std::chrono::seconds(1)); // } } -void zmq_bus_com::send_data_event() { +void zmq_bus_com::send_data_event() +{ // while (exit_flage) // { // std::this_thread::sleep_for(std::chrono::seconds(1)); // } } -zmq_bus_com::~zmq_bus_com() { +zmq_bus_com::~zmq_bus_com() +{ + if (exit_flage) { + stop(); + } } -unit_data::unit_data() { +unit_data::unit_data() +{ } -void unit_data::init_zmq(const std::string &url) { +void unit_data::init_zmq(const std::string &url) +{ inference_url = url; user_inference_chennal_ = std::make_unique(inference_url, ZMQ_PUB); } -void unit_data::send_msg(const std::string &json_str) { +void unit_data::send_msg(const std::string &json_str) +{ user_inference_chennal_->send_data(json_str); } -unit_data::~unit_data() { +unit_data::~unit_data() +{ user_inference_chennal_.reset(); } -int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_str) { +int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_str) +{ + if (work_id.empty()) { + SLOGW("work_id is empty"); + return -1; + } unit_data *unit_p = NULL; SAFE_READING(unit_p, unit_data *, work_id); if (unit_p) @@ -94,15 +149,98 @@ int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_s void *usr_context; -void zmq_com_send(int com_id, const std::string &out_str) { +void zmq_com_send(int com_id, const std::string &out_str) +{ char zmq_push_url[128]; sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); pzmq _zmq(zmq_push_url, ZMQ_PUSH); - _zmq.send_data(out_str); + std::string out = out_str + "\n"; + _zmq.send_data(out); +} + +void zmq_bus_work() +{ } -void zmq_bus_work() { +void zmq_bus_stop_work() +{ } -void zmq_bus_stop_work() { +void zmq_bus_com::select_json_str(const std::string &json_src, std::function out_fun) +{ + bool enloop = false; + std::shared_ptr src_str((std::string *)&json_src, [](std::string *pt) {}); + do { + enloop = false; + switch (reace_event_) { + case 0: { + json_str_.reserve(json_str_.length() + src_str->length()); + const char *data = src_str->c_str(); + for (int i = 0; i < src_str->length(); i++) { +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + if (src_str->length() - i >= 16) { + uint8x16_t target_open = vdupq_n_u8('{'); + uint8x16_t target_close = vdupq_n_u8('}'); + uint8x16_t input_vector = vld1q_u8((const uint8_t *)&data[i]); + uint8x16_t result_open = vceqq_u8(input_vector, target_open); + uint8x16_t result_close = vceqq_u8(input_vector, target_close); + uint8x16_t result_mask = vorrq_u8(result_open, result_close); + __uint128_t jflage; + vst1q_u8((uint8_t *)&jflage, result_mask); + if (jflage == 0) { + json_str_.append(data + i, 16); + i += 15; + continue; + } + } +#endif + json_str_ += data[i]; + int last_index = (i == 0) ? 0 : (i - 1); + if ((data[i] == '{') && (data[last_index] != '\\')) json_str_flage_++; + if ((data[i] == '}') && (data[last_index] != '\\')) json_str_flage_--; + if (json_str_flage_ == 0) { + if ((json_str_[0] == '{') && (json_str_[json_str_.length() - 1] == '}')) { + if ((json_str_.length() > 7) && (json_str_[1] == '\"') && (json_str_[2] == 'R') && + (json_str_[3] == 'A') && (json_str_[4] == 'W') && (json_str_[5] == '\"') && + (json_str_[6] == ':')) { + reace_event_ = 10; + raw_msg_len_ = std::stoi(StackFlows::sample_json_str_get(json_str_, "RAW")); + raw_msg_buff_.reserve(raw_msg_len_); + if (json_src.length() > i) { + src_str = std::make_shared(src_str->substr(i + 1)); + enloop = true; + } + break; + } + out_fun(json_str_); + } + json_str_.clear(); + } + if (json_str_flage_ < 0) { + json_str_flage_ = 0; + json_str_.clear(); + throw std::runtime_error("json package error"); + } + } + } break; + case 10: { + if (raw_msg_len_ > src_str->length()) { + raw_msg_buff_.append(src_str->c_str(), src_str->length()); + raw_msg_len_ -= src_str->length(); + } else { + reace_event_ = 0; + raw_msg_buff_.append(src_str->c_str(), raw_msg_len_); + on_raw_data(raw_msg_buff_); + raw_msg_buff_.clear(); + json_str_.clear(); + if (src_str->length() > raw_msg_len_) { + src_str = std::make_shared(src_str->substr(raw_msg_len_ + 1)); + enloop = true; + } + } + } break; + default: + break; + } + } while (enloop); } \ No newline at end of file diff --git a/projects/llm_framework/main_tokenizer/Kconfig b/projects/llm_framework/main_tokenizer/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_tokenizer/SConstruct b/projects/llm_framework/main_tokenizer/SConstruct new file mode 100644 index 00000000..72d39a34 --- /dev/null +++ b/projects/llm_framework/main_tokenizer/SConstruct @@ -0,0 +1,34 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = Glob('src/*.c*') +INCLUDE = [] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +STATIC_FILES += [AFile('_tokenizer.py')] + +env['COMPONENTS'].append({'target':'llm_tokenizer', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_tokenizer/_tokenizer.py b/projects/llm_framework/main_tokenizer/_tokenizer.py new file mode 100644 index 00000000..57d9ac1f --- /dev/null +++ b/projects/llm_framework/main_tokenizer/_tokenizer.py @@ -0,0 +1,130 @@ +#!/bin/env python3 +import zmq +from tokenizers import Tokenizer +import http.server +import socketserver +import threading +import socket +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +tokenizers_obj = {} +tokenizers_content = {} +server_obj = {} +url_path_map = {} + +context = zmq.Context() +zmq_socket = context.socket(zmq.REP) +zmq_socket.bind("ipc:///tmp/rpc.tokenizer") + + +class Request(BaseHTTPRequestHandler): + timeout = 5 + server_version = "Apache" + def do_GET(self): + server_ip, server_port = self.server.server_address + val = 'http://localhost:{}'.format(server_port) + token_obj = tokenizers_obj[url_path_map[val]] + + print(self.path) + self.send_response(200) + self.send_header("type", "get") + self.end_headers() + if self.path == "/bos_id": + bos_id = token_obj.bos_id + if bos_id is None: + msg = json.dumps({"bos_id": -1}) + else: + msg = json.dumps({"bos_id": bos_id}) + elif self.path == "/eos_id": + eos_id = token_obj.eos_id + if eos_id is None: + msg = json.dumps({"eos_id": -1}) + else: + msg = json.dumps({"eos_id": eos_id}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() + self.wfile.write(msg) + + def do_POST(self): + server_ip, server_port = self.server.server_address + val = 'http://localhost:{}'.format(server_port) + token_obj = tokenizers_obj[url_path_map[val]] + data = self.rfile.read( + int(self.headers["content-length"]) + ) + data = data.decode() + self.send_response(200) + self.send_header("type", "post") + self.end_headers() + if self.path == "/encode": + req = json.loads(data) + prompt = req['text'] + token_ids = token_obj.encode(prompt, tokenizers_content[url_path_map[val]]) + if token_ids is None: + msg = json.dumps({"token_ids": -1}) + else: + msg = json.dumps({"token_ids": token_ids}) + elif self.path == "/decode": + req = json.loads(data) + token_ids = req["token_ids"] + text = token_obj.decode(token_ids) + if text is None: + msg = json.dumps({"text": ""}) + else: + msg = json.dumps({"text": text}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() + self.wfile.write(msg) + + +def start_server(httpd): + httpd.serve_forever() + +def find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + +def rpc_forever(): + while True: + try: + message_parts = socket.recv_multipart() + action = message_parts[0].decode('utf-8') + rawmsg = message_parts[1].decode('utf-8') + + val = 'None' + if action == 'creat_tokenizer': + json_args = json.loads(rawmsg) + tokenizer_path = json_args['path'] + tokenizer_content = json_args['content'] + tokenizers_content[tokenizer_path] = tokenizer_content + tokenizers_obj[tokenizer_path] = Tokenizer.from_file(tokenizer_path) + server_obj[tokenizer_path] = socketserver.TCPServer(("", 0), Request) + server_ip, server_port = server_obj[tokenizer_path].server_address + val = 'http://localhost:{}'.format(server_port) + url_path_map[val] = tokenizer_path + thread = threading.Thread(target=start_server, args=(server_obj[tokenizer_path])) + thread.daemon = True + thread.start() + + if action == 'close_tokenizer': + tokenizer_path = rawmsg.decode('utf-8') + server_obj[tokenizer_path].shutdown() + server_obj[tokenizer_path].server_close() + del server_obj[tokenizer_path] + del tokenizers_obj[tokenizer_path] + del tokenizers_content[tokenizer_path] + for key, value in list(url_path_map.items()): + if value == tokenizer_path: + del url_path_map[key] + zmq_socket.send(val.encode('utf-8')) + except: + zmq_socket.send('error'.encode('utf-8')) + +if __name__ == "__main__": + rpc_forever() diff --git a/projects/llm_framework/main_tokenizer/src/main.cpp b/projects/llm_framework/main_tokenizer/src/main.cpp new file mode 100644 index 00000000..6b3579c9 --- /dev/null +++ b/projects/llm_framework/main_tokenizer/src/main.cpp @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include + +int main() +{ + if (access("./_tokenizer.py", F_OK) == 0) { + char *args[] = {(char *)"python3", (char *)"./_tokenizer.py", NULL}; + if (execvp("python3", args) == -1) { + perror("execvp"); + return 1; + } + } else if (access("/opt/m5stack/share/_tokenizer.py", F_OK) == 0) { + char *args[] = {(char *)"python3", (char *)"/opt/m5stack/share/_tokenizer.py", NULL}; + if (execvp("python3", args) == -1) { + perror("execvp"); + return 1; + } + } + perror("_tokenizer.py miss"); + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index 0ce3b6bb..c21e0aa8 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -25,9 +25,7 @@ REQUIREMENTS += ['tts'] INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), ADir('src/runner/include'), ADir('src/runner/src/header')] -STATIC_FILES += [AFile('single_speaker_english_fast.json'), - AFile('single_speaker_fast.json') - ] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_tts', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_tts/single_speaker_english_fast.json b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json similarity index 100% rename from projects/llm_framework/main_tts/single_speaker_english_fast.json rename to projects/llm_framework/main_tts/mode_single-speaker-english-fast.json diff --git a/projects/llm_framework/main_tts/single_speaker_fast.json b/projects/llm_framework/main_tts/mode_single-speaker-fast.json similarity index 100% rename from projects/llm_framework/main_tts/single_speaker_fast.json rename to projects/llm_framework/main_tts/mode_single-speaker-fast.json diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 944aac5f..055412e2 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -19,7 +19,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -42,11 +43,11 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: +private: float *dataW = NULL; int modelSize; - public: +public: std::unique_ptr synthesizer_; SynthesizerTrn_config mode_config_; std::vector inputs_; @@ -59,11 +60,13 @@ class llm_task { task_callback_t out_callback_; int awake_delay_ = 1000; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -86,14 +89,21 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + // Compatible operation + if (model_ == "single_speaker_english_fast") + config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, "single-speaker-english-fast"); + else if (model_ == "single_speaker_fast") + config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, "single-speaker-fast"); + try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -131,7 +141,8 @@ class llm_task { return 0; } - bool TTS(const std::string &msg) { + bool TTS(const std::string &msg) + { SLOGI("TTS msg:%s", msg.c_str()); int32_t dataLen; int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen); @@ -145,47 +156,42 @@ class llm_task { return false; } - bool delete_model() { + bool delete_model() + { synthesizer_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; #undef CONFIG_AUTO_SET class llm_tts : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_tts::_load_config success"); - return 0; - } - } - public: - llm_tts() : StackFlow("tts") { +public: + llm_tts() : StackFlow("tts") + { task_count_ = 1; - repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -207,7 +213,8 @@ class llm_tts : public StackFlow { } } - std::vector splitEachChar(const std::string &text) { + std::vector splitEachChar(const std::string &text) + { std::vector words; std::string input(text); int len = input.length(); @@ -229,9 +236,15 @@ class llm_tts : public StackFlow { return words; } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (data.empty() || (data == "None")) return; static std::string faster_stream_buff; nlohmann::json error_body; @@ -250,12 +263,13 @@ class llm_tts : public StackFlow { std::string tmp_msg2; if (enbase64) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { return; } next_data = &tmp_msg2; } - std::vector tmp_data = splitEachChar((*next_data)); + std::string user_msg = sample_unescapeString(*next_data); + std::vector tmp_data = splitEachChar(user_msg); for (auto cutf8 : tmp_data) { if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") { @@ -285,8 +299,15 @@ class llm_tts : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { @@ -296,13 +317,15 @@ class llm_tts : public StackFlow { if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "play_stop", data); } - llm_channel->subscriber_work_id(llm_task_obj->superior_id_, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + llm_task_obj->superior_id_, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -329,22 +352,26 @@ class llm_tts : public StackFlow { llm_channel->set_stream(llm_task_obj->enstream_); SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("tts") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if ((input.find("llm") != std::string::npos) || (input.find("vlm") != std::string::npos)) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); llm_task_obj->superior_id_ = input; llm_task_obj->superior_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -361,7 +388,8 @@ class llm_tts : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -376,14 +404,17 @@ class llm_tts : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if ((data.find("llm") != std::string::npos) || (data.find("vlm") != std::string::npos)) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->superior_id_ = data; llm_task_obj->superior_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { @@ -396,7 +427,8 @@ class llm_tts : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -424,7 +456,8 @@ class llm_tts : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -445,12 +478,13 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("tts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::exit:%s", data.c_str()); nlohmann::json error_body; @@ -468,21 +502,22 @@ class llm_tts : public StackFlow { return 0; } - ~llm_tts() { + ~llm_tts() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index b1f51c8a..b0e5739e 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -49,9 +49,8 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] STATIC_LIB += static_file * 4 -STATIC_FILES += [AFile('internvl2-1b-ax630c.json'), - AFile('internvl2-1b-ax630c_tokenizer.py') - ] +STATIC_FILES += [AFile('internvl2-1B-ax630c_tokenizer.py')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_vlm', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py b/projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py similarity index 65% rename from projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py rename to projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py index 19cd9aa7..678cf306 100644 --- a/projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py +++ b/projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py @@ -6,20 +6,18 @@ class Tokenizer_Http: - def __init__(self): - - path = "internvl2_tokenizer" + def __init__(self, model_id): self.tokenizer = AutoTokenizer.from_pretrained( - path, trust_remote_code=True, use_fast=False + model_id, trust_remote_code=True, use_fast=False ) - def encode(self, content): - prompt = f"<|im_start|>system\n你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。<|im_end|><|im_start|>user\n{content}<|im_end|><|im_start|>assistant\n" + def encode(self, prompt, content): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n{prompt}<|im_end|><|im_start|>assistant\n" input_ids = self.tokenizer.encode(prompt) return input_ids - def encode_vpm(self, content="Please describe the image shortly."): - prompt = f"<|im_start|>system\n你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。<|im_end|><|im_start|>user\n\n{content}<|im_end|><|im_start|>assistant\n" + def encode_vpm(self, prompt, content="Please describe the image shortly."): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n\n{prompt}<|im_end|><|im_start|>assistant\n" input_ids = self.tokenizer.encode(prompt) return input_ids @@ -42,26 +40,6 @@ def bos_token(self): def eos_token(self): return self.tokenizer.eos_token - -tokenizer = Tokenizer_Http() - -print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) -token_ids = tokenizer.encode_vpm() -# [151644, 8948, 198, 56568, 104625, 100633, 104455, 104800, 101101, 32022, 102022, 99602, 100013, 9370, 90286, 21287, 42140, 53772, 35243, 26288, 104949, 3837, 105205, 109641, 67916, 30698, 11, 54851, 46944, 115404, 42192, 99441, 100623, 48692, 100168, 110498, 1773, 151645, 151644, 872, 198, -# 151646, -# 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, -# 151647, -# 198, 5501, 7512, 279, 2168, 19620, 13, 151645, 151644, 77091, 198] -# 118 -print(token_ids) -print(len(token_ids)) -token_ids = tokenizer.encode("hello world") -# [151644, 8948, 198, 56568, 104625, 100633, 104455, 104800, 101101, 32022, 102022, 99602, 100013, 9370, 90286, 21287, 42140, 53772, 35243, 26288, 104949, 3837, 105205, 109641, 67916, 30698, 11, 54851, 46944, 115404, 42192, 99441, 100623, 48692, 100168, 110498, 1773, 151645, 151644, 872, 198, 14990, 1879, 151645, 151644, 77091, 198] -# 47 -print(token_ids) -print(len(token_ids)) - - class Request(BaseHTTPRequestHandler): # 通过类继承，新定义类 timeout = 5 @@ -117,7 +95,7 @@ def do_POST(self): if b_img_prompt: token_ids = tokenizer.encode_vpm(prompt) else: - token_ids = tokenizer.encode(prompt) + token_ids = tokenizer.encode(prompt, args.content) if token_ids is None: msg = json.dumps({"token_ids": -1}) else: @@ -144,8 +122,16 @@ def do_POST(self): args = argparse.ArgumentParser() args.add_argument("--host", type=str, default="localhost") args.add_argument("--port", type=int, default=8080) + args.add_argument('--model_id', type=str, default='internvl2_tokenizer') + args.add_argument('--content', type=str, default='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。') args = args.parse_args() + tokenizer = Tokenizer_Http(args.model_id) + + + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) + host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1' print("http://%s:%s" % host) server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例 diff --git a/projects/llm_framework/main_vlm/internvl2-1b-ax630c.json b/projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json similarity index 96% rename from projects/llm_framework/main_vlm/internvl2-1b-ax630c.json rename to projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json index a711350f..0818b10c 100644 --- a/projects/llm_framework/main_vlm/internvl2-1b-ax630c.json +++ b/projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json @@ -1,5 +1,5 @@ { - "mode":"internvl2-1b-ax630c", + "mode":"internvl2-1B-ax630c", "type":"vlm", "capabilities":[ "text_generation", diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index ed807c83..2c422f63 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -18,7 +18,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -35,8 +36,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: LLMAttrType mode_config_; std::unique_ptr lLaMa_; std::string model_; @@ -52,11 +53,13 @@ class llm_task { std::atomic_bool tokenizer_server_flage_; unsigned int port_ = 8080; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -80,14 +83,14 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -128,21 +131,19 @@ class llm_task { pid_t pid = fork(); if (pid == 0) { execl("/usr/bin/python3", "python3", - ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), - "--host", "localhost", - "--port", std::to_string(port_).c_str(), - "--model_id", (base_model + "tokenizer").c_str(), - "--content", ("'" + prompt_ + "'").c_str(), - nullptr); + ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), "--host", "localhost", + "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), + "--content", ("'" + prompt_ + "'").c_str(), nullptr); perror("execl failed"); exit(1); } tokenizer_server_flage_ = true; - SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); + SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), + (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); std::this_thread::sleep_for(std::chrono::seconds(10)); } } else { - mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; + mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; } SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str()); mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; @@ -166,7 +167,8 @@ class llm_task { return 0; } - std::string prompt_complete(const std::string &input) { + std::string prompt_complete(const std::string &input) + { std::ostringstream oss_prompt; switch (mode_config_.tokenizer_type) { case TKT_LLaMa: @@ -187,24 +189,20 @@ class llm_task { oss_prompt << input; break; } - SLOGI("prompt_complete:%s",oss_prompt.str().c_str()); + SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); return oss_prompt.str(); } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { try { if (image_data_.empty()) { lLaMa_->Encode(prompt_data_, prompt_complete(msg)); std::string out = lLaMa_->Run(prompt_data_); if (out_callback_) out_callback_(out, true); } else { - if (image_data_.size() > 2 && image_data_[0] == 0xFF && image_data_[1] == 0xD8 && - image_data_[image_data_.size() - 2] == 0xFF && image_data_[image_data_.size() - 1] == 0xD9) { - } else { - SLOGE("Invalid JPEG data."); - return; - } cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR); + if (src.empty()) return; image_data_.clear(); lLaMa_->Encode(src, img_embed); lLaMa_->Encode(img_embed, prompt_data_, prompt_complete(msg)); @@ -215,53 +213,49 @@ class llm_task { } } - bool pause() { + bool pause() + { lLaMa_->Stop(); return true; } - bool delete_model() { + bool delete_model() + { lLaMa_->Deinit(); lLaMa_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; #undef CONFIG_AUTO_SET class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_llm::_load_config success"); - return 0; - } - } - public: - llm_llm() : StackFlow("vlm") { +public: + llm_llm() : StackFlow("vlm") + { task_count_ = 2; - repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -282,21 +276,44 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; @@ -308,8 +325,15 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void task_asr_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -319,12 +343,20 @@ class llm_llm : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->lLaMa_->Stop(); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -352,21 +384,25 @@ class llm_llm : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("vlm") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("asr") != std::string::npos) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -383,7 +419,8 @@ class llm_llm : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -398,12 +435,15 @@ class llm_llm : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("asr") != std::string::npos) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { @@ -416,7 +456,8 @@ class llm_llm : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -440,7 +481,8 @@ class llm_llm : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -461,12 +503,13 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("vlm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::exit:%s", data.c_str()); nlohmann::json error_body; @@ -484,19 +527,22 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_yolo/Kconfig b/projects/llm_framework/main_yolo/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct new file mode 100644 index 00000000..e5ea4cd4 --- /dev/null +++ b/projects/llm_framework/main_yolo/SConstruct @@ -0,0 +1,46 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = append_srcs_dir(ADir('src')) +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'ax-samples'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] + +INCLUDE += [ADir('../include'), ADir('../include/opencv4')] + +static_file = Glob('../static_lib/module-llm/libabsl_*') +static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') +STATIC_LIB += static_file * 2 + +STATIC_FILES += Glob('mode_*.json') + + +env['COMPONENTS'].append({'target':'llm_yolo', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_yolo/mode_yolo11n-pose.json b/projects/llm_framework/main_yolo/mode_yolo11n-pose.json new file mode 100644 index 00000000..7bfc51f7 --- /dev/null +++ b/projects/llm_framework/main_yolo/mode_yolo11n-pose.json @@ -0,0 +1,106 @@ +{ + "mode":"yolo11n-pose", + "type":"cv", + "capabilities":[ + "Pose" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.yolobox" + ], + "mode_param":{ + "yolo_model":"yolo11n-pose.axmodel", + "model_type":"pose", + "img_h":320, + "img_w":320, + "cls_num":1, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/mode_yolo11n-seg.json b/projects/llm_framework/main_yolo/mode_yolo11n-seg.json new file mode 100644 index 00000000..776fae50 --- /dev/null +++ b/projects/llm_framework/main_yolo/mode_yolo11n-seg.json @@ -0,0 +1,106 @@ +{ + "mode":"yolo11s-seg", + "type":"cv", + "capabilities":[ + "Segmentation" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.yolobox" + ], + "mode_param":{ + "yolo_model":"yolo11n-seg.axmodel", + "model_type":"segment", + "img_h":320, + "img_w":320, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/mode_yolo11n.json b/projects/llm_framework/main_yolo/mode_yolo11n.json new file mode 100644 index 00000000..abd92827 --- /dev/null +++ b/projects/llm_framework/main_yolo/mode_yolo11n.json @@ -0,0 +1,106 @@ +{ + "mode":"yolo11n", + "type":"cv", + "capabilities":[ + "Detection" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.yolobox" + ], + "mode_param":{ + "yolo_model":"yolo11n.axmodel", + "model_type":"detect", + "img_h":320, + "img_w":320, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp new file mode 100644 index 00000000..e07a044e --- /dev/null +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -0,0 +1,401 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ +#include "detection.hpp" +#define UNUSE_STRUCT_OBJECT +#include "EngineWrapper.hpp" +#include "utils/io.hpp" +#include + +static const char* strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; + +/// @brief npu type +typedef enum axNPU_TYPE_E { + AX_NPU_DEFAULT = 0, /* running under default NPU according to system */ + AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ + AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ + AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ + AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ + AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ +} AX_NPU_TYPE_E; + +static AX_S32 CheckModelVNpu(const std::string& strModel, const AX_ENGINE_MODEL_TYPE_T& eModelType, + const AX_S32& nNpuType, AX_U32& nNpuSet) +{ + AX_ENGINE_NPU_ATTR_T stNpuAttr; + memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); + + auto ret = AX_ENGINE_GetVNPUAttr(&stNpuAttr); + if (ret == 0) { + // VNPU DISABLE + if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_DISABLE) { + nNpuSet = 0x01; // NON-VNPU (0b111) + } + // STD VNPU + else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { + // 7.2T & 10.8T no allow + if (eModelType == AX_ENGINE_MODEL_TYPE1 || eModelType == AX_ENGINE_MODEL_TYPE1) { + return -1; + } + + // default STD VNPU2 + if (nNpuType == 0) { + nNpuSet = 0x02; // VNPU2 (0b010) + } else { + if (nNpuType & AX_STD_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + if (nNpuType & AX_STD_VNPU_2) { + nNpuSet |= 0x02; // VNPU2 (0b010) + } + if (nNpuType & AX_STD_VNPU_3) { + nNpuSet |= 0x04; // VNPU3 (0b100) + } + } + } + // BL VNPU + else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { + // 10.8T no allow + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + return -1; + } + + // default BL VNPU + if (nNpuType == 0) { + // 7.2T default BL VNPU1 + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + nNpuSet = 0x01; // VNPU1 (0b001) + } + // 3.6T default BL VNPU2 + else { + nNpuSet = 0x02; // VNPU2 (0b010) + } + } else { + // 7.2T + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + // no allow set to BL VNPU2 + if (nNpuType & AX_BL_VNPU_2) { + return -1; + } + if (nNpuType & AX_BL_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + } + // 3.6T + else { + if (nNpuType & AX_BL_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + if (nNpuType & AX_BL_VNPU_2) { + nNpuSet |= 0x02; // VNPU2 (0b010) + } + } + } + } + } else { + printf("AX_ENGINE_GetVNPUAttr fail ret = %x\n", ret); + } + + return ret; +} + +int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) +{ + AX_S32 ret = 0; + + // 1. load model + AX_BOOL bLoadModelUseCmm = AX_FALSE; + AX_CHAR* pModelBufferVirAddr = nullptr; + AX_U64 u64ModelBufferPhyAddr = 0; + AX_U32 nModelBufferSize = 0; + + std::vector model_buffer; + + if (bLoadModelUseCmm) { + if (!utils::read_file(strModelPath, (AX_VOID**)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { + printf("ALGO read model(%s) fail\n", strModelPath); + return -1; + } + } else { + if (!utils::read_file(strModelPath, model_buffer)) { + printf("ALGO read model(%s) fail\n", strModelPath); + return -1; + } + + pModelBufferVirAddr = model_buffer.data(); + nModelBufferSize = model_buffer.size(); + } + + auto freeModelBuffer = [&]() { + if (bLoadModelUseCmm) { + if (u64ModelBufferPhyAddr != 0) { + AX_SYS_MemFree(u64ModelBufferPhyAddr, &pModelBufferVirAddr); + } + } else { + std::vector().swap(model_buffer); + } + return; + }; + + // 1.1 Get Model Type + AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; + ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); + if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { + printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); + freeModelBuffer(); + return -1; + } + + // 1.2 Check VNPU + AX_ENGINE_NPU_SET_T nNpuSet = 0; + ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); + if (0 != ret) { + printf("ALGO CheckModelVNpu fail\n"); + freeModelBuffer(); + return -1; + } + + // 2. create handle + AX_ENGINE_HANDLE handle = nullptr; + ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); + auto deinit_handle = [&handle]() { + if (handle) { + AX_ENGINE_DestroyHandle(handle); + } + return -1; + }; + + freeModelBuffer(); + + if (0 != ret || !handle) { + printf("ALGO Create model(%s) handle fail\n", strModelPath); + + return deinit_handle(); + } + + // 3. create context + ret = AX_ENGINE_CreateContext(handle); + if (0 != ret) { + return deinit_handle(); + } + + // 4. set io + m_io_info = nullptr; + ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); + if (0 != ret) { + return deinit_handle(); + } + m_input_num = m_io_info->nInputSize; + m_output_num = m_io_info->nOutputSize; + + // 4.1 query io + // AX_IMG_FORMAT_E eDtype; + // ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. + // if (0 != ret) { + // printf("model(%s) query model input size fail\n", strModelPath.c_str()); + // return deinit_handle(); + // } + + // if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || + // eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { + // printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); + // return deinit_handle(); + // } + + // 4.2 brief io +#ifdef __DEBUG__ + printf("brief_io_info\n"); + utils::brief_io_info(strModelPath, m_io_info); +#endif + + // 5. Config VNPU + // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); + // will do nothing for using create handle v2 api + + // 6. prepare io + // AX_U32 nIoDepth = (stCtx.vecOutputBufferFlag.size() == 0) ? 1 : stCtx.vecOutputBufferFlag.size(); + ret = utils::prepare_io(strModelPath, m_io_info, m_io, utils::IO_BUFFER_STRATEGY_DEFAULT); + if (0 != ret) { + printf("prepare io failed!\n"); + utils::free_io(m_io); + return deinit_handle(); + } + + m_handle = handle; + m_hasInit = true; + + return 0; +} + +int EngineWrapper::SetInput(void* pInput, int index) +{ + return utils::push_io_input(pInput, index, m_io); +} + +int EngineWrapper::RunSync() +{ + if (!m_hasInit) return -1; + + // 7.3 run & benchmark + auto ret = AX_ENGINE_RunSync(m_handle, &m_io); + if (0 != ret) { + printf("AX_ENGINE_RunSync failed. ret=0x%x\n", ret); + return ret; + } + + return 0; +} + +const char* CLASS_NAMES[] = { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", + "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", + "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", + "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush"}; + +const char* OBB_CLASS_NAMES[] = {"plane", + "ship", + "storage tank", + "baseball diamond", + "tennis court", + "basketball court", + "ground track field", + "harbor", + "bridge", + "large vehicle", + "small vehicle", + "helicopter", + "roundabout", + "soccer ball field", + "swimming pool"}; + +static const std::vector> COCO_COLORS = { + {56, 0, 255}, {226, 255, 0}, {0, 94, 255}, {0, 37, 255}, {0, 255, 94}, {255, 226, 0}, {0, 18, 255}, + {255, 151, 0}, {170, 0, 255}, {0, 255, 56}, {255, 0, 75}, {0, 75, 255}, {0, 255, 169}, {255, 0, 207}, + {75, 255, 0}, {207, 0, 255}, {37, 0, 255}, {0, 207, 255}, {94, 0, 255}, {0, 255, 113}, {255, 18, 0}, + {255, 0, 56}, {18, 0, 255}, {0, 255, 226}, {170, 255, 0}, {255, 0, 245}, {151, 255, 0}, {132, 255, 0}, + {75, 0, 255}, {151, 0, 255}, {0, 151, 255}, {132, 0, 255}, {0, 255, 245}, {255, 132, 0}, {226, 0, 255}, + {255, 37, 0}, {207, 255, 0}, {0, 255, 207}, {94, 255, 0}, {0, 226, 255}, {56, 255, 0}, {255, 94, 0}, + {255, 113, 0}, {0, 132, 255}, {255, 0, 132}, {255, 170, 0}, {255, 0, 188}, {113, 255, 0}, {245, 0, 255}, + {113, 0, 255}, {255, 188, 0}, {0, 113, 255}, {255, 0, 0}, {0, 56, 255}, {255, 0, 113}, {0, 255, 188}, + {255, 0, 94}, {255, 0, 18}, {18, 255, 0}, {0, 255, 132}, {0, 188, 255}, {0, 245, 255}, {0, 169, 255}, + {37, 255, 0}, {255, 0, 151}, {188, 0, 255}, {0, 255, 37}, {0, 255, 0}, {255, 0, 170}, {255, 0, 37}, + {255, 75, 0}, {0, 0, 255}, {255, 207, 0}, {255, 0, 226}, {255, 245, 0}, {188, 255, 0}, {0, 255, 18}, + {0, 255, 75}, {0, 255, 151}, {255, 56, 0}, {245, 255, 0}}; + +static const std::vector> KPS_COLORS = { + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {255, 128, 0}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {51, 153, 255}, + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}}; + +static const std::vector> LIMB_COLORS = { + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {255, 51, 255}, {255, 51, 255}, {255, 51, 255}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {0, 255, 0}, {0, 255, 0}, + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}}; + +static const std::vector> SKELETON = { + {16, 14}, {14, 12}, {17, 15}, {15, 13}, {12, 13}, {6, 12}, {7, 13}, {6, 7}, {6, 8}, {7, 9}, + {8, 10}, {9, 11}, {2, 3}, {1, 2}, {1, 3}, {2, 4}, {3, 5}, {4, 6}, {5, 7}}; + +void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, + int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects, + std::string& model_type) +{ + // std::vector objects; + std::vector proposals; + if (model_type == "detect") { + for (int i = 0; i < 3; ++i) { + auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, + cls_num); + } + detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + // detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); + } else if (model_type == "segment") { + float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, + (float*)io_data->pOutputs[2].pVirAddr}; + float* output_seg_ptr[3] = {(float*)io_data->pOutputs[3].pVirAddr, (float*)io_data->pOutputs[4].pVirAddr, + (float*)io_data->pOutputs[5].pVirAddr}; + for (int i = 0; i < 3; ++i) { + auto feat_ptr = output_ptr[i]; + auto feat_seg_ptr = output_seg_ptr[i]; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_seg_native(stride, feat_ptr, feat_seg_ptr, prob_threshold, proposals, + input_w, input_h, cls_num); + } + auto mask_proto_ptr = (float*)io_data->pOutputs[6].pVirAddr; + detection::get_out_bbox_mask(proposals, objects, mask_proto_ptr, 32, 4, nms_threshold, input_h, input_w, + mat.rows, mat.cols); + // detection::draw_objects_mask(mat, objects, CLASS_NAMES, COCO_COLORS, "yolo11_seg_out"); + } else if (model_type == "pose") { + float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, + (float*)io_data->pOutputs[2].pVirAddr}; + float* output_kps_ptr[3] = {(float*)io_data->pOutputs[3].pVirAddr, (float*)io_data->pOutputs[4].pVirAddr, + (float*)io_data->pOutputs[5].pVirAddr}; + + for (int i = 0; i < 3; ++i) { + auto feat_ptr = output_ptr[i]; + auto feat_kps_ptr = output_kps_ptr[i]; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_pose_native(stride, feat_ptr, feat_kps_ptr, prob_threshold, proposals, + input_h, input_w, 17, cls_num); + } + detection::get_out_bbox_kps(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + // detection::draw_keypoints(mat, objects, KPS_COLORS, LIMB_COLORS, SKELETON, "yolo11_pose_out"); + } else if (model_type == "obb") { + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + detection::generate_grids_and_stride(input_w, input_h, strides, grid_strides); + auto feat_ptr = (float*)io_data->pOutputs[0].pVirAddr; + detection::obb::generate_proposals_yolov8_obb_native(grid_strides, feat_ptr, prob_threshold, proposals, input_w, + input_h, cls_num); + detection::obb::get_out_obb_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + // detection::obb::draw_objects_obb(mat, objects, OBB_CLASS_NAMES, "yolo11_obb_out", 1); + } +} + +int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, + float& nms_threshold, std::vector& objects, std::string& model_type) +{ + post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects, model_type); + return 0; +} + +int EngineWrapper::GetOutput(void* pOutput, int index) +{ + return utils::push_io_output(pOutput, index, m_io); +} + +int EngineWrapper::GetInputSize(int index) +{ + return m_io.pInputs[index].nSize; +} + +int EngineWrapper::GetOutputSize(int index) +{ + return m_io.pOutputs[index].nSize; +} + +int EngineWrapper::Release() +{ + if (m_handle) { + utils::free_io(m_io); + AX_ENGINE_DestroyHandle(m_handle); + m_handle = nullptr; + } + return 0; +} diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.hpp b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp new file mode 100644 index 00000000..3a6dfa07 --- /dev/null +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp @@ -0,0 +1,68 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ + +#pragma once + +#include +#include +#include "ax_engine_api.h" + +#ifndef UNUSE_STRUCT_OBJECT +namespace detection { +typedef struct Object { + cv::Rect_ rect; + int label; + float prob; + cv::Point2f landmark[5]; + /* for yolov5-seg */ + cv::Mat mask; + std::vector mask_feat; + std::vector kps_feat; + /* for yolov8-obb */ + float angle; +} Object; + +} // namespace detection +#endif + +class EngineWrapper { +public: + EngineWrapper() : m_hasInit(false), m_handle(nullptr) + { + } + + ~EngineWrapper() + { + Release(); + } + + int Init(const char* strModelPath, uint32_t nNpuType = 0); + + int SetInput(void* pInput, int index); + + int RunSync(); + + int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, + std::vector& objects, std::string& model_type); + + int GetOutput(void* pOutput, int index); + + int GetInputSize(int index); + int GetOutputSize(int index); + + int Release(); + +protected: + bool m_hasInit; + AX_ENGINE_HANDLE m_handle; + AX_ENGINE_IO_INFO_T* m_io_info{}; + AX_ENGINE_IO_T m_io{}; + int m_input_num{}, m_output_num{}; +}; diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp new file mode 100644 index 00000000..9583811c --- /dev/null +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -0,0 +1,594 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "EngineWrapper.hpp" +#include "common.hpp" +#include +#include +#include + +#include "../../../../SDK/components/utilities/include/sample_log.h" + +using namespace StackFlows; + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + SLOGW("llm_yolo will be exit!"); + main_exit_flage = 1; +} + +static std::string base_model_path_; +static std::string base_model_config_path_; + +typedef struct { + std::string yolo_model; + std::string model_type = "detect"; + std::vector cls_name; + int img_h = 640; + int img_w = 640; + int cls_num = 80; + float pron_threshold = 0.45f; + float nms_threshold = 0.45; +} yolo_config; + +typedef std::function &data, bool finish)> task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: +public: + yolo_config mode_config_; + std::string model_; + std::unique_ptr yolo_; + std::string response_format_; + std::vector inputs_; + std::vector image_data_; + bool enoutput_; + bool enstream_; + static int ax_init_flage_; + task_callback_t out_callback_; + std::atomic_bool camera_flage_; + std::mutex inference_mtx_; + + bool parse_config(const nlohmann::json &config_body) + { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } else + throw std::string("error"); + } catch (...) { + SLOGE("setup config_body error"); + return true; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + nlohmann::json file_body; + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + CONFIG_AUTO_SET(file_body["mode_param"], yolo_model); + CONFIG_AUTO_SET(file_body["mode_param"], img_h); + CONFIG_AUTO_SET(file_body["mode_param"], img_w); + CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], cls_name); + CONFIG_AUTO_SET(file_body["mode_param"], cls_num); + CONFIG_AUTO_SET(file_body["mode_param"], model_type); + mode_config_.yolo_model = base_model + mode_config_.yolo_model; + yolo_ = std::make_unique(); + if (0 != yolo_->Init(mode_config_.yolo_model.c_str())) { + SLOGE("Init yolo_model model failed!\n"); + return -5; + } + } catch (...) { + SLOGE("config false"); + return -6; + } + return 0; + } + + std::string format_float(double value, int decimal_places) + { + std::ostringstream out; + out << std::fixed << std::setprecision(decimal_places) << value; + return out.str(); + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + bool inference_decode(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); + if (src.empty()) return true; + return inference(src); + } + + bool inference_raw_yuv(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 2) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); + cv::Mat rgb; + cv::cvtColor(camera_data, rgb, cv::COLOR_YUV2RGB_YUYV); + return inference(rgb, false); + } + + bool inference_raw_rgb(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + return inference(camera_data, false); + } + + bool inference_raw_bgr(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + return inference(camera_data); + } + + bool inference(cv::Mat &src, bool bgr2rgb = true) + { + try { + int ret = -1; + std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); + common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, bgr2rgb); + yolo_->SetInput((void *)image.data(), 0); + if (0 != yolo_->RunSync()) { + SLOGE("Run yolo model failed!\n"); + throw std::string("yolo_ RunSync error"); + } + std::vector objects; + yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, + mode_config_.pron_threshold, mode_config_.nms_threshold, objects, + mode_config_.model_type); + std::vector yolo_output; + for (size_t i = 0; i < objects.size(); i++) { + const detection::Object &obj = objects[i]; + nlohmann::json output; + output["class"] = mode_config_.cls_name[obj.label]; + output["confidence"] = format_float(obj.prob, 2); + output["bbox"] = nlohmann::json::array(); + output["bbox"].push_back(format_float(obj.rect.x, 0)); + output["bbox"].push_back(format_float(obj.rect.y, 0)); + output["bbox"].push_back(format_float(obj.rect.x + obj.rect.width, 0)); + output["bbox"].push_back(format_float(obj.rect.y + obj.rect.height, 0)); + if (mode_config_.model_type == "segment") output["mask"] = obj.mask_feat; + if (mode_config_.model_type == "pose") output["kps"] = obj.kps_feat; + if (mode_config_.model_type == "obb") output["angle"] = obj.angle; + yolo_output.push_back(output); + if (out_callback_) out_callback_(yolo_output, false); + } + if (out_callback_) out_callback_(yolo_output, true); + } catch (...) { + SLOGW("yolo_->Run have error!"); + return true; + } + return false; + } + + void _ax_init() + { + if (!ax_init_flage_) { + int ret = AX_SYS_Init(); + if (0 != ret) { + fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret); + } + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); + ret = AX_ENGINE_Init(&npu_attr); + if (0 != ret) { + fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret); + } + } + ax_init_flage_++; + } + + void _ax_deinit() + { + if (ax_init_flage_ > 0) { + --ax_init_flage_; + if (!ax_init_flage_) { + AX_ENGINE_Deinit(); + AX_SYS_Deinit(); + } + } + } + + llm_task(const std::string &workid) + { + _ax_init(); + } + + ~llm_task() + { + _ax_deinit(); + } +}; +int llm_task::ax_init_flage_ = 0; +#undef CONFIG_AUTO_SET + +class llm_yolo : public StackFlow { +private: + int task_count_; + std::unordered_map> llm_task_; + +public: + llm_yolo() : StackFlow("yolo") + { + task_count_ = 1; + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::vector &data, + bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + for (const auto &jsonObj : data) { + data_body["delta"].push_back(jsonObj); + } + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + // SLOGI("send utf-8"); + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); + return; + } + const std::string *next_data = &data; + bool enstream = (object.find("stream") == std::string::npos) ? false : true; + int ret; + std::string tmp_msg1; + if (enstream) { + static std::unordered_map stream_buff; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg1; + } + // must encode base64 + std::string tmp_msg2; + ret = decode_base64((*next_data), tmp_msg2); + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg2; + + if (llm_task_obj->inference_decode(*next_data)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + } + } + + void task_camera_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + SLOGE("Model run failed."); + return; + } + try { + llm_task_obj->inference_raw_yuv(data); + } catch (...) { + SLOGE("data format error"); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, unit_name_); + return -1; + } + + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + SLOGE("setup json format error."); + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + + llm_task_obj->set_output(std::bind(&llm_yolo::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + + for (const auto input : llm_task_obj->inputs_) { + if (input.find("yolo") != std::string::npos) { + llm_channel->subscriber_work_id( + "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + } else { + std::string input_url_name = input + ".out_port"; + std::string input_url = unit_call("sys", "sql_select", input_url_name); + if (!input_url.empty()) { + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; + llm_channel->subscriber( + input_url, [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { + this->task_camera_data(_llm_task_obj, _llm_channel, raw); + }); + } + } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + return 0; + } else { + SLOGE("load_mode Failed"); + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::link:%s", data.c_str()); + int ret = 1; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + if (data.find("yolo") != std::string::npos) { + ret = llm_channel->subscriber_work_id( + "", + std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->inputs_.push_back(data); + } else if (data.find("camera") != std::string::npos) { + std::string input_url_name = data + ".out_port"; + std::string input_url = unit_call("sys", "sql_select", input_url_name); + if (!input_url.empty()) { + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; + llm_channel->subscriber(input_url, + [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { + this->task_camera_data(_llm_task_obj, _llm_channel, raw); + }); + } + llm_task_obj->inputs_.push_back(data); + } + if (ret) { + error_body["code"] = -20; + error_body["message"] = "link false"; + send("None", "None", error_body, work_id); + return; + } else { + send("None", "None", LLM_NO_ERROR, work_id); + } + } + + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::unlink:%s", data.c_str()); + int ret = 0; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + llm_channel->stop_subscriber_work_id(data); + auto llm_task_obj = llm_task_[work_id_num]; + for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { + if (*it == data) { + it = llm_task_obj->inputs_.erase(it); + } else { + ++it; + } + } + send("None", "None", LLM_NO_ERROR, work_id); + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::taskinfo:%s", data.c_str()); + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("yolo.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs"] = llm_task_obj->inputs_; + send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::exit:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_yolo() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main() +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_yolo llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/setup.ini b/projects/llm_framework/setup.ini index d01d0fb1..e1363e0c 100644 --- a/projects/llm_framework/setup.ini +++ b/projects/llm_framework/setup.ini @@ -1,7 +1,7 @@ [ssh] local_file_path = dist remote_file_path = /root/dist -remote_host = 192.168.28.62 +remote_host = m5stack-LLM.local remote_port = 22 username = root password = 123456 \ No newline at end of file diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index d998f610..febaa5ac 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -31,7 +31,7 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): elif item.startswith('lib'): os.makedirs(os.path.join(deb_folder, 'opt/m5stack/lib'), exist_ok = True) shutil.copy2(os.path.join(src_folder, item), os.path.join(deb_folder, 'opt/m5stack/lib', item)) - else: + elif not item.startswith('mode_'): os.makedirs(os.path.join(deb_folder, 'opt/m5stack/share'), exist_ok = True) shutil.copy2(os.path.join(src_folder, item), os.path.join(deb_folder, 'opt/m5stack/share', item)) # os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data'), exist_ok = True) @@ -98,6 +98,12 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): f.write(f'[ -f "/lib/systemd/system/llm-llm.service" ] && systemctl start llm-llm.service\n') f.write(f'[ -f "/lib/systemd/system/llm-tts.service" ] && systemctl enable llm-tts.service\n') f.write(f'[ -f "/lib/systemd/system/llm-tts.service" ] && systemctl start llm-tts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl enable llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl start llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl enable llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl start llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl enable llm-melotts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl start llm-melotts.service\n') f.write(f'exit 0\n') with open(os.path.join(deb_folder, 'DEBIAN/prerm'),'w') as f: f.write(f'#!/bin/sh\n') @@ -113,6 +119,12 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): f.write(f'[ -f "/lib/systemd/system/llm-asr.service" ] && systemctl disable llm-asr.service\n') f.write(f'[ -f "/lib/systemd/system/llm-sys.service" ] && systemctl stop llm-sys.service\n') f.write(f'[ -f "/lib/systemd/system/llm-sys.service" ] && systemctl disable llm-sys.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl stop llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl disable llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl stop llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl disable llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl stop llm-melotts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl disable llm-melotts.service\n') f.write(f'exit 0\n') os.chmod(os.path.join(deb_folder, 'DEBIAN/postinst'), 0o755) os.chmod(os.path.join(deb_folder, 'DEBIAN/prerm'), 0o755) @@ -145,6 +157,14 @@ def create_data_deb(package_name, version, src_folder, revision = 'm5stack1'): if os.path.exists(zip_file_extrpath): shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/data')) + RED = "\033[31m" + RESET = "\033[0m" + os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data/models'), exist_ok = True) + if os.path.exists(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:]))): + shutil.copy2(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), os.path.join(deb_folder, 'opt/m5stack/data/models', 'mode_{}.json'.format(package_name[4:]))) + else: + print(RED, os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), " miss", RESET) + os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: f.write(f'Package: {package_name}\n') @@ -239,7 +259,7 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): os.system('rm ./*.deb m5stack_* -rf') exit(0) - version = '1.2' + version = '1.3' data_version = '0.2' src_folder = '../dist' revision = 'm5stack1' @@ -258,6 +278,12 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_bin_deb('llm-llm', version, src_folder, revision) create_bin_deb('llm-tts', version, src_folder, revision) create_bin_deb('llm-melotts', version, src_folder, revision) + create_bin_deb('llm-camera', version, src_folder, revision) + create_bin_deb('llm-vlm', version, src_folder, revision) + create_bin_deb('llm-yolo', version, src_folder, revision) + create_bin_deb('llm-skel', version, src_folder, revision) + # create_bin_deb('llm-tokenizer', version, src_folder, revision) + if (create_data): create_data_deb('llm-audio-en-us', data_version, src_folder, revision) create_data_deb('llm-audio-zh-cn', data_version, src_folder, revision) @@ -271,3 +297,10 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_data_deb('llm-single-speaker-english-fast', data_version, src_folder, revision) create_data_deb('llm-single-speaker-fast', data_version, src_folder, revision) create_data_deb('llm-melotts-zh-cn', data_version, src_folder, revision) + create_data_deb('llm-yolo11n', data_version, src_folder, revision) + create_data_deb('llm-yolo11n-pose', data_version, src_folder, revision) + create_data_deb('llm-yolo11n-seg', data_version, src_folder, revision) + create_data_deb('llm-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision) + create_data_deb('llm-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision) + create_data_deb('llm-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision) + create_data_deb('llm-internvl2-1B-ax630c', data_version, src_folder, revision) \ No newline at end of file