From b1392809f1bd21402e69fed56d151f2a9e4d0b51 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 19 Nov 2024 19:07:14 +0800 Subject: [PATCH 01/44] [add] main_llm_tokenizer experiment --- .../llm_framework/main_llm_tokenizer/Kconfig | 0 .../main_llm_tokenizer/SConstruct | 34 +++++ .../main_llm_tokenizer/_tokenizer.py | 132 ++++++++++++++++++ .../main_llm_tokenizer/src/main.cpp | 25 ++++ 4 files changed, 191 insertions(+) create mode 100644 projects/llm_framework/main_llm_tokenizer/Kconfig create mode 100644 projects/llm_framework/main_llm_tokenizer/SConstruct create mode 100644 projects/llm_framework/main_llm_tokenizer/_tokenizer.py create mode 100644 projects/llm_framework/main_llm_tokenizer/src/main.cpp diff --git a/projects/llm_framework/main_llm_tokenizer/Kconfig b/projects/llm_framework/main_llm_tokenizer/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_llm_tokenizer/SConstruct b/projects/llm_framework/main_llm_tokenizer/SConstruct new file mode 100644 index 00000000..72d39a34 --- /dev/null +++ b/projects/llm_framework/main_llm_tokenizer/SConstruct @@ -0,0 +1,34 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = Glob('src/*.c*') +INCLUDE = [] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +STATIC_FILES += [AFile('_tokenizer.py')] + +env['COMPONENTS'].append({'target':'llm_tokenizer', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_llm_tokenizer/_tokenizer.py b/projects/llm_framework/main_llm_tokenizer/_tokenizer.py new file mode 100644 index 00000000..c4c18b30 --- /dev/null +++ b/projects/llm_framework/main_llm_tokenizer/_tokenizer.py @@ -0,0 +1,132 @@ +#!/bin/env python3 +import zmq +from tokenizers import Tokenizer +import http.server +import socketserver +import threading +import socket +import json +from http.server import HTTPServer, BaseHTTPRequestHandler +tokenizers_obj = {} +tokenizers_content = {} +server_obj = {} +url_path_map = {} + +context = zmq.Context() +zmq_socket = context.socket(zmq.REP) +zmq_socket.bind("ipc:///tmp/rpc.tokenizer") + + +class Request(BaseHTTPRequestHandler): + timeout = 5 + server_version = "Apache" + def do_GET(self): + server_ip, server_port = self.server.server_address + val = 'http://localhost:{}'.format(server_port) + token_obj = tokenizers_obj[url_path_map[val]] + + print(self.path) + self.send_response(200) + self.send_header("type", "get") + self.end_headers() + if self.path == "/bos_id": + bos_id = token_obj.bos_id + # print(bos_id) + # to json + if bos_id is None: + msg = json.dumps({"bos_id": -1}) + else: + msg = json.dumps({"bos_id": bos_id}) + elif self.path == "/eos_id": + eos_id = token_obj.eos_id + if eos_id is None: + msg = json.dumps({"eos_id": -1}) + else: + msg = json.dumps({"eos_id": eos_id}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() + self.wfile.write(msg) + + def do_POST(self): + server_ip, server_port = self.server.server_address + val = 'http://localhost:{}'.format(server_port) + token_obj = tokenizers_obj[url_path_map[val]] + data = self.rfile.read( + int(self.headers["content-length"]) + ) + data = data.decode() + self.send_response(200) + self.send_header("type", "post") + self.end_headers() + if self.path == "/encode": + req = json.loads(data) + prompt = req['text'] + token_ids = token_obj.encode(prompt, tokenizers_content[url_path_map[val]]) + if token_ids is None: + msg = json.dumps({"token_ids": -1}) + else: + msg = json.dumps({"token_ids": token_ids}) + elif self.path == "/decode": + req = json.loads(data) + token_ids = req["token_ids"] + text = token_obj.decode(token_ids) + if text is None: + msg = json.dumps({"text": ""}) + else: + msg = json.dumps({"text": text}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() + self.wfile.write(msg) + + +def start_server(httpd): + httpd.serve_forever() + +def find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + +def rpc_forever(): + while True: + try: + message_parts = socket.recv_multipart() + action = message_parts[0].decode('utf-8') + rawmsg = message_parts[1].decode('utf-8') + + val = 'None' + if action == 'creat_tokenizer': + json_args = json.loads(rawmsg) + tokenizer_path = json_args['path'] + tokenizer_content = json_args['content'] + tokenizers_content[tokenizer_path] = tokenizer_content + tokenizers_obj[tokenizer_path] = Tokenizer.from_file(tokenizer_path) + server_obj[tokenizer_path] = socketserver.TCPServer(("", 0), Request) + server_ip, server_port = server_obj[tokenizer_path].server_address + val = 'http://localhost:{}'.format(server_port) + url_path_map[val] = tokenizer_path + thread = threading.Thread(target=start_server, args=(server_obj[tokenizer_path])) + thread.daemon = True + thread.start() + + if action == 'close_tokenizer': + tokenizer_path = rawmsg.decode('utf-8') + server_obj[tokenizer_path].shutdown() + server_obj[tokenizer_path].server_close() + del server_obj[tokenizer_path] + del tokenizers_obj[tokenizer_path] + del tokenizers_content[tokenizer_path] + for key, value in list(url_path_map.items()): + if value == tokenizer_path: + del url_path_map[key] + zmq_socket.send(val.encode('utf-8')) + except: + pass + +if __name__ == "__main__": + rpc_forever() diff --git a/projects/llm_framework/main_llm_tokenizer/src/main.cpp b/projects/llm_framework/main_llm_tokenizer/src/main.cpp new file mode 100644 index 00000000..e10e6212 --- /dev/null +++ b/projects/llm_framework/main_llm_tokenizer/src/main.cpp @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include + +int main() { + if (access("./_tokenizer.py", F_OK) == 0) { + char *args[] = {"python3", "./_tokenizer.py", NULL}; + if (execvp("python3", args) == -1) { + perror("execvp"); + return 1; + } + } else if (access("/opt/m5stack/share/_tokenizer.py", F_OK) == 0) { + char *args[] = {"python3", "/opt/m5stack/share/_tokenizer.py", NULL}; + if (execvp("python3", args) == -1) { + perror("execvp"); + return 1; + } + } + perror("_tokenizer.py miss"); + return 0; +} \ No newline at end of file From ab093b6708e5a57046075695c1f4d64757878bbb Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 20 Nov 2024 17:38:42 +0800 Subject: [PATCH 02/44] [format] new clang-format --- .clang-format | 167 ++++++++++++++++++ .gitignore | 1 + .../StackFlow/stackflow/StackFlow.cpp | 142 ++++++++++----- .../StackFlow/stackflow/StackFlow.h | 83 +++++---- .../StackFlow/stackflow/StackFlowUtil.cpp | 33 ++-- ext_components/StackFlow/stackflow/pzmq.hpp | 69 +++++--- projects/llm_framework/main/src/main.cpp | 53 ++++-- projects/llm_framework/main_asr/src/main.cpp | 98 ++++++---- .../llm_framework/main_audio/src/main.cpp | 82 ++++++--- .../main_audio/src/sample_audio.c | 27 ++- projects/llm_framework/main_kws/src/main.cpp | 71 +++++--- projects/llm_framework/main_llm/src/main.cpp | 93 ++++++---- .../main_llm_tokenizer/_tokenizer.py | 4 +- .../main_llm_tokenizer/src/main.cpp | 3 +- .../llm_framework/main_melotts/src/main.cpp | 83 ++++++--- .../main_sys/include/event_loop.h | 8 +- .../main_sys/include/remote_action.h | 8 +- .../main_sys/include/remote_server.h | 8 +- .../llm_framework/main_sys/include/serial.h | 8 +- .../llm_framework/main_sys/include/zmq_bus.h | 8 +- .../llm_framework/main_sys/src/config.cpp | 11 +- .../llm_framework/main_sys/src/event_loop.cpp | 71 +++++--- projects/llm_framework/main_sys/src/main.cpp | 21 ++- .../main_sys/src/remote_action.cpp | 9 +- .../main_sys/src/remote_server.cpp | 51 ++++-- .../llm_framework/main_sys/src/serial_com.cpp | 27 +-- .../llm_framework/main_sys/src/tcp_com.cpp | 32 ++-- .../llm_framework/main_sys/src/zmq_bus.cpp | 48 +++-- projects/llm_framework/main_tts/src/main.cpp | 71 +++++--- projects/llm_framework/main_vlm/src/main.cpp | 93 ++++++---- 30 files changed, 1008 insertions(+), 475 deletions(-) create mode 100644 .clang-format create mode 100644 .gitignore diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..87d39608 --- /dev/null +++ b/.clang-format @@ -0,0 +1,167 @@ +--- +Language: Cpp +BasedOnStyle: Google +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: false +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 0 + - Regex: '^<.*\.h>' + Priority: 1 + SortPriority: 0 + - Regex: '^<.*' + Priority: 2 + SortPriority: 0 + - Regex: '.*' + Priority: 3 + SortPriority: 0 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: true +SortIncludes: false +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c6f9a448 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/settings.json diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index dace7cb4..b141fe8d 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -12,17 +12,20 @@ std::string llm_channel_obj::uart_push_url; llm_channel_obj::llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name) - : unit_name_(unit_name), inference_url_(inference_url) { + : unit_name_(unit_name), inference_url_(inference_url) +{ zmq_url_index_ = -1000; zmq_[-1] = std::make_unique(_publisher_url, ZMQ_PUB); zmq_[-2].reset(); } -llm_channel_obj::~llm_channel_obj() { +llm_channel_obj::~llm_channel_obj() +{ } void llm_channel_obj::subscriber_event_call(const std::function &call, - const std::string &raw) { + const std::string &raw) +{ // SLOGI("object%s", sample_json_str_get(raw, "object").c_str()); // SLOGI("request_id%s", sample_json_str_get(raw, "request_id").c_str()); // SLOGI("work_id%s", sample_json_str_get(raw, "work_id").c_str()); @@ -38,7 +41,8 @@ void llm_channel_obj::subscriber_event_call(const std::function &call) { + const std::function &call) +{ int id_num; std::string subscriber_url; std::regex pattern(R"((\w+)\.(\d+))"); @@ -65,7 +69,8 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, return 0; } -void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) { +void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) +{ int id_num; std::regex pattern(R"((\w+)\.(\d+))"); std::smatch matches; @@ -80,12 +85,14 @@ void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) { if (zmq_.find(id_num) != zmq_.end()) zmq_.erase(id_num); } -void llm_channel_obj::subscriber(const std::string &zmq_url, const std::function &call) { +void llm_channel_obj::subscriber(const std::string &zmq_url, const std::function &call) +{ zmq_url_map_[zmq_url] = zmq_url_index_--; zmq_[zmq_url_map_[zmq_url]] = std::make_unique(zmq_url, ZMQ_SUB, call); } -void llm_channel_obj::stop_subscriber(const std::string &zmq_url) { +void llm_channel_obj::stop_subscriber(const std::string &zmq_url) +{ if (zmq_url.empty()) { zmq_.clear(); zmq_url_map_.clear(); @@ -95,11 +102,13 @@ void llm_channel_obj::stop_subscriber(const std::string &zmq_url) { } } -int llm_channel_obj::send_raw_to_pub(const std::string &raw) { +int llm_channel_obj::send_raw_to_pub(const std::string &raw) +{ return zmq_[-1]->send_data(raw); } -int llm_channel_obj::send_raw_to_usr(const std::string &raw) { +int llm_channel_obj::send_raw_to_usr(const std::string &raw) +{ if (zmq_[-2]) { return zmq_[-2]->send_data(raw); } else { @@ -107,28 +116,33 @@ int llm_channel_obj::send_raw_to_usr(const std::string &raw) { } } -void llm_channel_obj::set_push_url(const std::string &url) { +void llm_channel_obj::set_push_url(const std::string &url) +{ if (output_url_ != url) { output_url_ = url; zmq_[-2].reset(new pzmq(output_url_, ZMQ_PUSH)); } } -void llm_channel_obj::cear_push_url() { +void llm_channel_obj::cear_push_url() +{ zmq_[-2].reset(); } -int llm_channel_obj::send_raw_for_url(const std::string &zmq_url, const std::string &raw) { +int llm_channel_obj::send_raw_for_url(const std::string &zmq_url, const std::string &raw) +{ pzmq _zmq(zmq_url, ZMQ_PUSH); return _zmq.send_data(raw); } -int llm_channel_obj::output_to_uart(const std::string &data) { +int llm_channel_obj::output_to_uart(const std::string &data) +{ return send_raw_for_url(uart_push_url, data); } StackFlow::StackFlow::StackFlow(const std::string &unit_name) - : unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) { + : unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) +{ event_queue_.appendListener(EVENT_NONE, std::bind(&StackFlow::_none_event, this, std::placeholders::_1, std::placeholders::_2)); event_queue_.appendListener(EVENT_PAUSE, @@ -163,7 +177,8 @@ StackFlow::StackFlow::StackFlow(const std::string &unit_name) event_queue_.enqueue(EVENT_SYS_INIT, "", ""); } -StackFlow::~StackFlow() { +StackFlow::~StackFlow() +{ while (1) { auto iteam = llm_task_channel_.begin(); if (iteam == llm_task_channel_.end()) { @@ -177,17 +192,20 @@ StackFlow::~StackFlow() { even_loop_thread_->join(); } -void StackFlow::even_loop() { +void StackFlow::even_loop() +{ while (!exit_flage_.load()) { event_queue_.wait(); event_queue_.process(); } } -void StackFlow::_none_event(const std::string &data1, const std::string &data2) { +void StackFlow::_none_event(const std::string &data1, const std::string &data2) +{ } -void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) { +void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) +{ // serial_zmq_url std::string serial_zmq_url; serial_zmq_url = sys_sql_select("serial_zmq_url"); @@ -201,14 +219,16 @@ void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) { } } -std::string StackFlow::_rpc_setup(const std::string &data) { +std::string StackFlow::_rpc_setup(const std::string &data) +{ SLOGI("_rpc_setup:%s", data.c_str()); event_queue_.enqueue(EVENT_SETUP, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -int StackFlow::setup(const std::string &zmq_url, const std::string &raw) { +int StackFlow::setup(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::setup raw zmq_url:%s raw:%s", zmq_url.c_str(), raw.c_str()); int workid_num = sys_register_unit(unit_name_); std::string work_id = unit_name_ + "." + std::to_string(workid_num); @@ -222,7 +242,8 @@ int StackFlow::setup(const std::string &zmq_url, const std::string &raw) { return 0; } -int StackFlow::setup(const std::string &work_id, const std::string &object, const std::string &data) { +int StackFlow::setup(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::setup"); if (_setup_) { return _setup_(work_id, object, data); @@ -234,13 +255,15 @@ int StackFlow::setup(const std::string &work_id, const std::string &object, cons return -1; } -std::string StackFlow::_rpc_link(const std::string &data) { +std::string StackFlow::_rpc_link(const std::string &data) +{ event_queue_.enqueue(EVENT_LINK, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -void StackFlow::link(const std::string &zmq_url, const std::string &raw) { +void StackFlow::link(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::link raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -251,7 +274,8 @@ void StackFlow::link(const std::string &zmq_url, const std::string &raw) { link(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::link(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::link(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::link"); if (_link_) { _link_(work_id, object, data); @@ -263,13 +287,15 @@ void StackFlow::link(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_unlink(const std::string &data) { +std::string StackFlow::_rpc_unlink(const std::string &data) +{ event_queue_.enqueue(EVENT_UNLINK, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) { +void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::unlink raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -280,7 +306,8 @@ void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) { unlink(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::unlink(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::unlink(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::unlink"); if (_unlink_) { _unlink_(work_id, object, data); @@ -292,13 +319,15 @@ void StackFlow::unlink(const std::string &work_id, const std::string &object, co send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_work(const std::string &data) { +std::string StackFlow::_rpc_work(const std::string &data) +{ event_queue_.enqueue(EVENT_WORK, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -void StackFlow::work(const std::string &zmq_url, const std::string &raw) { +void StackFlow::work(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::work raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -309,7 +338,8 @@ void StackFlow::work(const std::string &zmq_url, const std::string &raw) { work(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::work(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::work(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::work"); if (_work_) { _work_(work_id, object, data); @@ -321,13 +351,15 @@ void StackFlow::work(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_exit(const std::string &data) { +std::string StackFlow::_rpc_exit(const std::string &data) +{ event_queue_.enqueue(EVENT_EXIT, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -int StackFlow::exit(const std::string &zmq_url, const std::string &raw) { +int StackFlow::exit(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::exit raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -341,7 +373,8 @@ int StackFlow::exit(const std::string &zmq_url, const std::string &raw) { return 0; } -int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string &data) { +int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::exit"); if (_exit_) { return _exit_(work_id, object, data); @@ -353,13 +386,15 @@ int StackFlow::exit(const std::string &work_id, const std::string &object, const return 0; } -std::string StackFlow::_rpc_pause(const std::string &data) { +std::string StackFlow::_rpc_pause(const std::string &data) +{ event_queue_.enqueue(EVENT_PAUSE, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -void StackFlow::pause(const std::string &zmq_url, const std::string &raw) { +void StackFlow::pause(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::pause raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -370,7 +405,8 @@ void StackFlow::pause(const std::string &zmq_url, const std::string &raw) { pause(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::pause(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::pause(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::pause"); if (_pause_) { _pause_(work_id, object, data); @@ -382,13 +418,15 @@ void StackFlow::pause(const std::string &work_id, const std::string &object, con send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_taskinfo(const std::string &data) { +std::string StackFlow::_rpc_taskinfo(const std::string &data) +{ event_queue_.enqueue(EVENT_TASKINFO, sample_json_str_get(data, "zmq_com"), sample_unescapeString(sample_json_str_get(data, "raw_data"))); return std::string("None"); } -void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) { +void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) +{ SLOGI("StackFlow::taskinfo raw"); std::string work_id = sample_json_str_get(raw, "work_id"); try { @@ -399,7 +437,8 @@ void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) { taskinfo(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } -void StackFlow::taskinfo(const std::string &work_id, const std::string &object, const std::string &data) { +void StackFlow::taskinfo(const std::string &work_id, const std::string &object, const std::string &data) +{ SLOGI("StackFlow::taskinfo"); if (_taskinfo_) { _taskinfo_(work_id, object, data); @@ -411,7 +450,8 @@ void StackFlow::taskinfo(const std::string &work_id, const std::string &object, send("None", "None", error_body, work_id); } -int StackFlow::sys_register_unit(const std::string &unit_name) { +int StackFlow::sys_register_unit(const std::string &unit_name) +{ int work_id_number; std::string out_port; std::string inference_port; @@ -428,7 +468,8 @@ int StackFlow::sys_register_unit(const std::string &unit_name) { return work_id_number; } -bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) { +bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) +{ std::string _work_id; int _work_id_num; if (work_id.empty()) { @@ -446,14 +487,16 @@ bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) { return false; } -std::string StackFlow::sys_sql_select(const std::string &key) { +std::string StackFlow::sys_sql_select(const std::string &key) +{ std::string val; pzmq _call("sys"); _call.call_rpc_action("sql_select", key, [&val](const std::string &data) { val = data; }); return val; } -void StackFlow::sys_sql_set(const std::string &key, const std::string &val) { +void StackFlow::sys_sql_set(const std::string &key, const std::string &val) +{ nlohmann::json out_body; out_body["key"] = key; out_body["val"] = val; @@ -461,21 +504,23 @@ void StackFlow::sys_sql_set(const std::string &key, const std::string &val) { _call.call_rpc_action("sql_set", out_body.dump(), [](const std::string &data) {}); } -void StackFlow::sys_sql_unset(const std::string &key) { +void StackFlow::sys_sql_unset(const std::string &key) +{ std::string val; pzmq _call("sys"); _call.call_rpc_action("sql_select", key, [](const std::string &data) {}); } -std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, - const std::string &data) { +std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) +{ std::string value; pzmq _call(unit_name); _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); return value; } -void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) { +void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) +{ repeat_callback_fun_mutex_.lock(); const auto call_fun = repeat_callback_fun_[action]; repeat_callback_fun_mutex_.unlock(); @@ -496,7 +541,8 @@ void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) { } } -void StackFlow::repeat_event(int ms, std::function repeat_fun, bool now) { +void StackFlow::repeat_event(int ms, std::function repeat_fun, bool now) +{ repeat_callback_fun_mutex_.lock(); std::string action = std::to_string(repeat_callback_fun_.size() + 1); repeat_callback_fun_[action] = repeat_fun; diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index f7c911fc..df157d5e 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -27,30 +27,34 @@ namespace StackFlows { template class ThreadSafeWrapper { - public: +public: template - ThreadSafeWrapper(Args &&...args) : object(std::make_unique(std::forward(args)...)) { + ThreadSafeWrapper(Args &&...args) : object(std::make_unique(std::forward(args)...)) + { } template - auto access(Func func) -> decltype(func(std::declval())) { + auto access(Func func) -> decltype(func(std::declval())) + { std::lock_guard lock(mutex); return func(*object); } - ThreadSafeWrapper &operator=(const T &newValue) { + ThreadSafeWrapper &operator=(const T &newValue) + { std::lock_guard lock(mutex); *object = newValue; return *this; } - ThreadSafeWrapper &operator=(T &&newValue) { + ThreadSafeWrapper &operator=(T &&newValue) + { std::lock_guard lock(mutex); *object = std::move(newValue); return *this; } - private: +private: std::unique_ptr object; mutable std::mutex mutex; }; @@ -59,12 +63,12 @@ class ThreadSafeWrapper { #define LLM_NONE std::string("None") class llm_channel_obj { - private: +private: std::unordered_map> zmq_; std::atomic zmq_url_index_; std::unordered_map zmq_url_map_; - public: +public: std::string unit_name_; bool enoutput_; bool enstream_; @@ -77,16 +81,20 @@ class llm_channel_obj { llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name); ~llm_channel_obj(); - inline void set_output(bool flage) { + inline void set_output(bool flage) + { enoutput_ = flage; } - inline bool get_output() { + inline bool get_output() + { return enoutput_; } - inline void set_stream(bool flage) { + inline void set_stream(bool flage) + { enstream_ = flage; } - inline bool get_stream() { + inline bool get_stream() + { return enstream_; } void subscriber_event_call(const std::function &call, @@ -100,7 +108,8 @@ class llm_channel_obj { int send_raw_to_pub(const std::string &raw); int send_raw_to_usr(const std::string &raw); template - int output_data(const std::string &object, const T &data, const U &error_msg) { + int output_data(const std::string &object, const T &data, const U &error_msg) + { return output_data(request_id_, work_id_, object, data, error_msg); } void set_push_url(const std::string &url); @@ -110,7 +119,8 @@ class llm_channel_obj { template static int output_data_for_url(const std::string &zmq_url, const std::string &request_id, const std::string &work_id, const std::string &object, const U &data, - const T &error_msg, bool outuart = false) { + const T &error_msg, bool outuart = false) + { nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -130,7 +140,8 @@ class llm_channel_obj { } template int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id = "", - bool outuart = false) { + bool outuart = false) + { nlohmann::json out_body; out_body["request_id"] = request_id_; out_body["work_id"] = work_id.empty() ? work_id_ : work_id; @@ -150,7 +161,8 @@ class llm_channel_obj { } template int output_data(const std::string &request_id, const std::string &work_id, const std::string &object, const T &data, - const U &error_msg) { + const U &error_msg) + { nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -171,8 +183,8 @@ class llm_channel_obj { }; class StackFlow { - private: - protected: +private: +protected: std::string unit_name_; typedef enum { EVENT_NONE = 0, @@ -202,7 +214,7 @@ class StackFlow { void _repeat_loop(const std::string &zmq_url, const std::string &raw); - public: +public: std::string request_id_; std::string out_zmq_url_; @@ -222,7 +234,8 @@ class StackFlow { void _none_event(const std::string &data1, const std::string &data2); template - std::shared_ptr get_channel(T workid) { + std::shared_ptr get_channel(T workid) + { int _work_id_num; if constexpr (std::is_same::value) { _work_id_num = workid; @@ -235,7 +248,8 @@ class StackFlow { } std::string _rpc_setup(const std::string &data); - void _setup(const std::string &zmq_url, const std::string &data) { + void _setup(const std::string &zmq_url, const std::string &data) + { // printf("void _setup run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -245,7 +259,8 @@ class StackFlow { virtual int setup(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_link(const std::string &data); - void _link(const std::string &zmq_url, const std::string &data) { + void _link(const std::string &zmq_url, const std::string &data) + { // printf("void _link run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -255,7 +270,8 @@ class StackFlow { virtual void link(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_unlink(const std::string &data); - void _unlink(const std::string &zmq_url, const std::string &data) { + void _unlink(const std::string &zmq_url, const std::string &data) + { // printf("void _unlink run \n"); request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; @@ -265,7 +281,8 @@ class StackFlow { virtual void unlink(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_exit(const std::string &data); - void _exit(const std::string &zmq_url, const std::string &data) { + void _exit(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) exit(zmq_url, data); @@ -274,7 +291,8 @@ class StackFlow { virtual int exit(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_work(const std::string &data); - void _work(const std::string &zmq_url, const std::string &data) { + void _work(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) work(zmq_url, data); @@ -283,7 +301,8 @@ class StackFlow { virtual void work(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_pause(const std::string &data); - void _pause(const std::string &zmq_url, const std::string &data) { + void _pause(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) pause(zmq_url, data); @@ -292,7 +311,8 @@ class StackFlow { virtual void pause(const std::string &work_id, const std::string &object, const std::string &data); std::string _rpc_taskinfo(const std::string &data); - void _taskinfo(const std::string &zmq_url, const std::string &data) { + void _taskinfo(const std::string &zmq_url, const std::string &data) + { request_id_ = sample_json_str_get(data, "request_id"); out_zmq_url_ = zmq_url; if (status_.load()) taskinfo(zmq_url, data); @@ -305,7 +325,8 @@ class StackFlow { void user_output(const std::string &zmq_url, const std::string &request_id, const std::string &data); template int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id, - const std::string &zmq_url = "") { + const std::string &zmq_url = "") + { nlohmann::json out_body; out_body["request_id"] = request_id_; out_body["work_id"] = work_id; @@ -326,7 +347,8 @@ class StackFlow { } } - void llm_firework_exit() { + void llm_firework_exit() + { } std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); @@ -335,7 +357,8 @@ class StackFlow { void sys_sql_unset(const std::string &key); int sys_register_unit(const std::string &unit_name); template - bool sys_release_unit(T workid) { + bool sys_release_unit(T workid) + { std::string _work_id; int _work_id_num; if constexpr (std::is_same::value) { diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index fde8ecfc..956809c2 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -6,7 +6,8 @@ #include "StackFlowUtil.h" #include -std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { +std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) +{ std::string key_val; std::string format_val; // SLOGD("json_str: %s json_key:%s\n", json_str.c_str(), json_key.c_str()); @@ -81,7 +82,8 @@ std::string StackFlows::sample_json_str_get(const std::string &json_str, const s return key_val; } -int StackFlows::sample_get_work_id_num(const std::string &work_id) { +int StackFlows::sample_get_work_id_num(const std::string &work_id) +{ int a = work_id.find("."); if ((a == std::string::npos) || (a == work_id.length() - 1)) { return WORK_ID_NONE; @@ -89,7 +91,8 @@ int StackFlows::sample_get_work_id_num(const std::string &work_id) { return std::stoi(work_id.substr(a + 1)); } -std::string StackFlows::sample_get_work_id_name(const std::string &work_id) { +std::string StackFlows::sample_get_work_id_name(const std::string &work_id) +{ int a = work_id.find("."); if (a == std::string::npos) { return work_id; @@ -98,11 +101,13 @@ std::string StackFlows::sample_get_work_id_name(const std::string &work_id) { } } -std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &unit_name) { +std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &unit_name) +{ return unit_name + "." + std::to_string(work_id_num); } -std::string StackFlows::sample_escapeString(const std::string &input) { +std::string StackFlows::sample_escapeString(const std::string &input) +{ std::string escaped; for (char c : input) { switch (c) { @@ -132,7 +137,8 @@ std::string StackFlows::sample_escapeString(const std::string &input) { return escaped; } -std::string StackFlows::sample_unescapeString(const std::string &input) { +std::string StackFlows::sample_unescapeString(const std::string &input) +{ std::string unescaped; for (size_t i = 0; i < input.length(); ++i) { if (input[i] == '\\' && i + 1 < input.length()) { @@ -173,7 +179,8 @@ std::string StackFlows::sample_unescapeString(const std::string &input) { } bool StackFlows::decode_stream(const std::string &in, std::string &out, - std::unordered_map &stream_buff) { + std::unordered_map &stream_buff) +{ try { int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); std::string finish = StackFlows::sample_json_str_get(in, "finish"); @@ -310,7 +317,8 @@ static const signed char base64de[] = { 51, }; -static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) { +static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) +{ unsigned int i = 0, j = 0; for (; i < inlen; i++) { @@ -345,7 +353,8 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) return j; } -static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) { +static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) +{ unsigned int i = 0, j = 0; for (; i < inlen; i++) { @@ -380,12 +389,14 @@ static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) return j; } -int StackFlows::decode_base64(const std::string &in, std::string &out) { +int StackFlows::decode_base64(const std::string &in, std::string &out) +{ out.resize(BASE64_DECODE_OUT_SIZE(in.length())); return base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); } -int StackFlows::encode_base64(const std::string &in, std::string &out) { +int StackFlows::encode_base64(const std::string &in, std::string &out) +{ out.resize(BASE64_ENCODE_OUT_SIZE(in.length())); return base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index b334097d..c6511a84 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -16,7 +16,7 @@ #define ZMQ_RPC_CALL (ZMQ_REQ | 0x80) namespace StackFlows { class pzmq { - private: +private: const std::string unix_socket_head_ = "ipc://"; const std::string rpc_url_head_ = "ipc:///tmp/rpc."; void *zmq_ctx_; @@ -29,24 +29,29 @@ class pzmq { std::string zmq_url_; int timeout_; - public: - pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) { +public: + pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) + { } pzmq(const std::string &server) - : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) { + : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) + { } pzmq(const std::string &url, int mode, const std::function &raw_call = nullptr) - : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) { + : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) + { if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call); } - void set_timeout(int ms) { + void set_timeout(int ms) + { timeout_ = ms; } - int get_timeout() { + int get_timeout() + { return timeout_; } - int register_rpc_action(const std::string &action, - const std::function &raw_call) { + int register_rpc_action(const std::string &action, const std::function &raw_call) + { if (zmq_fun_.find(action) != zmq_fun_.end()) { zmq_fun_[action] = raw_call; return 0; @@ -63,7 +68,8 @@ class pzmq { return creat(url); } int call_rpc_action(const std::string &action, const std::string &data, - const std::function &raw_call) { + const std::function &raw_call) + { int ret; zmq_msg_t msg; zmq_msg_init(&msg); @@ -83,7 +89,9 @@ class pzmq { zmq_send(zmq_socket_, data.c_str(), data.length(), 0); } // action - { zmq_msg_recv(&msg, zmq_socket_, 0); } + { + zmq_msg_recv(&msg, zmq_socket_, 0); + } raw_call(std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); } catch (int e) { ret = e; @@ -92,7 +100,8 @@ class pzmq { close_zmq(); return ret; } - int creat(const std::string &url, const std::function &raw_call = nullptr) { + int creat(const std::string &url, const std::function &raw_call = nullptr) + { zmq_url_ = url; do { zmq_ctx_ = zmq_ctx_new(); @@ -137,7 +146,8 @@ class pzmq { } return 0; } - int check_zmq_errno(int code) { + int check_zmq_errno(int code) + { if (code == -1) { // SLOGE("Error occurred: %s", zmq_strerror(errno)); switch (errno) { @@ -162,41 +172,49 @@ class pzmq { } return code; } - int send_data(const std::string &raw) { + int send_data(const std::string &raw) + { return zmq_send(zmq_socket_, raw.c_str(), raw.length(), 0); } - int send_data(const char *raw, int size) { + int send_data(const char *raw, int size) + { return zmq_send(zmq_socket_, raw, size, 0); } - inline int creat_pub(const std::string &url) { + inline int creat_pub(const std::string &url) + { return zmq_bind(zmq_socket_, url.c_str()); } - inline int creat_push(const std::string &url) { + inline int creat_push(const std::string &url) + { int timeout = 3000; zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout, sizeof(timeout)); zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout, sizeof(timeout)); return zmq_connect(zmq_socket_, url.c_str()); } - inline int creat_pull(const std::string &url, const std::function &raw_call) { + inline int creat_pull(const std::string &url, const std::function &raw_call) + { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int subscriber_url(const std::string &url, const std::function &raw_call) { + inline int subscriber_url(const std::string &url, const std::function &raw_call) + { int ret = zmq_connect(zmq_socket_, url.c_str()); zmq_setsockopt(zmq_socket_, ZMQ_SUBSCRIBE, "", 0); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int creat_rep(const std::string &url, const std::function &raw_call) { + inline int creat_rep(const std::string &url, const std::function &raw_call) + { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int creat_req(const std::string &url) { + inline int creat_req(const std::string &url) + { int pos = url.find(unix_socket_head_); if (pos != std::string::npos) { std::string socket_file = url.substr(pos + unix_socket_head_.length()); @@ -208,7 +226,8 @@ class pzmq { zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout_, sizeof(timeout_)); return zmq_connect(zmq_socket_, url.c_str()); } - void zmq_event_loop(const std::function &raw_call) { + void zmq_event_loop(const std::function &raw_call) + { int ret; zmq_pollitem_t items[1]; if (mode_ == ZMQ_PULL) { @@ -258,7 +277,8 @@ class pzmq { zmq_msg_close(&msg); } } - void close_zmq() { + void close_zmq() + { zmq_close(zmq_socket_); zmq_ctx_destroy(zmq_ctx_); int pos = zmq_url_.find(unix_socket_head_); @@ -273,7 +293,8 @@ class pzmq { zmq_socket_ = NULL; zmq_ctx_ = NULL; } - ~pzmq() { + ~pzmq() + { if (!zmq_socket_) { return; } diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 9c834ff9..62d3ee81 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -13,13 +13,14 @@ #include using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ main_exit_flage = 1; } typedef std::function task_callback_t; class llm_task { - private: - public: +private: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -27,11 +28,13 @@ class llm_task { bool enoutput_; bool enstream_; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -52,37 +55,43 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } return 0; } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { std::cout << msg << std::endl; if (out_callback_) out_callback_(std::string("hello"), true); } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - public: - llm_llm() : StackFlow("test") { +public: + llm_llm() : StackFlow("test") + { task_count_ = 1; } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { if (llm_channel->enstream_) { static int count = 0; nlohmann::json data_body; @@ -102,7 +111,8 @@ class llm_llm : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -122,7 +132,8 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -161,7 +172,8 @@ class llm_llm : public StackFlow { } } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { @@ -186,7 +198,8 @@ class llm_llm : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { @@ -202,7 +215,8 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -214,7 +228,8 @@ class llm_llm : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 4ed598f1..fafb6d70 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -24,7 +24,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_asr will be exit!"); main_exit_flage = 1; } @@ -41,12 +42,12 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: +private: sherpa_ncnn::RecognizerConfig mode_config_; std::unique_ptr recognizer_; std::unique_ptr recognizer_stream_; - public: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -62,7 +63,8 @@ class llm_task { std::function pause; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -84,7 +86,8 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -163,7 +166,7 @@ class llm_task { mode_config_.model_config.decoder_bin = base_model + mode_config_.model_config.decoder_bin; mode_config_.model_config.joiner_param = base_model + mode_config_.model_config.joiner_param; mode_config_.model_config.joiner_bin = base_model + mode_config_.model_config.joiner_bin; - recognizer_ = std::make_unique(mode_config_); + recognizer_ = std::make_unique(mode_config_); } catch (...) { SLOGE("config false"); return -3; @@ -171,11 +174,13 @@ class llm_task { return 0; } - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) { + void sys_pcm_on_data(const std::string &raw) + { static int count = 0; if (count < delay_audio_frame_) { buffer_write_char(pcmdata, raw.c_str(), raw.length()); @@ -225,22 +230,26 @@ class llm_task { } } - void kws_awake() { + void kws_awake() + { awake_flage_ = true; } - bool delete_model() { + bool delete_model() + { recognizer_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { ensleep_ = false; awake_flage_ = false; pcmdata = buffer_create(); } - ~llm_task() { + ~llm_task() + { if (recognizer_stream_) { recognizer_stream_.reset(); } @@ -251,14 +260,15 @@ class llm_task { #undef CONFIG_AUTO_SET class llm_asr : public StackFlow { - public: +public: enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_TASK_PAUSE }; - private: +private: int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -273,8 +283,9 @@ class llm_asr : public StackFlow { } } - public: - llm_asr() : StackFlow("asr") { +public: + llm_asr() : StackFlow("asr") + { task_count_ = 1; repeat_event(1000, std::bind(&llm_asr::_load_config, this)); event_queue_.appendListener( @@ -282,7 +293,8 @@ class llm_asr : public StackFlow { } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { std::string tmp_msg1; const std::string *next_data = &data; if (finish) { @@ -304,7 +316,8 @@ class llm_asr : public StackFlow { } } - int decode_wav(const std::string &in, std::string &out) { + int decode_wav(const std::string &in, std::string &out) + { int post = 0; if (in.length() > 10) for (int i = 0; i < in.length() - 4; i++) { @@ -322,13 +335,15 @@ class llm_asr : public StackFlow { return 0; } - int decode_mp3(const std::string &in, std::string &out) { + int decode_mp3(const std::string &in, std::string &out) + { return 0; } void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { std::string tmp_msg1; const std::string *next_data = &data; int ret; @@ -364,7 +379,8 @@ class llm_asr : public StackFlow { llm_task_obj->sys_pcm_on_data((*next_data)); } - void _task_pause(const std::string &work_id, const std::string &data) { + void _task_pause(const std::string &work_id, const std::string &data) + { int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { return; @@ -377,11 +393,13 @@ class llm_asr : public StackFlow { } } - void task_pause(const std::string &work_id, const std::string &data) { + void task_pause(const std::string &work_id, const std::string &data) + { event_queue_.enqueue(EVENT_TASK_PAUSE, work_id, ""); } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + { llm_task_obj->kws_awake(); if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { llm_channel->subscriber(audio_url_, @@ -391,12 +409,14 @@ class llm_asr : public StackFlow { } void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_)); task_work(llm_task_obj, llm_channel); } - void work(const std::string &work_id, const std::string &object, const std::string &data) override { + void work(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -411,7 +431,8 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void pause(const std::string &work_id, const std::string &object, const std::string &data) override { + void pause(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -426,7 +447,8 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -490,7 +512,8 @@ class llm_asr : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::link:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -525,7 +548,8 @@ class llm_asr : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -549,9 +573,10 @@ class llm_asr : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::taskinfo:%s", data.c_str()); - + nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { @@ -576,7 +601,8 @@ class llm_asr : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::exit:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -596,7 +622,8 @@ class llm_asr : public StackFlow { return 0; } - ~llm_asr() { + ~llm_asr() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -613,7 +640,8 @@ class llm_asr : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 8b646145..5cc92682 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -15,7 +15,8 @@ #include "../../../../SDK/components/utilities/include/sample_log.h" int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -31,7 +32,7 @@ static void __sigint(int iSigNo) { using namespace StackFlows; class llm_audio : public StackFlow { - private: +private: enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_QUEUE_PLAY }; std::atomic_bool audio_clear_flage_; std::string sys_pcm_cap_channel = "ipc:///tmp/llm/pcm.cap.socket"; @@ -42,11 +43,13 @@ class llm_audio : public StackFlow { std::unique_ptr audio_cap_thread_; std::mutex ax_play_mtx; - static void on_cap_sample(const char *data, int size) { + static void on_cap_sample(const char *data, int size) + { self->pub_ctx_->send_data((const char *)data, size); } - void hw_queue_play(const std::string &audio_data, const std::string &None) { + void hw_queue_play(const std::string &audio_data, const std::string &None) + { if (audio_clear_flage_) { return; } @@ -55,25 +58,29 @@ class llm_audio : public StackFlow { play_config.bit, audio_data.c_str(), audio_data.length()); } - void hw_play(const std::string &audio_data) { + void hw_play(const std::string &audio_data) + { std::lock_guard guard(ax_play_mtx); ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, play_config.bit, audio_data.c_str(), audio_data.length()); } - void hw_cap() { + void hw_cap() + { ax_cap_start(cap_config.card, cap_config.device, cap_config.volume, cap_config.channel, cap_config.rate, cap_config.bit, llm_audio::on_cap_sample); } - void _play(const std::string &audio_data) { + void _play(const std::string &audio_data) + { if (audio_play_thread_) { _play_stop(); } audio_play_thread_ = std::make_unique(std::bind(&llm_audio::hw_play, this, audio_data)); } - void _play_stop() { + void _play_stop() + { ax_close_play(); if (audio_play_thread_) { audio_play_thread_->join(); @@ -81,14 +88,16 @@ class llm_audio : public StackFlow { } } - void _cap() { + void _cap() + { if (!audio_cap_thread_) { pub_ctx_ = std::make_unique(sys_pcm_cap_channel, ZMQ_PUB); audio_cap_thread_ = std::make_unique(std::bind(&llm_audio::hw_cap, this)); } } - void _cap_stop() { + void _cap_stop() + { ax_close_cap(); if (audio_cap_thread_) { audio_cap_thread_->join(); @@ -97,8 +106,9 @@ class llm_audio : public StackFlow { } } - public: - llm_audio() : StackFlow("audio") { +public: + llm_audio() : StackFlow("audio") + { event_queue_.appendListener( EVENT_QUEUE_PLAY, std::bind(&llm_audio::hw_queue_play, this, std::placeholders::_1, std::placeholders::_2)); setup("", "{\"object\":\"audio.play\",\"data\":{\"None\":\"None\"}}"); @@ -116,7 +126,8 @@ class llm_audio : public StackFlow { rpc_ctx_->register_rpc_action("cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1)); } - int setup(const std::string &zmq_url, const std::string &raw) override { + int setup(const std::string &zmq_url, const std::string &raw) override + { std::string object = sample_json_str_get(raw, "object"); std::string data = sample_json_str_get(raw, "data"); nlohmann::json config_body; @@ -288,7 +299,8 @@ class llm_audio : public StackFlow { return -1; } - bool destream(const std::string &indata, std::string &outdata) { + bool destream(const std::string &indata, std::string &outdata) + { static std::unordered_map _stream_data_buff; int index = std::stoi(sample_json_str_get(indata, "index")); std::string finish = sample_json_str_get(indata, "finish"); @@ -308,7 +320,8 @@ class llm_audio : public StackFlow { return true; } - std::string dewav(const std::string &rawdata) { + std::string dewav(const std::string &rawdata) + { int post = 0; if (rawdata.length() > 10) for (int i = 0; i < rawdata.length() - 4; i++) { @@ -326,15 +339,18 @@ class llm_audio : public StackFlow { return std::string("playwav"); } - std::string demp3(const std::string &rawdata) { + std::string demp3(const std::string &rawdata) + { return LLM_NONE; } - std::string depcm(const std::string &rawdata) { + std::string depcm(const std::string &rawdata) + { return LLM_NONE; } - std::string parse_data(const std::string &object, const std::string &data) { + std::string parse_data(const std::string &object, const std::string &data) + { std::string _data; bool s = (object.find("stream") == std::string::npos) ? false : true; bool wav = (object.find("wav") == std::string::npos) ? false : true; @@ -354,7 +370,8 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string play(const std::string &rawdata) { + std::string play(const std::string &rawdata) + { if (rawdata.size() < 3) return LLM_NONE; if ((rawdata[0] == '{') && (rawdata[rawdata.size() - 1] == '}')) { std::string src_data = sample_unescapeString(sample_json_str_get(rawdata, "raw_data")); @@ -365,13 +382,15 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string enqueue_play(const std::string &rawdata) { + std::string enqueue_play(const std::string &rawdata) + { audio_clear_flage_ = false; event_queue_.enqueue(EVENT_QUEUE_PLAY, rawdata, ""); return LLM_NONE; } - std::string audio_status(const std::string &rawdata) { + std::string audio_status(const std::string &rawdata) + { if (rawdata == "play") { if (ax_play_status()) { return std::string("None"); @@ -403,17 +422,20 @@ class llm_audio : public StackFlow { } } - std::string play_stop(const std::string &rawdata) { + std::string play_stop(const std::string &rawdata) + { _play_stop(); return LLM_NONE; } - std::string queue_play_stop(const std::string &rawdata) { + std::string queue_play_stop(const std::string &rawdata) + { audio_clear_flage_ = true; return LLM_NONE; } - std::string cap(const std::string &rawdata) { + std::string cap(const std::string &rawdata) + { if (cap_status_ == 0) { _cap(); } @@ -421,7 +443,8 @@ class llm_audio : public StackFlow { return sys_pcm_cap_channel; } - std::string cap_stop(const std::string &rawdata) { + std::string cap_stop(const std::string &rawdata) + { if (cap_status_ > 0) { cap_status_--; if (cap_status_ == 0) { @@ -431,20 +454,23 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string cap_stop_all(const std::string &rawdata) { + std::string cap_stop_all(const std::string &rawdata) + { cap_status_ = 0; _cap_stop(); return LLM_NONE; } - ~llm_audio() { + ~llm_audio() + { _play_stop(); _cap_stop(); } }; llm_audio *llm_audio::self; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_audio/src/sample_audio.c b/projects/llm_framework/main_audio/src/sample_audio.c index 1a7902ae..400d46be 100644 --- a/projects/llm_framework/main_audio/src/sample_audio.c +++ b/projects/llm_framework/main_audio/src/sample_audio.c @@ -41,7 +41,8 @@ static AX_S32 gplayLoopExit = 0; static AX_S32 gcapLoopExit = 0; -int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) { +int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) +{ switch (bits) { case 32: *format = AX_AUDIO_BIT_WIDTH_32; @@ -60,20 +61,23 @@ int BitsToFormat(unsigned int bits, AX_AUDIO_BIT_WIDTH_E *format) { return 0; } -int IsUpTalkVqeEnabled(const AX_AP_UPTALKVQE_ATTR_T *pstVqeAttr) { +int IsUpTalkVqeEnabled(const AX_AP_UPTALKVQE_ATTR_T *pstVqeAttr) +{ int ret = ((pstVqeAttr->stAecCfg.enAecMode != AX_AEC_MODE_DISABLE) || (pstVqeAttr->stNsCfg.bNsEnable != AX_FALSE) || (pstVqeAttr->stAgcCfg.bAgcEnable != AX_FALSE)); return ret; } -int IsDnVqeEnabled(const AX_AP_DNVQE_ATTR_T *pstVqeAttr) { +int IsDnVqeEnabled(const AX_AP_DNVQE_ATTR_T *pstVqeAttr) +{ int ret = ((pstVqeAttr->stNsCfg.bNsEnable != AX_FALSE) || (pstVqeAttr->stAgcCfg.bAgcEnable != AX_FALSE)); return ret; } AX_AUDIO_SAMPLE_CONFIG_t play_config; void ax_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data, - int size) { + int size) +{ gplayLoopExit = 0; int ret = 0; AX_AUDIO_BIT_WIDTH_E format = AX_AUDIO_BIT_WIDTH_16; @@ -264,17 +268,20 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, return; } -void ax_close_play() { +void ax_close_play() +{ gplayLoopExit = 1; } -int ax_play_status() { +int ax_play_status() +{ return gplayLoopExit; } AX_AUDIO_SAMPLE_CONFIG_t cap_config; void ax_cap_start(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, - AUDIOCallback callback) { + AUDIOCallback callback) +{ int ret; AX_AUDIO_BIT_WIDTH_E format; unsigned int totalFrames = 0; @@ -458,10 +465,12 @@ void ax_cap_start(unsigned int card, unsigned int device, float Volume, int chan gcapLoopExit = 2; } -void ax_close_cap() { +void ax_close_cap() +{ gcapLoopExit = 1; } -int ax_cap_status() { +int ax_cap_status() +{ return gcapLoopExit; } \ No newline at end of file diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 8e53258b..d2135f25 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -25,7 +25,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_kws will be exit!"); main_exit_flage = 1; } @@ -40,12 +41,12 @@ static std::string base_model_config_path_; mode_config_.key = obj[#key]; class llm_task { - private: +private: sherpa_onnx::KeywordSpotterConfig mode_config_; std::unique_ptr spotter_; std::unique_ptr spotter_stream_; - public: +public: std::string model_; std::string response_format_; std::vector inputs_; @@ -61,7 +62,8 @@ class llm_task { std::function out_callback_; std::function play_awake_wav; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -89,7 +91,8 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -195,11 +198,13 @@ class llm_task { return 0; } - void set_output(std::function out_callback) { + void set_output(std::function out_callback) + { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) { + void sys_pcm_on_data(const std::string &raw) + { static int count = 0; if (count < delay_audio_frame_) { buffer_write_char(pcmdata, raw.c_str(), raw.length()); @@ -234,16 +239,19 @@ class llm_task { } } - bool delete_model() { + bool delete_model() + { spotter_.reset(); return true; } - llm_task(const std::string &workid) : audio_flage_(false) { + llm_task(const std::string &workid) : audio_flage_(false) + { pcmdata = buffer_create(); } - ~llm_task() { + ~llm_task() + { if (spotter_stream_) { spotter_stream_.reset(); } @@ -253,11 +261,12 @@ class llm_task { #undef CONFIG_AUTO_SET class llm_kws : public StackFlow { - private: +private: int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -272,13 +281,15 @@ class llm_kws : public StackFlow { } } - public: - llm_kws() : StackFlow("kws") { +public: + llm_kws() : StackFlow("kws") + { task_count_ = 1; repeat_event(1000, std::bind(&llm_kws::_load_config, this)); } - void play_awake_wav(const std::string &wav_file) { + void play_awake_wav(const std::string &wav_file) + { FILE *fp = fopen(wav_file.c_str(), "rb"); if (!fp) { printf("Open %s failed!\n", wav_file.c_str()); @@ -303,14 +314,16 @@ class llm_kws : public StackFlow { } } - void task_pause(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_pause(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + { if (llm_task_obj->audio_flage_) { if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_); llm_task_obj->audio_flage_ = false; } } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) { + void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + { if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); @@ -318,7 +331,8 @@ class llm_kws : public StackFlow { } } - void work(const std::string &work_id, const std::string &object, const std::string &data) override { + void work(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -333,7 +347,8 @@ class llm_kws : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void pause(const std::string &work_id, const std::string &object, const std::string &data) override { + void pause(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_asr::work:%s", data.c_str()); nlohmann::json error_body; @@ -350,7 +365,8 @@ class llm_kws : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { std::string tmp_msg1; const std::string *next_data = &data; int ret; @@ -370,7 +386,8 @@ class llm_kws : public StackFlow { llm_task_obj->sys_pcm_on_data((*next_data)); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -426,7 +443,8 @@ class llm_kws : public StackFlow { } } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_kws::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -452,7 +470,8 @@ class llm_kws : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_kws::exit:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -471,11 +490,13 @@ class llm_kws : public StackFlow { return 0; } - ~llm_kws() { + ~llm_kws() + { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 568611a6..e94cc53e 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -17,7 +17,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -34,8 +35,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: LLMAttrType mode_config_; std::unique_ptr lLaMa_; std::string model_; @@ -48,11 +49,13 @@ class llm_task { std::atomic_bool tokenizer_server_flage_; unsigned int port_ = 8080; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -76,7 +79,8 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -122,21 +126,19 @@ class llm_task { pid_t pid = fork(); if (pid == 0) { execl("/usr/bin/python3", "python3", - ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), - "--host", "localhost", - "--port", std::to_string(port_).c_str(), - "--model_id", (base_model + "tokenizer").c_str(), - "--content", ("'" + prompt_ + "'").c_str(), - nullptr); + ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), "--host", "localhost", + "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), + "--content", ("'" + prompt_ + "'").c_str(), nullptr); perror("execl failed"); exit(1); } tokenizer_server_flage_ = true; - SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); + SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), + (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); std::this_thread::sleep_for(std::chrono::seconds(10)); } } else { - mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; + mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; } SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str()); mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; @@ -159,7 +161,8 @@ class llm_task { return 0; } - std::string prompt_complete(const std::string &input) { + std::string prompt_complete(const std::string &input) + { std::ostringstream oss_prompt; switch (mode_config_.tokenizer_type) { case TKT_LLaMa: @@ -180,11 +183,12 @@ class llm_task { oss_prompt << input; break; } - SLOGI("prompt_complete:%s",oss_prompt.str().c_str()); + SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); return oss_prompt.str(); } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { try { std::string out = lLaMa_->Run(prompt_complete(msg)); if (out_callback_) out_callback_(out, true); @@ -193,31 +197,36 @@ class llm_task { } } - bool pause() { + bool pause() + { lLaMa_->Stop(); return true; } - bool delete_model() { + bool delete_model() + { lLaMa_->Deinit(); lLaMa_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; #undef CONFIG_AUTO_SET class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -232,14 +241,16 @@ class llm_llm : public StackFlow { } } - public: - llm_llm() : StackFlow("llm") { +public: + llm_llm() : StackFlow("llm") + { task_count_ = 2; repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -262,7 +273,8 @@ class llm_llm : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -283,7 +295,8 @@ class llm_llm : public StackFlow { } void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -294,11 +307,13 @@ class llm_llm : public StackFlow { } void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { llm_task_obj->lLaMa_->Stop(); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -357,7 +372,8 @@ class llm_llm : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -390,7 +406,8 @@ class llm_llm : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -414,7 +431,8 @@ class llm_llm : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::taskinfo:%s", data.c_str()); // int ret = 0; nlohmann::json req_body; @@ -441,7 +459,8 @@ class llm_llm : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::exit:%s", data.c_str()); nlohmann::json error_body; @@ -459,7 +478,8 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -471,7 +491,8 @@ class llm_llm : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_llm_tokenizer/_tokenizer.py b/projects/llm_framework/main_llm_tokenizer/_tokenizer.py index c4c18b30..57d9ac1f 100644 --- a/projects/llm_framework/main_llm_tokenizer/_tokenizer.py +++ b/projects/llm_framework/main_llm_tokenizer/_tokenizer.py @@ -31,8 +31,6 @@ def do_GET(self): self.end_headers() if self.path == "/bos_id": bos_id = token_obj.bos_id - # print(bos_id) - # to json if bos_id is None: msg = json.dumps({"bos_id": -1}) else: @@ -126,7 +124,7 @@ def rpc_forever(): del url_path_map[key] zmq_socket.send(val.encode('utf-8')) except: - pass + zmq_socket.send('error'.encode('utf-8')) if __name__ == "__main__": rpc_forever() diff --git a/projects/llm_framework/main_llm_tokenizer/src/main.cpp b/projects/llm_framework/main_llm_tokenizer/src/main.cpp index e10e6212..78e648f3 100644 --- a/projects/llm_framework/main_llm_tokenizer/src/main.cpp +++ b/projects/llm_framework/main_llm_tokenizer/src/main.cpp @@ -6,7 +6,8 @@ #include #include -int main() { +int main() +{ if (access("./_tokenizer.py", F_OK) == 0) { char *args[] = {"python3", "./_tokenizer.py", NULL}; if (execvp("python3", args) == -1) { diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index c346e7ad..9c736322 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -27,7 +27,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -51,7 +52,8 @@ typedef struct { float noise_scale_w = 0.6f; float sdp_ratio = 0.2f; - float get_length_scale() { + float get_length_scale() + { return (float)(length_scale / spacker_speed); } } melotts_config; @@ -65,8 +67,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: melotts_config mode_config_; std::unique_ptr encoder_; std::unique_ptr decoder_; @@ -84,7 +86,8 @@ class llm_task { bool enaudio_; int awake_delay_ = 1000; - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -108,7 +111,8 @@ class llm_task { return false; } - std::vector intersperse(const std::vector &lst, int item) { + std::vector intersperse(const std::vector &lst, int item) + { std::vector result(lst.size() * 2 + 1, item); for (size_t i = 1; i < result.size(); i += 2) { result[i] = lst[i / 2]; @@ -116,7 +120,8 @@ class llm_task { return result; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -192,12 +197,14 @@ class llm_task { return 0; } - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } void resample_audio(float *input_buffer, int input_length, float *output_buffer, int *output_length, - double src_ratio) { + double src_ratio) + { SRC_STATE *src_state; int error; src_state = src_new(SRC_SINC_FASTEST, 1, &error); @@ -222,7 +229,8 @@ class llm_task { src_delete(src_state); } - bool TTS(const std::string &msg_str) { + bool TTS(const std::string &msg_str) + { try { std::vector phones_bef, tones_bef; lexicon_->convert(msg_str, phones_bef, tones_bef); @@ -283,7 +291,8 @@ class llm_task { return false; } - std::vector split(const std::string &s, char delim) { + std::vector split(const std::string &s, char delim) + { std::vector result; std::stringstream ss(s); std::string item; @@ -293,7 +302,8 @@ class llm_task { return result; } - void _ax_init() { + void _ax_init() + { if (!ax_init_flage_) { int ret = AX_SYS_Init(); if (0 != ret) { @@ -309,7 +319,8 @@ class llm_task { ax_init_flage_++; } - void _ax_deinit() { + void _ax_deinit() + { if (ax_init_flage_ > 0) { --ax_init_flage_; if (!ax_init_flage_) { @@ -319,12 +330,14 @@ class llm_task { } } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { enaudio_ = true; _ax_init(); } - ~llm_task() { + ~llm_task() + { _ax_deinit(); } }; @@ -332,10 +345,11 @@ int llm_task::ax_init_flage_ = 0; #undef CONFIG_AUTO_SET class llm_tts : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -350,14 +364,16 @@ class llm_tts : public StackFlow { } } - public: - llm_tts() : StackFlow("melotts") { +public: + llm_tts() : StackFlow("melotts") + { task_count_ = 1; repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -381,7 +397,8 @@ class llm_tts : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { if (data.empty() || (data == "None")) return; static std::string faster_stream_buff; nlohmann::json error_body; @@ -436,7 +453,8 @@ class llm_tts : public StackFlow { } void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { @@ -452,7 +470,8 @@ class llm_tts : public StackFlow { } } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -511,7 +530,8 @@ class llm_tts : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -546,7 +566,8 @@ class llm_tts : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -573,7 +594,8 @@ class llm_tts : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_melotts::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -599,7 +621,8 @@ class llm_tts : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::exit:%s", data.c_str()); nlohmann::json error_body; @@ -617,7 +640,8 @@ class llm_tts : public StackFlow { return 0; } - ~llm_tts() { + ~llm_tts() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -631,7 +655,8 @@ class llm_tts : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_sys/include/event_loop.h b/projects/llm_framework/main_sys/include/event_loop.h index d2dbcc85..26d06101 100644 --- a/projects/llm_framework/main_sys/include/event_loop.h +++ b/projects/llm_framework/main_sys/include/event_loop.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #include #include diff --git a/projects/llm_framework/main_sys/include/remote_action.h b/projects/llm_framework/main_sys/include/remote_action.h index 3c894df9..3f4a8fe0 100644 --- a/projects/llm_framework/main_sys/include/remote_action.h +++ b/projects/llm_framework/main_sys/include/remote_action.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #if __cplusplus diff --git a/projects/llm_framework/main_sys/include/remote_server.h b/projects/llm_framework/main_sys/include/remote_server.h index f3c7b225..93f815b0 100644 --- a/projects/llm_framework/main_sys/include/remote_server.h +++ b/projects/llm_framework/main_sys/include/remote_server.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #if __cplusplus diff --git a/projects/llm_framework/main_sys/include/serial.h b/projects/llm_framework/main_sys/include/serial.h index 2ac12314..eb5a7f17 100644 --- a/projects/llm_framework/main_sys/include/serial.h +++ b/projects/llm_framework/main_sys/include/serial.h @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #pragma once #include "all.h" diff --git a/projects/llm_framework/main_sys/include/zmq_bus.h b/projects/llm_framework/main_sys/include/zmq_bus.h index eb07db76..1542b7f2 100644 --- a/projects/llm_framework/main_sys/include/zmq_bus.h +++ b/projects/llm_framework/main_sys/include/zmq_bus.h @@ -21,10 +21,10 @@ void zmq_bus_stop_work(); using namespace StackFlows; class unit_data { - private: +private: std::unique_ptr user_inference_chennal_; - public: +public: std::string work_id; std::string input_url; std::string output_url; @@ -41,14 +41,14 @@ int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_s void zmq_com_send(int com_id, const std::string &out_str); class zmq_bus_com { - protected: +protected: std::string _zmq_url; int exit_flage; int err_count; int _port; std::unique_ptr reace_data_event_thread; - public: +public: std::unique_ptr user_chennal_; zmq_bus_com(); void work(const std::string &zmq_url_format, int port); diff --git a/projects/llm_framework/main_sys/src/config.cpp b/projects/llm_framework/main_sys/src/config.cpp index d3ef2ee6..457eb984 100644 --- a/projects/llm_framework/main_sys/src/config.cpp +++ b/projects/llm_framework/main_sys/src/config.cpp @@ -1,15 +1,16 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #include #include #include "all.h" #include "json.hpp" -void load_default_config() { +void load_default_config() +{ std::ifstream file("./sys_config.json"); if (!file.is_open()) { return; diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 32d26dd0..7c6fa0ac 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -36,7 +36,8 @@ #include "remote_action.h" void usr_print_error(const std::string &request_id, const std::string &work_id, const std::string &error_msg, - int zmq_out) { + int zmq_out) +{ nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -49,7 +50,8 @@ void usr_print_error(const std::string &request_id, const std::string &work_id, } template -void usr_out(const std::string &request_id, const std::string &work_id, const T &data, int zmq_out) { +void usr_out(const std::string &request_id, const std::string &work_id, const T &data, int zmq_out) +{ nlohmann::json out_body; out_body["request_id"] = request_id; out_body["work_id"] = work_id; @@ -65,13 +67,15 @@ void usr_out(const std::string &request_id, const std::string &work_id, const T zmq_com_send(zmq_out, out); } -int sys_ping(int com_id, const nlohmann::json &json_obj) { +int sys_ping(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"\"}", com_id); return out; } -void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) { +void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) +{ SAFE_SETTING("config_serial_baud", (int)json_obj["data"]["baud"]); SAFE_SETTING("config_serial_data_bits", (int)json_obj["data"]["data_bits"]); SAFE_SETTING("config_serial_stop_bits", (int)json_obj["data"]["stop_bits"]); @@ -82,14 +86,16 @@ void _sys_uartsetup(int com_id, const nlohmann::json &json_obj) { serial_work(); } -int sys_uartsetup(int com_id, const nlohmann::json &json_obj) { +int sys_uartsetup(int com_id, const nlohmann::json &json_obj) +{ int out = 0; std::thread t(_sys_uartsetup, com_id, json_obj); t.detach(); return out; } -void get_memory_info(unsigned long *total_memory, unsigned long *free_memory) { +void get_memory_info(unsigned long *total_memory, unsigned long *free_memory) +{ FILE *meminfo = fopen("/proc/meminfo", "r"); if (meminfo == NULL) { perror("fopen"); @@ -113,7 +119,8 @@ struct cpu_use_t { long time; }; -int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) { +int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) +{ unsigned long one; unsigned long two; unsigned long temp; @@ -154,14 +161,16 @@ int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) { return 0; } -int sys_hwinfo(int com_id, const nlohmann::json &json_obj) { +int sys_hwinfo(int com_id, const nlohmann::json &json_obj) +{ int out = 0; std::thread t(_sys_hwinfo, com_id, json_obj); t.detach(); return out; } -int sys_lsmode(int com_id, const nlohmann::json &json_obj) { +int sys_lsmode(int com_id, const nlohmann::json &json_obj) +{ int out; nlohmann::json out_body; int stream = false; @@ -220,7 +229,8 @@ int sys_lsmode(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_lstask(int com_id, const nlohmann::json &json_obj) { +int sys_lstask(int com_id, const nlohmann::json &json_obj) +{ int out; sys_lstask_err_1: @@ -229,7 +239,8 @@ int sys_lstask(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_push(int com_id, const nlohmann::json &json_obj) { +int sys_push(int com_id, const nlohmann::json &json_obj) +{ int out; nlohmann::json out_body; out_body["request_id"] = json_obj["request_id"]; @@ -279,7 +290,8 @@ int sys_push(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_pull(int com_id, const nlohmann::json &json_obj) { +int sys_pull(int com_id, const nlohmann::json &json_obj) +{ int out; sys_pull_err_1: @@ -288,7 +300,8 @@ int sys_pull(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_update(int com_id, const nlohmann::json &json_obj) { +int sys_update(int com_id, const nlohmann::json &json_obj) +{ int out; const std::string command = "find /mnt -name \"llm_update_*.deb\" >> /tmp/find_update_file_out.txt 2>&1 "; out = system(command.c_str()); @@ -311,7 +324,8 @@ int sys_update(int com_id, const nlohmann::json &json_obj) { "{\"code\":-10, \"message\":\"Not available at the moment.\"}", com_id); return out; } -int sys_upgrade(int com_id, const nlohmann::json &json_obj) { +int sys_upgrade(int com_id, const nlohmann::json &json_obj) +{ int out; std::string version; try { @@ -336,8 +350,8 @@ int sys_upgrade(int com_id, const nlohmann::json &json_obj) { return out; } -int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std::string bashcmd, int stream, - int base64) { +int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std::string bashcmd, int stream, int base64) +{ int out; int stream_size = 512; /****************************************************/ @@ -404,7 +418,8 @@ int _sys_bashexec(int com_id, std::string request_id, std::string work_id, std:: return out; } -int sys_bashexec(int com_id, const nlohmann::json &json_obj) { +int sys_bashexec(int com_id, const nlohmann::json &json_obj) +{ std::string request_id = json_obj["request_id"]; std::string work_id = json_obj["work_id"]; std::string bashcmd; @@ -438,7 +453,8 @@ int sys_bashexec(int com_id, const nlohmann::json &json_obj) { return 0; } -int sys_reset(int com_id, const nlohmann::json &json_obj) { +int sys_reset(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"llm server restarting ...\"}", com_id); @@ -449,13 +465,15 @@ int sys_reset(int com_id, const nlohmann::json &json_obj) { return out; } -int sys_version(int com_id, const nlohmann::json &json_obj) { +int sys_version(int com_id, const nlohmann::json &json_obj) +{ usr_out(json_obj["request_id"], json_obj["work_id"], std::string("v1.2"), com_id); int out = 0; return out; } -int sys_reboot(int com_id, const nlohmann::json &json_obj) { +int sys_reboot(int com_id, const nlohmann::json &json_obj) +{ int out = 0; usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"rebooting ...\"}", com_id); usleep(200000); @@ -463,7 +481,8 @@ int sys_reboot(int com_id, const nlohmann::json &json_obj) { return out; } -void server_work() { +void server_work() +{ key_sql["sys.ping"] = sys_ping; key_sql["sys.lsmode"] = sys_lsmode; key_sql["sys.lstask"] = sys_lstask; @@ -476,14 +495,16 @@ void server_work() { key_sql["sys.uartsetup"] = sys_uartsetup; key_sql["sys.reset"] = sys_reset; key_sql["sys.reboot"] = sys_reboot; - key_sql["sys.version"] = sys_version; + key_sql["sys.version"] = sys_version; } -void server_stop_work() { +void server_stop_work() +{ } typedef int (*sys_fun_call)(int, const nlohmann::json &); -void unit_action_match(int com_id, const std::string &json_str) { +void unit_action_match(int com_id, const std::string &json_str) +{ int ret; std::string out_str; nlohmann::json req_body; @@ -515,7 +536,7 @@ void unit_action_match(int com_id, const std::string &json_str) { if ((work_id_fragment.size() > 0) && (work_id_fragment[0] == "sys")) { std::string unit_action = "sys." + action; - sys_fun_call call_fun = NULL; + sys_fun_call call_fun = NULL; SAFE_READING(call_fun, sys_fun_call, unit_action); if (call_fun) call_fun(com_id, req_body); diff --git a/projects/llm_framework/main_sys/src/main.cpp b/projects/llm_framework/main_sys/src/main.cpp index d48e9a95..84765bf4 100644 --- a/projects/llm_framework/main_sys/src/main.cpp +++ b/projects/llm_framework/main_sys/src/main.cpp @@ -31,12 +31,14 @@ std::string zmq_s_format; std::string zmq_c_format; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ printf("llm_sys will be exit!\n"); main_exit_flage = 1; } -void get_run_config() { +void get_run_config() +{ key_sql["config_serial_dev"] = std::string("/dev/ttyS1"); key_sql["config_serial_baud"] = 115200; key_sql["config_serial_data_bits"] = 8; @@ -72,7 +74,8 @@ void get_run_config() { reset_init = 1; \ } -void uart_reset_check() { +void uart_reset_check() +{ uart_t uart_parm; uart_parm.baud = any_cast(key_sql["config_serial_baud"]); uart_parm.data_bits = any_cast(key_sql["config_serial_data_bits"]); @@ -126,7 +129,8 @@ void tcp_work(); void tcp_stop_work(); -void all_work() { +void all_work() +{ zmq_s_format = any_cast(key_sql["config_zmq_s_format"]); zmq_c_format = any_cast(key_sql["config_zmq_c_format"]); server_work(); @@ -139,7 +143,8 @@ void all_work() { if (enable_tcp) tcp_work(); } -void all_stop_work() { +void all_stop_work() +{ int enable_tcp = 0; SAFE_READING(enable_tcp, int, "config_enable_tcp"); @@ -150,10 +155,12 @@ void all_stop_work() { zmq_bus_stop_work(); } -void all_work_check() { +void all_work_check() +{ } -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_sys/src/remote_action.cpp b/projects/llm_framework/main_sys/src/remote_action.cpp index 56854e2d..a6e99c84 100644 --- a/projects/llm_framework/main_sys/src/remote_action.cpp +++ b/projects/llm_framework/main_sys/src/remote_action.cpp @@ -14,7 +14,8 @@ using namespace StackFlows; int unit_call_timeout; -int remote_call(int com_id, const std::string &json_str) { +int remote_call(int com_id, const std::string &json_str) +{ std::string work_id = sample_json_str_get(json_str, "work_id"); std::string work_unit = work_id.substr(0, work_id.find(".")); std::string action = sample_json_str_get(json_str, "action"); @@ -29,9 +30,11 @@ int remote_call(int com_id, const std::string &json_str) { return clent.call_rpc_action(action, send_data, [](const std::string &val) {}); } -void remote_action_work() { +void remote_action_work() +{ SAFE_READING(unit_call_timeout, int, "config_unit_call_timeout"); } -void remote_action_stop_work() { +void remote_action_stop_work() +{ } diff --git a/projects/llm_framework/main_sys/src/remote_server.cpp b/projects/llm_framework/main_sys/src/remote_server.cpp index 65c201b9..18ff306a 100644 --- a/projects/llm_framework/main_sys/src/remote_server.cpp +++ b/projects/llm_framework/main_sys/src/remote_server.cpp @@ -23,21 +23,25 @@ int port_list_start; std::vector port_list; std::unique_ptr sys_rpc_server_; -std::string sys_sql_select(const std::string &key) { +std::string sys_sql_select(const std::string &key) +{ std::string out; SAFE_READING(out, std::string, key); return out; } -void sys_sql_set(const std::string &key, const std::string &val) { +void sys_sql_set(const std::string &key, const std::string &val) +{ SAFE_SETTING(key, val); } -void sys_sql_unset(const std::string &key) { +void sys_sql_unset(const std::string &key) +{ SAFE_ERASE(key); } -unit_data *sys_allocate_unit(const std::string &unit) { +unit_data *sys_allocate_unit(const std::string &unit) +{ unit_data *unit_p = new unit_data(); { unit_p->port_ = work_id_number_counter++; @@ -112,7 +116,8 @@ unit_data *sys_allocate_unit(const std::string &unit) { return unit_p; } -int sys_release_unit(const std::string &unit) { +int sys_release_unit(const std::string &unit) +{ unit_data *unit_p = NULL; SAFE_READING(unit_p, unit_data *, unit); if (NULL == unit_p) { @@ -133,21 +138,25 @@ int sys_release_unit(const std::string &unit) { return 0; } -char const *c_sys_sql_select(char const *key) { +char const *c_sys_sql_select(char const *key) +{ static std::string val; val = sys_sql_select(key); return val.c_str(); } -void c_sys_sql_set(char const *key, char const *val) { +void c_sys_sql_set(char const *key, char const *val) +{ sys_sql_set(key, val); } -void c_sys_sql_unset(char const *key) { +void c_sys_sql_unset(char const *key) +{ sys_sql_unset(key); } -void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, int *work_id) { +void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, int *work_id) +{ unit_data *unit_p = sys_allocate_unit(unit); strcpy(input_url, unit_p->inference_url.c_str()); strcpy(output_url, unit_p->output_url.c_str()); @@ -157,15 +166,18 @@ void c_sys_allocate_unit(char const *unit, char *input_url, char *output_url, in sscanf(unit_p->work_id.c_str(), work_id_format.c_str(), work_id); } -int c_sys_release_unit(char const *unit) { +int c_sys_release_unit(char const *unit) +{ return sys_release_unit(unit); } -std::string rpc_sql_select(const std::string &raw) { +std::string rpc_sql_select(const std::string &raw) +{ return sys_sql_select(raw); } -std::string rpc_allocate_unit(const std::string &raw) { +std::string rpc_allocate_unit(const std::string &raw) +{ unit_data *unit_info = sys_allocate_unit(raw); std::string retval = "{\"work_id_number\":"; retval += std::to_string(unit_info->port_); @@ -177,12 +189,14 @@ std::string rpc_allocate_unit(const std::string &raw) { return retval; } -std::string rpc_release_unit(const std::string &raw) { +std::string rpc_release_unit(const std::string &raw) +{ sys_release_unit(raw); return "Success"; } -std::string rpc_sql_set(const std::string &raw) { +std::string rpc_sql_set(const std::string &raw) +{ std::string key = sample_json_str_get(raw, "key"); std::string val = sample_json_str_get(raw, "val"); if (key.empty()) return "False"; @@ -190,12 +204,14 @@ std::string rpc_sql_set(const std::string &raw) { return "Success"; } -std::string rpc_sql_unset(const std::string &raw) { +std::string rpc_sql_unset(const std::string &raw) +{ sys_sql_unset(raw); return "Success"; } -void remote_server_work() { +void remote_server_work() +{ int port_list_end; SAFE_READING(work_id_number_counter, int, "config_work_id"); SAFE_READING(port_list_start, int, "config_zmq_min_port"); @@ -210,6 +226,7 @@ void remote_server_work() { sys_rpc_server_->register_rpc_action("sql_unset", std::bind(rpc_sql_unset, std::placeholders::_1)); } -void remote_server_stop_work() { +void remote_server_stop_work() +{ sys_rpc_server_.reset(); } \ No newline at end of file diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 74a4164a..871d90f5 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -26,11 +26,12 @@ #include class serial_com : public zmq_bus_com { - private: +private: int uart_fd; - public: - serial_com(uart_t *uart_parm, const std::string &dev_name) : zmq_bus_com() { +public: + serial_com(uart_t *uart_parm, const std::string &dev_name) : zmq_bus_com() + { uart_fd = linux_uart_init((char *)dev_name.c_str(), uart_parm); if (uart_fd <= 0) { SLOGE("open %s false!", dev_name.c_str()); @@ -38,12 +39,14 @@ class serial_com : public zmq_bus_com { } } - void send_data(const std::string &data) { + void send_data(const std::string &data) + { SLOGD("serial send:%s", data.c_str()); linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); } - void reace_data_event() { + void reace_data_event() + { std::string json_str; int flage = 0; fd_set readfds; @@ -83,12 +86,14 @@ class serial_com : public zmq_bus_com { } } - void stop() { + void stop() + { zmq_bus_com::stop(); linux_uart_deinit(uart_fd); } - - ~serial_com() { + + ~serial_com() + { if (exit_flage) { stop(); } @@ -96,7 +101,8 @@ class serial_com : public zmq_bus_com { }; std::unique_ptr serial_con_; -void serial_work() { +void serial_work() +{ uart_t uart_parm; std::string dev_name; int port; @@ -132,6 +138,7 @@ void serial_work() { sync(); } -void serial_stop_work() { +void serial_stop_work() +{ serial_con_.reset(); } diff --git a/projects/llm_framework/main_sys/src/tcp_com.cpp b/projects/llm_framework/main_sys/src/tcp_com.cpp index 16115ea5..2ba0f368 100644 --- a/projects/llm_framework/main_sys/src/tcp_com.cpp +++ b/projects/llm_framework/main_sys/src/tcp_com.cpp @@ -1,8 +1,8 @@ /* -* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD -* -* SPDX-License-Identifier: MIT -*/ + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ #include "all.h" #include "hv/TcpServer.h" #include @@ -31,31 +31,34 @@ std::atomic counter_port(8000); TcpServer srv; class tcp_com : public zmq_bus_com { - private: - public: +private: +public: SocketChannelPtr channel; std::string json_str; int flage; std::mutex tcp_server_mutex; - tcp_com() : zmq_bus_com() { + tcp_com() : zmq_bus_com() + { flage = 0; } - void send_data(const std::string& data) { + void send_data(const std::string& data) + { tcp_server_mutex.lock(); if (exit_flage) channel->write(data); tcp_server_mutex.unlock(); } }; -void onConnection(const SocketChannelPtr& channel) { +void onConnection(const SocketChannelPtr& channel) +{ std::string peeraddr = channel->peeraddr(); tcp_com* con_data; if (channel->isConnected()) { con_data = new tcp_com(); con_data->channel = channel; con_data->work(zmq_s_format, counter_port.fetch_add(1)); - if(counter_port.load() > 65535) counter_port.store(8000); + if (counter_port.load() > 65535) counter_port.store(8000); channel->setContext(con_data); } else { con_data = (tcp_com*)channel->context(); @@ -66,7 +69,8 @@ void onConnection(const SocketChannelPtr& channel) { } } -void onMessage(const SocketChannelPtr& channel, Buffer* buf) { +void onMessage(const SocketChannelPtr& channel, Buffer* buf) +{ int len = (int)buf->size(); char* data = (char*)buf->data(); tcp_com* con_data = (tcp_com*)channel->context(); @@ -96,7 +100,8 @@ void onMessage(const SocketChannelPtr& channel, Buffer* buf) { con_data->tcp_server_mutex.unlock(); } -void tcp_work() { +void tcp_work() +{ int listenport = 0; SAFE_READING(listenport, int, "config_tcp_server"); @@ -110,6 +115,7 @@ void tcp_work() { srv.setThreadNum(1); srv.start(); } -void tcp_stop_work() { +void tcp_stop_work() +{ srv.stop(); } diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index bd29328d..84113a73 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -15,12 +15,14 @@ using namespace StackFlows; void unit_action_match(int com_id, const std::string &json_str); -zmq_bus_com::zmq_bus_com() { +zmq_bus_com::zmq_bus_com() +{ exit_flage = 1; err_count = 0; } -void zmq_bus_com::work(const std::string &zmq_url_format, int port) { +void zmq_bus_com::work(const std::string &zmq_url_format, int port) +{ _port = port; exit_flage = 1; std::string ports = std::to_string(port); @@ -33,54 +35,65 @@ void zmq_bus_com::work(const std::string &zmq_url_format, int port) { reace_data_event_thread = std::make_unique(std::bind(&zmq_bus_com::reace_data_event, this)); } -void zmq_bus_com::stop() { +void zmq_bus_com::stop() +{ exit_flage = 0; user_chennal_.reset(); reace_data_event_thread->join(); } -void zmq_bus_com::on_data(const std::string &data) { +void zmq_bus_com::on_data(const std::string &data) +{ unit_action_match(_port, data); } -void zmq_bus_com::send_data(const std::string &data) { +void zmq_bus_com::send_data(const std::string &data) +{ // printf("zmq_bus_com::send_data : send:%s\n", data.c_str()); } -void zmq_bus_com::reace_data_event() { +void zmq_bus_com::reace_data_event() +{ // while (exit_flage) // { // std::this_thread::sleep_for(std::chrono::seconds(1)); // } } -void zmq_bus_com::send_data_event() { +void zmq_bus_com::send_data_event() +{ // while (exit_flage) // { // std::this_thread::sleep_for(std::chrono::seconds(1)); // } } -zmq_bus_com::~zmq_bus_com() { +zmq_bus_com::~zmq_bus_com() +{ } -unit_data::unit_data() { +unit_data::unit_data() +{ } -void unit_data::init_zmq(const std::string &url) { +void unit_data::init_zmq(const std::string &url) +{ inference_url = url; user_inference_chennal_ = std::make_unique(inference_url, ZMQ_PUB); } -void unit_data::send_msg(const std::string &json_str) { +void unit_data::send_msg(const std::string &json_str) +{ user_inference_chennal_->send_data(json_str); } -unit_data::~unit_data() { +unit_data::~unit_data() +{ user_inference_chennal_.reset(); } -int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_str) { +int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_str) +{ unit_data *unit_p = NULL; SAFE_READING(unit_p, unit_data *, work_id); if (unit_p) @@ -94,15 +107,18 @@ int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_s void *usr_context; -void zmq_com_send(int com_id, const std::string &out_str) { +void zmq_com_send(int com_id, const std::string &out_str) +{ char zmq_push_url[128]; sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); pzmq _zmq(zmq_push_url, ZMQ_PUSH); _zmq.send_data(out_str); } -void zmq_bus_work() { +void zmq_bus_work() +{ } -void zmq_bus_stop_work() { +void zmq_bus_stop_work() +{ } \ No newline at end of file diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 944aac5f..0db2bbe7 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -19,7 +19,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -42,11 +43,11 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: +private: float *dataW = NULL; int modelSize; - public: +public: std::unique_ptr synthesizer_; SynthesizerTrn_config mode_config_; std::vector inputs_; @@ -59,11 +60,13 @@ class llm_task { task_callback_t out_callback_; int awake_delay_ = 1000; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -86,7 +89,8 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -131,7 +135,8 @@ class llm_task { return 0; } - bool TTS(const std::string &msg) { + bool TTS(const std::string &msg) + { SLOGI("TTS msg:%s", msg.c_str()); int32_t dataLen; int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen); @@ -145,25 +150,29 @@ class llm_task { return false; } - bool delete_model() { + bool delete_model() + { synthesizer_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; #undef CONFIG_AUTO_SET class llm_tts : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -178,14 +187,16 @@ class llm_tts : public StackFlow { } } - public: - llm_tts() : StackFlow("tts") { +public: + llm_tts() : StackFlow("tts") + { task_count_ = 1; repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -207,7 +218,8 @@ class llm_tts : public StackFlow { } } - std::vector splitEachChar(const std::string &text) { + std::vector splitEachChar(const std::string &text) + { std::vector words; std::string input(text); int len = input.length(); @@ -231,7 +243,8 @@ class llm_tts : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { if (data.empty() || (data == "None")) return; static std::string faster_stream_buff; nlohmann::json error_body; @@ -286,7 +299,8 @@ class llm_tts : public StackFlow { } void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { @@ -302,7 +316,8 @@ class llm_tts : public StackFlow { } } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -361,7 +376,8 @@ class llm_tts : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -396,7 +412,8 @@ class llm_tts : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -424,7 +441,8 @@ class llm_tts : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -450,7 +468,8 @@ class llm_tts : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_tts::exit:%s", data.c_str()); nlohmann::json error_body; @@ -468,7 +487,8 @@ class llm_tts : public StackFlow { return 0; } - ~llm_tts() { + ~llm_tts() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -482,7 +502,8 @@ class llm_tts : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index ed807c83..fd9ada83 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -18,7 +18,8 @@ using namespace StackFlows; int main_exit_flage = 0; -static void __sigint(int iSigNo) { +static void __sigint(int iSigNo) +{ SLOGW("llm_sys will be exit!"); main_exit_flage = 1; } @@ -35,8 +36,8 @@ typedef std::function task_callback_ mode_config_.key = obj[#key]; class llm_task { - private: - public: +private: +public: LLMAttrType mode_config_; std::unique_ptr lLaMa_; std::string model_; @@ -52,11 +53,13 @@ class llm_task { std::atomic_bool tokenizer_server_flage_; unsigned int port_ = 8080; - void set_output(task_callback_t out_callback) { + void set_output(task_callback_t out_callback) + { out_callback_ = out_callback; } - bool parse_config(const nlohmann::json &config_body) { + bool parse_config(const nlohmann::json &config_body) + { try { model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); @@ -80,7 +83,8 @@ class llm_task { return false; } - int load_model(const nlohmann::json &config_body) { + int load_model(const nlohmann::json &config_body) + { if (parse_config(config_body)) { return -1; } @@ -128,21 +132,19 @@ class llm_task { pid_t pid = fork(); if (pid == 0) { execl("/usr/bin/python3", "python3", - ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), - "--host", "localhost", - "--port", std::to_string(port_).c_str(), - "--model_id", (base_model + "tokenizer").c_str(), - "--content", ("'" + prompt_ + "'").c_str(), - nullptr); + ("/opt/m5stack/scripts/" + model_ + "_tokenizer.py").c_str(), "--host", "localhost", + "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), + "--content", ("'" + prompt_ + "'").c_str(), nullptr); perror("execl failed"); exit(1); } tokenizer_server_flage_ = true; - SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); + SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(), + (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str()); std::this_thread::sleep_for(std::chrono::seconds(10)); } } else { - mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; + mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model; } SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str()); mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; @@ -166,7 +168,8 @@ class llm_task { return 0; } - std::string prompt_complete(const std::string &input) { + std::string prompt_complete(const std::string &input) + { std::ostringstream oss_prompt; switch (mode_config_.tokenizer_type) { case TKT_LLaMa: @@ -187,11 +190,12 @@ class llm_task { oss_prompt << input; break; } - SLOGI("prompt_complete:%s",oss_prompt.str().c_str()); + SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); return oss_prompt.str(); } - void inference(const std::string &msg) { + void inference(const std::string &msg) + { try { if (image_data_.empty()) { lLaMa_->Encode(prompt_data_, prompt_complete(msg)); @@ -215,31 +219,36 @@ class llm_task { } } - bool pause() { + bool pause() + { lLaMa_->Stop(); return true; } - bool delete_model() { + bool delete_model() + { lLaMa_->Deinit(); lLaMa_.reset(); return true; } - llm_task(const std::string &workid) { + llm_task(const std::string &workid) + { } - ~llm_task() { + ~llm_task() + { } }; #undef CONFIG_AUTO_SET class llm_llm : public StackFlow { - private: +private: int task_count_; std::unordered_map> llm_task_; - int _load_config() { + int _load_config() + { if (base_model_path_.empty()) { base_model_path_ = sys_sql_select("config_base_mode_path"); } @@ -254,14 +263,16 @@ class llm_llm : public StackFlow { } } - public: - llm_llm() : StackFlow("vlm") { +public: + llm_llm() : StackFlow("vlm") + { task_count_ = 2; repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) { + const std::string &data, bool finish) + { SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -284,7 +295,8 @@ class llm_llm : public StackFlow { void task_user_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, const std::string &object, - const std::string &data) { + const std::string &data) + { const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -309,7 +321,8 @@ class llm_llm : public StackFlow { } void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -320,11 +333,13 @@ class llm_llm : public StackFlow { } void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) { + const std::string &object, const std::string &data) + { llm_task_obj->lLaMa_->Stop(); } - int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { nlohmann::json error_body; if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; @@ -383,7 +398,8 @@ class llm_llm : public StackFlow { } } - void link(const std::string &work_id, const std::string &object, const std::string &data) override { + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; @@ -416,7 +432,8 @@ class llm_llm : public StackFlow { } } - void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; @@ -440,7 +457,8 @@ class llm_llm : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); @@ -466,7 +484,8 @@ class llm_llm : public StackFlow { } } - int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { SLOGI("llm_llm::exit:%s", data.c_str()); nlohmann::json error_body; @@ -484,7 +503,8 @@ class llm_llm : public StackFlow { return 0; } - ~llm_llm() { + ~llm_llm() + { while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -496,7 +516,8 @@ class llm_llm : public StackFlow { } }; -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); From a5b6935fdce1551ab5821f0e10114a6dbb28f02c Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Thu, 21 Nov 2024 09:19:44 +0800 Subject: [PATCH 03/44] [fix] llm_task_channel_ not erase --- projects/llm_framework/main/src/main.cpp | 10 ++++++++++ projects/llm_framework/main_asr/src/main.cpp | 12 +++++++++-- projects/llm_framework/main_kws/src/main.cpp | 20 +++++++++++++++++++ projects/llm_framework/main_llm/src/main.cpp | 10 ++++++++++ .../main_llm_tokenizer/src/main.cpp | 4 ++-- .../llm_framework/main_melotts/src/main.cpp | 12 +++++++++-- projects/llm_framework/main_tts/src/main.cpp | 12 +++++++++-- projects/llm_framework/main_vlm/src/main.cpp | 10 ++++++++++ 8 files changed, 82 insertions(+), 8 deletions(-) diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 62d3ee81..1ded545c 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -210,6 +210,7 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -217,6 +218,15 @@ class llm_llm : public StackFlow { ~llm_llm() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index fafb6d70..d32024d5 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -617,6 +617,7 @@ class llm_asr : public StackFlow { if (llm_task_[work_id_num]->audio_flage_) { unit_call("audio", "cap_stop", "None"); } + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -624,13 +625,20 @@ class llm_asr : public StackFlow { ~llm_asr() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); if (iteam->second->audio_flage_) { unit_call("audio", "cap_stop", "None"); } diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index d2135f25..b744029b 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -492,6 +492,26 @@ class llm_kws : public StackFlow { ~llm_kws() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + if (iteam->second->audio_flage_) { + unit_call("audio", "cap_stop", "None"); + } + iteam->second.reset(); + llm_task_.erase(iteam->first); + } } }; diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index e94cc53e..5430be9e 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -473,6 +473,7 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -480,6 +481,15 @@ class llm_llm : public StackFlow { ~llm_llm() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { diff --git a/projects/llm_framework/main_llm_tokenizer/src/main.cpp b/projects/llm_framework/main_llm_tokenizer/src/main.cpp index 78e648f3..6a16adba 100644 --- a/projects/llm_framework/main_llm_tokenizer/src/main.cpp +++ b/projects/llm_framework/main_llm_tokenizer/src/main.cpp @@ -9,13 +9,13 @@ int main() { if (access("./_tokenizer.py", F_OK) == 0) { - char *args[] = {"python3", "./_tokenizer.py", NULL}; + char *args[] = {(char*)"python3", (char*)"./_tokenizer.py", NULL}; if (execvp("python3", args) == -1) { perror("execvp"); return 1; } } else if (access("/opt/m5stack/share/_tokenizer.py", F_OK) == 0) { - char *args[] = {"python3", "/opt/m5stack/share/_tokenizer.py", NULL}; + char *args[] = {(char*)"python3", (char*)"/opt/m5stack/share/_tokenizer.py", NULL}; if (execvp("python3", args) == -1) { perror("execvp"); return 1; diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 9c736322..af8ab43c 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -635,6 +635,7 @@ class llm_tts : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -642,13 +643,20 @@ class llm_tts : public StackFlow { ~llm_tts() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 0db2bbe7..581d056c 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -482,6 +482,7 @@ class llm_tts : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -489,13 +490,20 @@ class llm_tts : public StackFlow { ~llm_tts() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } - auto llm_channel = get_channel(iteam->first); - llm_channel->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index fd9ada83..1fbeb5b5 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -498,6 +498,7 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -505,6 +506,15 @@ class llm_llm : public StackFlow { ~llm_llm() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + iteam->second->stop_subscriber(""); + iteam->second.reset(); + llm_task_channel_.erase(iteam->first); + } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { From b49fe252e15c1dca8b76ea8ea2ec37e016f30f3b Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Thu, 21 Nov 2024 13:16:57 +0800 Subject: [PATCH 04/44] [fix] Circular reference error --- projects/llm_framework/main/src/main.cpp | 29 ++++-- projects/llm_framework/main_asr/src/main.cpp | 55 ++++++++--- projects/llm_framework/main_kws/src/main.cpp | 28 +++++- projects/llm_framework/main_llm/src/main.cpp | 76 +++++++++++---- .../llm_framework/main_melotts/src/main.cpp | 95 +++++++++++++------ projects/llm_framework/main_tts/src/main.cpp | 64 +++++++++---- projects/llm_framework/main_vlm/src/main.cpp | 67 +++++++++---- 7 files changed, 296 insertions(+), 118 deletions(-) diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 1ded545c..dd3c6e13 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -89,9 +89,14 @@ class llm_llm : public StackFlow { task_count_ = 1; } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_channel->enstream_) { static int count = 0; nlohmann::json data_body; @@ -109,10 +114,15 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -157,10 +167,13 @@ class llm_llm : public StackFlow { if (ret == 0) { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); - llm_channel->subscriber_work_id("", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + llm_channel->subscriber_work_id( + "", + std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_[work_id_num] = llm_task_obj; send("None", "None", LLM_NO_ERROR, work_id); return 0; diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index d32024d5..ecf01951 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -292,9 +292,14 @@ class llm_asr : public StackFlow { EVENT_TASK_PAUSE, std::bind(&llm_asr::_task_pause, this, std::placeholders::_1, std::placeholders::_2)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string tmp_msg1; const std::string *next_data = &data; if (finish) { @@ -340,10 +345,15 @@ class llm_asr : public StackFlow { return 0; } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string tmp_msg1; const std::string *next_data = &data; int ret; @@ -398,8 +408,14 @@ class llm_asr : public StackFlow { event_queue_.enqueue(EVENT_TASK_PAUSE, work_id, ""); } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + void task_work(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->kws_awake(); if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { llm_channel->subscriber(audio_url_, @@ -408,9 +424,15 @@ class llm_asr : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_)); task_work(llm_task_obj, llm_channel); } @@ -476,10 +498,9 @@ class llm_asr : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); llm_task_obj->pause = std::bind(&llm_asr::task_pause, this, work_id, ""); - SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); - SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_asr::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_asr::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { @@ -489,13 +510,15 @@ class llm_asr : public StackFlow { llm_task_obj->audio_flage_ = true; } else if (input.find("asr") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_asr::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_asr::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_task_obj->ensleep_ = true; task_pause(work_id, ""); llm_channel->subscriber_work_id( - input, std::bind(&llm_asr::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_asr::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -534,8 +557,10 @@ class llm_asr : public StackFlow { llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { llm_task_obj->ensleep_ = true; - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_asr::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_asr::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index b744029b..cb0e7f00 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -314,16 +314,28 @@ class llm_kws : public StackFlow { } } - void task_pause(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + void task_pause(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->audio_flage_) { if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_); llm_task_obj->audio_flage_ = false; } } - void task_work(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel) + void task_work(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); @@ -363,10 +375,15 @@ class llm_kws : public StackFlow { send("None", "None", LLM_NO_ERROR, work_id); } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string tmp_msg1; const std::string *next_data = &data; int ret; @@ -426,7 +443,8 @@ class llm_kws : public StackFlow { llm_task_obj->audio_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_kws::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_kws::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 5430be9e..078d2f8e 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -152,7 +152,11 @@ class llm_task { } }; lLaMa_ = std::make_unique(); - if (!lLaMa_->Init(mode_config_)) return -2; + if (!lLaMa_->Init(mode_config_)) { + lLaMa_->Deinit(); + lLaMa_.reset(); + return -2; + } } catch (...) { SLOGE("config false"); @@ -216,6 +220,9 @@ class llm_task { ~llm_task() { + if (lLaMa_) { + lLaMa_->Deinit(); + } } }; @@ -248,9 +255,14 @@ class llm_llm : public StackFlow { repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -271,10 +283,15 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -294,9 +311,15 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void task_asr_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -306,9 +329,15 @@ class llm_llm : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->lLaMa_->Stop(); } @@ -341,21 +370,25 @@ class llm_llm : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("llm") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("asr") != std::string::npos) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -388,12 +421,15 @@ class llm_llm : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("asr") != std::string::npos) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index af8ab43c..6bfa5684 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -85,6 +85,7 @@ class llm_task { task_callback_t out_callback_; bool enaudio_; int awake_delay_ = 1000; + std::string tts_string_stream_buff; bool parse_config(const nlohmann::json &config_body) { @@ -371,9 +372,14 @@ class llm_tts : public StackFlow { repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -395,12 +401,25 @@ class llm_tts : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + bool is_breakpoint(const std::string &cutf8) + { + if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || + cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") + return true; + else + return false; + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (data.empty() || (data == "None")) return; - static std::string faster_stream_buff; nlohmann::json error_body; const std::string *next_data = &data; bool enbase64 = (object.find("base64") == std::string::npos) ? false : true; @@ -424,25 +443,24 @@ class llm_tts : public StackFlow { } std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar((*next_data)); for (auto cutf8 : tmp_data) { - if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || - cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") { - faster_stream_buff += cutf8; - ret = llm_task_obj->TTS(faster_stream_buff); - faster_stream_buff.clear(); + if (is_breakpoint(cutf8)) { + llm_task_obj->tts_string_stream_buff += cutf8; + ret = llm_task_obj->TTS(llm_task_obj->tts_string_stream_buff); + llm_task_obj->tts_string_stream_buff.clear(); if (ret) { error_body["code"] = -11; error_body["message"] = "Model run failed."; llm_channel->send("None", "None", error_body, llm_channel->work_id_); } } else { - faster_stream_buff += cutf8; + llm_task_obj->tts_string_stream_buff += cutf8; } } if (finish_flage) { - if (!faster_stream_buff.empty()) { - faster_stream_buff.push_back('.'); - ret = llm_task_obj->TTS(faster_stream_buff); - faster_stream_buff.clear(); + if (!llm_task_obj->tts_string_stream_buff.empty()) { + llm_task_obj->tts_string_stream_buff.push_back('.'); + ret = llm_task_obj->TTS(llm_task_obj->tts_string_stream_buff); + llm_task_obj->tts_string_stream_buff.clear(); if (ret) { error_body["code"] = -11; error_body["message"] = "Model run failed."; @@ -452,11 +470,18 @@ class llm_tts : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); + llm_task_obj->tts_string_stream_buff.clear(); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "queue_play_stop", data); } @@ -464,9 +489,10 @@ class llm_tts : public StackFlow { if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "play_stop", data); } - llm_channel->subscriber_work_id(llm_task_obj->superior_id_, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + llm_task_obj->superior_id_, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -498,22 +524,26 @@ class llm_tts : public StackFlow { llm_channel->set_stream(llm_task_obj->enstream_); SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("tts") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if ((input.find("llm") != std::string::npos) || (input.find("vlm") != std::string::npos)) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); llm_task_obj->superior_id_ = input; llm_task_obj->superior_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -546,14 +576,17 @@ class llm_tts : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if ((data.find("llm") != std::string::npos) || (data.find("vlm") != std::string::npos)) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->superior_id_ = data; llm_task_obj->superior_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 581d056c..50e38c0a 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -194,9 +194,14 @@ class llm_tts : public StackFlow { repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } std::string base64_data; int len = encode_base64(data, base64_data); if (llm_channel->enstream_) { @@ -241,10 +246,15 @@ class llm_tts : public StackFlow { return words; } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (data.empty() || (data == "None")) return; static std::string faster_stream_buff; nlohmann::json error_body; @@ -298,9 +308,15 @@ class llm_tts : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (llm_task_obj->superior_flage_) { llm_channel->stop_subscriber_work_id(llm_task_obj->superior_id_); if (llm_task_obj->response_format_.find("sys") != std::string::npos) { @@ -310,9 +326,10 @@ class llm_tts : public StackFlow { if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "play_stop", data); } - llm_channel->subscriber_work_id(llm_task_obj->superior_id_, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + llm_task_obj->superior_id_, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -344,22 +361,26 @@ class llm_tts : public StackFlow { llm_channel->set_stream(llm_task_obj->enstream_); SLOGI("llm_task_obj->enoutput_:%d", llm_task_obj->enoutput_); SLOGI("llm_task_obj->enstream_:%d", llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_tts::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("tts") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if ((input.find("llm") != std::string::npos) || (input.find("vlm") != std::string::npos)) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); llm_task_obj->superior_id_ = input; llm_task_obj->superior_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -392,14 +413,17 @@ class llm_tts : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if ((data.find("llm") != std::string::npos) || (data.find("vlm") != std::string::npos)) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_tts::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->superior_id_ = data; llm_task_obj->superior_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_tts::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_tts::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 1fbeb5b5..62b78010 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -270,9 +270,14 @@ class llm_llm : public StackFlow { repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } - void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &data, bool finish) + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } SLOGI("send:%s", data.c_str()); if (llm_channel->enstream_) { static int count = 0; @@ -293,10 +298,15 @@ class llm_llm : public StackFlow { } } - void task_user_data(const std::shared_ptr llm_task_obj, - const std::shared_ptr llm_channel, const std::string &object, + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } const std::string *next_data = &data; int ret; std::string tmp_msg1; @@ -320,9 +330,15 @@ class llm_llm : public StackFlow { llm_task_obj->inference((*next_data)); } - void task_asr_data(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void task_asr_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { llm_task_obj->inference(sample_json_str_get(data, "delta")); @@ -332,9 +348,15 @@ class llm_llm : public StackFlow { } } - void kws_awake(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, - const std::string &object, const std::string &data) + void kws_awake(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } llm_task_obj->lLaMa_->Stop(); } @@ -367,21 +389,25 @@ class llm_llm : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("vlm") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, std::placeholders::_1, + "", std::bind(&llm_llm::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } else if (input.find("asr") != std::string::npos) { - llm_channel->subscriber_work_id(input, - std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id( + input, std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( - input, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, std::placeholders::_1, + input, std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } } @@ -414,12 +440,15 @@ class llm_llm : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("asr") != std::string::npos) { ret = llm_channel->subscriber_work_id( - data, std::bind(&llm_llm::task_asr_data, this, llm_task_obj, llm_channel, std::placeholders::_1, - std::placeholders::_2)); + data, + std::bind(&llm_llm::task_asr_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { - ret = llm_channel->subscriber_work_id(data, std::bind(&llm_llm::kws_awake, this, llm_task_obj, llm_channel, - std::placeholders::_1, std::placeholders::_2)); + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_llm::kws_awake, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } if (ret) { From b19299713b34bc149d4f6393aa1de6b3e0fce1b6 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Thu, 21 Nov 2024 15:47:05 +0800 Subject: [PATCH 05/44] [fix] llm_task_channel_ resource release --- .../StackFlow/stackflow/StackFlow.cpp | 8 +- .../StackFlow/stackflow/StackFlow.h | 1 + ext_components/StackFlow/stackflow/pzmq.hpp | 88 ++++++++++++++----- projects/llm_framework/main/src/main.cpp | 11 +-- projects/llm_framework/main_asr/src/main.cpp | 11 +-- projects/llm_framework/main_kws/src/main.cpp | 11 +-- projects/llm_framework/main_llm/src/main.cpp | 11 +-- .../llm_framework/main_melotts/src/main.cpp | 11 +-- projects/llm_framework/main_tts/src/main.cpp | 11 +-- projects/llm_framework/main_vlm/src/main.cpp | 11 +-- 10 files changed, 78 insertions(+), 96 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index b141fe8d..d2593b51 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -26,11 +26,6 @@ llm_channel_obj::~llm_channel_obj() void llm_channel_obj::subscriber_event_call(const std::function &call, const std::string &raw) { - // SLOGI("object%s", sample_json_str_get(raw, "object").c_str()); - // SLOGI("request_id%s", sample_json_str_get(raw, "request_id").c_str()); - // SLOGI("work_id%s", sample_json_str_get(raw, "work_id").c_str()); - // SLOGI("action%s", sample_json_str_get(raw, "action").c_str()); - // SLOGI("data%s", sample_json_str_get(raw, "data").c_str()); if (sample_json_str_get(raw, "action") == "inference") { std::string zmq_com = sample_json_str_get(raw, "zmq_com"); if (!zmq_com.empty()) set_push_url(zmq_com); @@ -141,7 +136,7 @@ int llm_channel_obj::output_to_uart(const std::string &data) } StackFlow::StackFlow::StackFlow(const std::string &unit_name) - : unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) + : work_id_num_cout_(1000), unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) { event_queue_.appendListener(EVENT_NONE, std::bind(&StackFlow::_none_event, this, std::placeholders::_1, std::placeholders::_2)); @@ -185,6 +180,7 @@ StackFlow::~StackFlow() break; } sys_release_unit(iteam->first, ""); + iteam->second.reset(); llm_task_channel_.erase(iteam->first); } exit_flage_.store(true); diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index df157d5e..6cff463e 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -184,6 +184,7 @@ class llm_channel_obj { class StackFlow { private: + std::atomic_int work_id_num_cout_; protected: std::string unit_name_; typedef enum { diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index c6511a84..e4a99539 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -12,16 +12,19 @@ #include #include #include +#include +#include #define ZMQ_RPC_FUN (ZMQ_REP | 0x80) #define ZMQ_RPC_CALL (ZMQ_REQ | 0x80) namespace StackFlows { class pzmq { private: - const std::string unix_socket_head_ = "ipc://"; - const std::string rpc_url_head_ = "ipc:///tmp/rpc."; + const int rpc_url_head_length = 6; + std::string rpc_url_head_ = "ipc:///tmp/rpc."; void *zmq_ctx_; void *zmq_socket_; std::unordered_map> zmq_fun_; + std::mutex zmq_fun_mtx_; std::atomic flage_; std::unique_ptr zmq_thread_; int mode_; @@ -29,6 +32,14 @@ class pzmq { std::string zmq_url_; int timeout_; + bool is_bind() + { + if ((mode_ == ZMQ_PUB) || (mode_ == ZMQ_PULL) || (mode_ == ZMQ_RPC_FUN)) + return true; + else + return false; + } + public: pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) { @@ -36,6 +47,9 @@ class pzmq { pzmq(const std::string &server) : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) { + if (server.find("://") != std::string::npos) { + rpc_url_head_.clear(); + } } pzmq(const std::string &url, int mode, const std::function &raw_call = nullptr) : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) @@ -50,22 +64,60 @@ class pzmq { { return timeout_; } + std::string get_zmq_url() + { + if ((!flage_.load()) && is_bind() && (zmq_url_.find("tcp://") != std::string::npos) && + (zmq_url_.find(":*") != std::string::npos)) { + std::vector zmq_url(zmq_url_.length() + 8, 0); + size_t addr_len = zmq_url.size(); + if (zmq_getsockopt(zmq_socket_, ZMQ_LAST_ENDPOINT, zmq_url.data(), &addr_len) != 0) { + return zmq_url_; + } + zmq_url_ = std::string(zmq_url.data()); + } + return zmq_url_; + } + std::string _rpc_list_action(const std::string &_None) + { + std::string action_list; + action_list.reserve(128); + action_list = "{\"actions\":["; + for (auto i = zmq_fun_.begin();;) { + action_list += "\""; + action_list += i->first; + action_list += "\""; + if (++i == zmq_fun_.end()) { + action_list += "]}"; + break; + } else { + action_list += ","; + } + } + return action_list; + } int register_rpc_action(const std::string &action, const std::function &raw_call) { + int ret = 0; + std::unique_lock lock(zmq_fun_mtx_); if (zmq_fun_.find(action) != zmq_fun_.end()) { zmq_fun_[action] = raw_call; - return 0; + return ret; } - std::string url = rpc_url_head_ + rpc_server_; - if (!flage_) { - flage_ = true; - zmq_ctx_shutdown(zmq_ctx_); - zmq_thread_->join(); - close_zmq(); + if (zmq_fun_.empty()) { + std::string url = rpc_url_head_ + rpc_server_; + mode_ = ZMQ_RPC_FUN; + zmq_fun_["list_action"] = std::bind(&pzmq::_rpc_list_action, this, std::placeholders::_1); + ret = creat(url); } zmq_fun_[action] = raw_call; - mode_ = ZMQ_RPC_FUN; - return creat(url); + return ret; + } + void unregister_rpc_action(const std::string &action) + { + std::unique_lock lock(zmq_fun_mtx_); + if (zmq_fun_.find(action) != zmq_fun_.end()) { + zmq_fun_.erase(action); + } } int call_rpc_action(const std::string &action, const std::string &data, const std::function &raw_call) @@ -215,9 +267,8 @@ class pzmq { } inline int creat_req(const std::string &url) { - int pos = url.find(unix_socket_head_); - if (pos != std::string::npos) { - std::string socket_file = url.substr(pos + unix_socket_head_.length()); + if (!rpc_url_head_.empty()) { + std::string socket_file = url.substr(rpc_url_head_length); if (access(socket_file.c_str(), F_OK) != 0) { return -1; } @@ -249,10 +300,8 @@ class pzmq { continue; } } - // SLOGI("zmq_msg_recv"); ret = zmq_msg_recv(&msg, zmq_socket_, 0); if (ret <= 0) { - // SLOGE("zmq_connect false:%s", zmq_strerror(zmq_errno())); zmq_msg_close(&msg); continue; } @@ -264,6 +313,7 @@ class pzmq { std::string _raw_data((const char *)zmq_msg_data(&msg1), zmq_msg_size(&msg1)); std::string retval; try { + std::unique_lock lock(zmq_fun_mtx_); retval = zmq_fun_.at(raw_data)(_raw_data); } catch (...) { retval = "NotAction"; @@ -271,7 +321,6 @@ class pzmq { zmq_send(zmq_socket_, retval.c_str(), retval.length(), 0); zmq_msg_close(&msg1); } else { - // SLOGI("zmq_msg:%s", zmq_msg_data(&msg)); raw_call(raw_data); } zmq_msg_close(&msg); @@ -281,10 +330,9 @@ class pzmq { { zmq_close(zmq_socket_); zmq_ctx_destroy(zmq_ctx_); - int pos = zmq_url_.find(unix_socket_head_); if ((mode_ == ZMQ_PUB) || (mode_ == ZMQ_PULL) || (mode_ == ZMQ_RPC_FUN)) { - if (pos != std::string::npos) { - std::string socket_file = zmq_url_.substr(pos + unix_socket_head_.length()); + if (!rpc_url_head_.empty()) { + std::string socket_file = zmq_url_.substr(rpc_url_head_length); if (access(socket_file.c_str(), F_OK) == 0) { remove(socket_file.c_str()); } diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index dd3c6e13..67c6fa3e 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -223,7 +223,6 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -231,20 +230,12 @@ class llm_llm : public StackFlow { ~llm_llm() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index ecf01951..27152b7a 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -642,7 +642,6 @@ class llm_asr : public StackFlow { if (llm_task_[work_id_num]->audio_flage_) { unit_call("audio", "cap_stop", "None"); } - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -650,15 +649,6 @@ class llm_asr : public StackFlow { ~llm_asr() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -667,6 +657,7 @@ class llm_asr : public StackFlow { if (iteam->second->audio_flage_) { unit_call("audio", "cap_stop", "None"); } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index cb0e7f00..9eaedbfa 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -502,7 +502,6 @@ class llm_kws : public StackFlow { if (llm_task_[work_id_num]->audio_flage_) { unit_call("audio", "cap_stop", "None"); } - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -510,15 +509,6 @@ class llm_kws : public StackFlow { ~llm_kws() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { @@ -527,6 +517,7 @@ class llm_kws : public StackFlow { if (iteam->second->audio_flage_) { unit_call("audio", "cap_stop", "None"); } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 078d2f8e..f040d67a 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -509,7 +509,6 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -517,20 +516,12 @@ class llm_llm : public StackFlow { ~llm_llm() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 6bfa5684..ba18fffd 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -668,7 +668,6 @@ class llm_tts : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -676,20 +675,12 @@ class llm_tts : public StackFlow { ~llm_tts() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 50e38c0a..b87647d8 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -506,7 +506,6 @@ class llm_tts : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -514,20 +513,12 @@ class llm_tts : public StackFlow { ~llm_tts() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 62b78010..f04bbddf 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -527,7 +527,6 @@ class llm_llm : public StackFlow { } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); - llm_task_channel_.erase(work_id_num); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; @@ -535,20 +534,12 @@ class llm_llm : public StackFlow { ~llm_llm() { - while (1) { - auto iteam = llm_task_channel_.begin(); - if (iteam == llm_task_channel_.end()) { - break; - } - iteam->second->stop_subscriber(""); - iteam->second.reset(); - llm_task_channel_.erase(iteam->first); - } while (1) { auto iteam = llm_task_.begin(); if (iteam == llm_task_.end()) { break; } + get_channel(iteam->first)->stop_subscriber(""); iteam->second.reset(); llm_task_.erase(iteam->first); } From 82b580eea7322e708f883a13a8713cc6649ea56c Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 08:59:48 +0800 Subject: [PATCH 06/44] [update] audio setup && fix StackFlow sys_sql_unset --- ext_components/StackFlow/stackflow/StackFlow.cpp | 2 +- projects/llm_framework/main_audio/src/main.cpp | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index d2593b51..862d26ca 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -504,7 +504,7 @@ void StackFlow::sys_sql_unset(const std::string &key) { std::string val; pzmq _call("sys"); - _call.call_rpc_action("sql_select", key, [](const std::string &data) {}); + _call.call_rpc_action("sql_unset", key, [](const std::string &data) {}); } std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 5cc92682..cd8913d5 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -111,8 +111,8 @@ class llm_audio : public StackFlow { { event_queue_.appendListener( EVENT_QUEUE_PLAY, std::bind(&llm_audio::hw_queue_play, this, std::placeholders::_1, std::placeholders::_2)); - setup("", "{\"object\":\"audio.play\",\"data\":{\"None\":\"None\"}}"); - setup("", "{\"object\":\"audio.cap\",\"data\":{\"None\":\"None\"}}"); + setup("", "audio.play", "{\"None\":\"None\"}"); + setup("", "audio.cap", "{\"None\":\"None\"}"); self = this; cap_status_ = 0; rpc_ctx_->register_rpc_action("play", std::bind(&llm_audio::play, this, std::placeholders::_1)); @@ -125,11 +125,8 @@ class llm_audio : public StackFlow { rpc_ctx_->register_rpc_action("cap_stop", std::bind(&llm_audio::cap_stop, this, std::placeholders::_1)); rpc_ctx_->register_rpc_action("cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1)); } - - int setup(const std::string &zmq_url, const std::string &raw) override + int setup(const std::string &work_id, const std::string &object, const std::string &data) override { - std::string object = sample_json_str_get(raw, "object"); - std::string data = sample_json_str_get(raw, "data"); nlohmann::json config_body; nlohmann::json file_body; nlohmann::json error_body; From 0ceff540daf48b1b47a5e679623ba3756e5837aa Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 13:50:43 +0800 Subject: [PATCH 07/44] [update] change StackFlow component msg --- .../StackFlow/stackflow/StackFlow.cpp | 71 +++++++------------ .../StackFlow/stackflow/StackFlow.h | 2 +- .../StackFlow/stackflow/StackFlowUtil.cpp | 9 +++ .../StackFlow/stackflow/StackFlowUtil.h | 14 ++++ .../llm_framework/main_audio/src/main.cpp | 21 ++++-- projects/llm_framework/main_kws/src/main.cpp | 2 +- .../main_sys/src/remote_action.cpp | 12 ++-- .../main_sys/src/remote_server.cpp | 24 +++---- projects/llm_framework/setup.ini | 2 +- 9 files changed, 81 insertions(+), 76 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index 862d26ca..7fbc6da3 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -10,6 +10,12 @@ using namespace StackFlows; std::string llm_channel_obj::uart_push_url; +// Deprecated +#define RPC_PARSE_TO_PARAM_OLD(obj) \ + sample_json_str_get(obj, "zmq_com"), sample_unescapeString(sample_json_str_get(obj, "raw_data")) + +#define RPC_PARSE_TO_PARAM(obj) RPC_PARSE_TO_FIRST(obj), RPC_PARSE_TO_SECOND(obj) + llm_channel_obj::llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name) : unit_name_(unit_name), inference_url_(inference_url) @@ -47,9 +53,7 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, // std::string part1 = matches[1].str(); id_num = std::stoi(matches[2].str()); std::string input_url_name = work_id + ".out_port"; - std::string input_url; - pzmq _call("sys"); - _call.call_rpc_action("sql_select", input_url_name, [&](const std::string &data) { input_url = data; }); + std::string input_url = unit_call("sys", "sql_select", input_url_name); if (input_url.empty()) { return -1; } @@ -217,9 +221,7 @@ void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) std::string StackFlow::_rpc_setup(const std::string &data) { - SLOGI("_rpc_setup:%s", data.c_str()); - event_queue_.enqueue(EVENT_SETUP, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_SETUP, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -253,8 +255,7 @@ int StackFlow::setup(const std::string &work_id, const std::string &object, cons std::string StackFlow::_rpc_link(const std::string &data) { - event_queue_.enqueue(EVENT_LINK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_LINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -285,8 +286,7 @@ void StackFlow::link(const std::string &work_id, const std::string &object, cons std::string StackFlow::_rpc_unlink(const std::string &data) { - event_queue_.enqueue(EVENT_UNLINK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_UNLINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -317,8 +317,7 @@ void StackFlow::unlink(const std::string &work_id, const std::string &object, co std::string StackFlow::_rpc_work(const std::string &data) { - event_queue_.enqueue(EVENT_WORK, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_WORK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -349,8 +348,7 @@ void StackFlow::work(const std::string &work_id, const std::string &object, cons std::string StackFlow::_rpc_exit(const std::string &data) { - event_queue_.enqueue(EVENT_EXIT, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_EXIT, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -384,8 +382,7 @@ int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string StackFlow::_rpc_pause(const std::string &data) { - event_queue_.enqueue(EVENT_PAUSE, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_PAUSE, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -416,8 +413,7 @@ void StackFlow::pause(const std::string &work_id, const std::string &object, con std::string StackFlow::_rpc_taskinfo(const std::string &data) { - event_queue_.enqueue(EVENT_TASKINFO, sample_json_str_get(data, "zmq_com"), - sample_unescapeString(sample_json_str_get(data, "raw_data"))); + event_queue_.enqueue(EVENT_TASKINFO, RPC_PARSE_TO_PARAM(data)); return std::string("None"); } @@ -449,15 +445,13 @@ void StackFlow::taskinfo(const std::string &work_id, const std::string &object, int StackFlow::sys_register_unit(const std::string &unit_name) { int work_id_number; - std::string out_port; - std::string inference_port; - - pzmq _call("sys"); - _call.call_rpc_action("register_unit", unit_name, [&](const std::string &unit_info) { - work_id_number = std::stoi(sample_json_str_get(unit_info, "work_id_number")); - out_port = sample_json_str_get(unit_info, "out_port"); - inference_port = sample_json_str_get(unit_info, "inference_port"); - }); + std::string component_msg = unit_call("sys", "register_unit", unit_name); + std::string str_port = RPC_PARSE_TO_FIRST(component_msg); + work_id_number = std::stoi(str_port); + std::string tmp_buf = RPC_PARSE_TO_SECOND(component_msg); + std::string out_port = RPC_PARSE_TO_FIRST(tmp_buf); + std::string inference_port = RPC_PARSE_TO_SECOND(tmp_buf); + SLOGI("work_id_number:%d, out_port:%s, inference_port:%s ", work_id_number, out_port.c_str(), inference_port.c_str()); llm_task_channel_[work_id_number] = std::make_shared(out_port, inference_port, unit_name_); @@ -475,8 +469,7 @@ bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) _work_id = work_id; _work_id_num = sample_get_work_id_num(work_id); } - pzmq _call("sys"); - _call.call_rpc_action("release_unit", _work_id, [&](const std::string &unit_info) {}); + unit_call("sys", "release_unit", _work_id); llm_task_channel_[_work_id_num].reset(); llm_task_channel_.erase(_work_id_num); SLOGI("release work_id %s success", _work_id.c_str()); @@ -485,10 +478,7 @@ bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) std::string StackFlow::sys_sql_select(const std::string &key) { - std::string val; - pzmq _call("sys"); - _call.call_rpc_action("sql_select", key, [&val](const std::string &data) { val = data; }); - return val; + return sample_unescapeString(unit_call("sys", "sql_select", key)); } void StackFlow::sys_sql_set(const std::string &key, const std::string &val) @@ -496,23 +486,12 @@ void StackFlow::sys_sql_set(const std::string &key, const std::string &val) nlohmann::json out_body; out_body["key"] = key; out_body["val"] = val; - pzmq _call("sys"); - _call.call_rpc_action("sql_set", out_body.dump(), [](const std::string &data) {}); + unit_call("sys", "sql_set", out_body.dump()); } void StackFlow::sys_sql_unset(const std::string &key) { - std::string val; - pzmq _call("sys"); - _call.call_rpc_action("sql_unset", key, [](const std::string &data) {}); -} - -std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) -{ - std::string value; - pzmq _call(unit_name); - _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); - return value; + unit_call("sys", "sql_unset", key); } void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index 6cff463e..6f7d397e 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -185,6 +185,7 @@ class llm_channel_obj { class StackFlow { private: std::atomic_int work_id_num_cout_; + protected: std::string unit_name_; typedef enum { @@ -352,7 +353,6 @@ class StackFlow { { } - std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); std::string sys_sql_select(const std::string &key); void sys_sql_set(const std::string &key, const std::string &val); void sys_sql_unset(const std::string &key); diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 956809c2..f50b9229 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -5,6 +5,7 @@ */ #include "StackFlowUtil.h" #include +#include "pzmq.hpp" std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { @@ -399,4 +400,12 @@ int StackFlows::encode_base64(const std::string &in, std::string &out) { out.resize(BASE64_ENCODE_OUT_SIZE(in.length())); return base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); +} + +std::string StackFlows::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) +{ + std::string value; + pzmq _call(unit_name); + _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); + return value; } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h index 1b5df5f8..ffbe00f6 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.h +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -5,8 +5,21 @@ */ #pragma once #include +#include #include #define WORK_ID_NONE -100 + +#define RPC_PUSH_PARAM(_obj, _data1, _data2) \ + do { \ + _obj.resize(1 + _data1.size() + _data2.size()); \ + _obj[0] = (unsigned char)_data1.size(); \ + memcpy((void *)(_obj.data() + 1), (void *)_data1.data(), _data1.size()); \ + memcpy((void *)(_obj.data() + _data1.size() + 1), (void *)_data2.data(), _data2.size()); \ + } while (0) + +#define RPC_PARSE_TO_FIRST(_obj) _obj.substr(1, static_cast(_obj[0])) +#define RPC_PARSE_TO_SECOND(_obj) _obj.substr(static_cast(_obj[0]) + 1) + namespace StackFlows { std::string sample_json_str_get(const std::string &json_str, const std::string &json_key); int sample_get_work_id_num(const std::string &work_id); @@ -17,4 +30,5 @@ std::string sample_unescapeString(const std::string &input); bool decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff); int decode_base64(const std::string &in, std::string &out); int encode_base64(const std::string &in, std::string &out); +std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); }; // namespace StackFlows diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index cd8913d5..141c5313 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -116,6 +116,7 @@ class llm_audio : public StackFlow { self = this; cap_status_ = 0; rpc_ctx_->register_rpc_action("play", std::bind(&llm_audio::play, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("play_raw", std::bind(&llm_audio::play_raw, this, std::placeholders::_1)); rpc_ctx_->register_rpc_action("queue_play", std::bind(&llm_audio::enqueue_play, this, std::placeholders::_1)); rpc_ctx_->register_rpc_action("play_stop", std::bind(&llm_audio::play_stop, this, std::placeholders::_1)); rpc_ctx_->register_rpc_action("queue_play_stop", @@ -369,13 +370,19 @@ class llm_audio : public StackFlow { std::string play(const std::string &rawdata) { - if (rawdata.size() < 3) return LLM_NONE; - if ((rawdata[0] == '{') && (rawdata[rawdata.size() - 1] == '}')) { - std::string src_data = sample_unescapeString(sample_json_str_get(rawdata, "raw_data")); - return parse_data(sample_json_str_get(src_data, "object"), sample_json_str_get(src_data, "data")); - } else { - _play(rawdata); - } + std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); + std::string audio_json = RPC_PARSE_TO_SECOND(rawdata); + std::string ret_val = parse_data(sample_json_str_get(audio_json, "object"), sample_json_str_get(audio_json, "data")); + request_id_ = sample_json_str_get(audio_json, "request_id"); + send(LLM_NONE, LLM_NONE, LLM_NO_ERROR, sample_json_str_get(audio_json, "work_id"), zmq_url); + return ret_val; + } + + std::string play_raw(const std::string &rawdata) + { + if(rawdata.empty()) + return std::string("rawdata empty"); + _play(rawdata); return LLM_NONE; } diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 9eaedbfa..e2ac7165 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -310,7 +310,7 @@ class llm_kws : public StackFlow { } } if (post != 0) { - unit_call("audio", "play", std::string((char *)(wav_data.data() + post), size - post)); + unit_call("audio", "play_raw", std::string((char *)(wav_data.data() + post), size - post)); } } diff --git a/projects/llm_framework/main_sys/src/remote_action.cpp b/projects/llm_framework/main_sys/src/remote_action.cpp index a6e99c84..03ff87e6 100644 --- a/projects/llm_framework/main_sys/src/remote_action.cpp +++ b/projects/llm_framework/main_sys/src/remote_action.cpp @@ -19,13 +19,11 @@ int remote_call(int com_id, const std::string &json_str) std::string work_id = sample_json_str_get(json_str, "work_id"); std::string work_unit = work_id.substr(0, work_id.find(".")); std::string action = sample_json_str_get(json_str, "action"); - char com_url[128]; - sprintf(com_url, zmq_c_format.c_str(), com_id); - std::string send_data = "{\"zmq_com\":\""; - send_data += std::string(com_url); - send_data += "\",\"raw_data\":\""; - send_data += sample_escapeString(json_str); - send_data += "\"}"; + char com_url[256]; + int length = snprintf(com_url, 255, zmq_c_format.c_str(), com_id); + std::string send_data; + std::string com_urls(com_url); + RPC_PUSH_PARAM(send_data, com_urls, json_str); pzmq clent(work_unit); return clent.call_rpc_action(action, send_data, [](const std::string &val) {}); } diff --git a/projects/llm_framework/main_sys/src/remote_server.cpp b/projects/llm_framework/main_sys/src/remote_server.cpp index 18ff306a..83335dfe 100644 --- a/projects/llm_framework/main_sys/src/remote_server.cpp +++ b/projects/llm_framework/main_sys/src/remote_server.cpp @@ -171,22 +171,15 @@ int c_sys_release_unit(char const *unit) return sys_release_unit(unit); } -std::string rpc_sql_select(const std::string &raw) -{ - return sys_sql_select(raw); -} - std::string rpc_allocate_unit(const std::string &raw) { unit_data *unit_info = sys_allocate_unit(raw); - std::string retval = "{\"work_id_number\":"; - retval += std::to_string(unit_info->port_); - retval += ",\"out_port\":\""; - retval += unit_info->output_url; - retval += "\",\"inference_port\":\""; - retval += unit_info->inference_url; - retval += "\"}"; - return retval; + std::string send_data; + std::string send_data1; + std::string str_port = std::to_string(unit_info->port_); + RPC_PUSH_PARAM(send_data1, unit_info->output_url, unit_info->inference_url); + RPC_PUSH_PARAM(send_data, str_port, send_data1); + return send_data; } std::string rpc_release_unit(const std::string &raw) @@ -195,6 +188,11 @@ std::string rpc_release_unit(const std::string &raw) return "Success"; } +std::string rpc_sql_select(const std::string &raw) +{ + return sys_sql_select(raw); +} + std::string rpc_sql_set(const std::string &raw) { std::string key = sample_json_str_get(raw, "key"); diff --git a/projects/llm_framework/setup.ini b/projects/llm_framework/setup.ini index d01d0fb1..e1363e0c 100644 --- a/projects/llm_framework/setup.ini +++ b/projects/llm_framework/setup.ini @@ -1,7 +1,7 @@ [ssh] local_file_path = dist remote_file_path = /root/dist -remote_host = 192.168.28.62 +remote_host = m5stack-LLM.local remote_port = 22 username = root password = 123456 \ No newline at end of file From 814c91f9282ec46b574023c667d5c80b0b5d028c Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 14:32:25 +0800 Subject: [PATCH 08/44] [update] sys.reset --- projects/llm_framework/main_sys/src/event_loop.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 7c6fa0ac..7b7bae8c 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -459,8 +459,8 @@ int sys_reset(int com_id, const nlohmann::json &json_obj) usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"llm server restarting ...\"}", com_id); const char *cmd = - "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm/reset.lock ; sync ; sleep 1 ; systemctl restart " - "llm-* \" > /dev/null 2>&1 & "; + "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm/reset.lock ; sync ; sleep 1 ; rm /tmp/llm/* -f ; " + "systemctl restart llm-* \" > /dev/null 2>&1 & "; system(cmd); return out; } From a0b6082aaf7bddb13d67ec3cadcef177844b80e0 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Fri, 22 Nov 2024 14:53:35 +0800 Subject: [PATCH 09/44] [update] cv support yolo. --- projects/llm_framework/main_cv/Kconfig | 0 projects/llm_framework/main_cv/SConstruct | 63 + projects/llm_framework/main_cv/src/main.cpp | 489 +++ .../main_cv/src/runner/EngineWrapper.cpp | 311 ++ .../main_cv/src/runner/EngineWrapper.hpp | 49 + .../main_cv/src/runner/base/common.hpp | 151 + .../main_cv/src/runner/base/detection.h | 22 + .../main_cv/src/runner/base/detection.hpp | 3122 +++++++++++++++++ .../main_cv/src/runner/base/pose.hpp | 432 +++ .../main_cv/src/runner/base/score.hpp | 33 + .../main_cv/src/runner/base/topk.hpp | 52 + .../main_cv/src/runner/base/transform.hpp | 47 + .../main_cv/src/runner/base/yolo.hpp | 578 +++ .../main_cv/src/runner/utils/checker.h | 32 + .../main_cv/src/runner/utils/io.hpp | 601 ++++ .../main_cv/src/runner/utils/logger.h | 82 + .../main_cv/src/runner/utils/timer.hpp | 61 + projects/llm_framework/main_cv/yolo11s.json | 23 + projects/llm_framework/main_vlm/src/main.cpp | 7 +- 19 files changed, 6149 insertions(+), 6 deletions(-) create mode 100644 projects/llm_framework/main_cv/Kconfig create mode 100644 projects/llm_framework/main_cv/SConstruct create mode 100644 projects/llm_framework/main_cv/src/main.cpp create mode 100644 projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp create mode 100644 projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/common.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/detection.h create mode 100644 projects/llm_framework/main_cv/src/runner/base/detection.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/pose.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/score.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/topk.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/transform.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/base/yolo.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/utils/checker.h create mode 100644 projects/llm_framework/main_cv/src/runner/utils/io.hpp create mode 100644 projects/llm_framework/main_cv/src/runner/utils/logger.h create mode 100644 projects/llm_framework/main_cv/src/runner/utils/timer.hpp create mode 100644 projects/llm_framework/main_cv/yolo11s.json diff --git a/projects/llm_framework/main_cv/Kconfig b/projects/llm_framework/main_cv/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_cv/SConstruct b/projects/llm_framework/main_cv/SConstruct new file mode 100644 index 00000000..b98b2017 --- /dev/null +++ b/projects/llm_framework/main_cv/SConstruct @@ -0,0 +1,63 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = append_srcs_dir(ADir('src')) +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] + +INCLUDE += [ADir('src/runner')] +INCLUDE += [ADir('../include/opencv4')] +# LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] +# LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', +# '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', +# '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] + +# STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), +# AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] +static_file = Glob('../static_lib/module-llm/libabsl_*') +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] +STATIC_LIB += static_file * 4 + +env['COMPONENTS'].append({'target':'llm_cv', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_cv/src/main.cpp b/projects/llm_framework/main_cv/src/main.cpp new file mode 100644 index 00000000..7df50864 --- /dev/null +++ b/projects/llm_framework/main_cv/src/main.cpp @@ -0,0 +1,489 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "EngineWrapper.hpp" +#include "base/common.hpp" +#include "base/detection.h" +#include +#include +#include + +#include "../../../../SDK/components/utilities/include/sample_log.h" + +using namespace StackFlows; + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + SLOGW("llm_cv will be exit!"); + main_exit_flage = 1; +} + +static std::string base_model_path_; +static std::string base_model_config_path_; + +typedef struct { + std::string cv_model; + int img_h = 640; + int img_w = 640; + int cls_num = 80; + float pron_threshold = 0.45f; + float nms_threshold = 0.45; +} cv_config; + +typedef std::function &data, bool finish)> task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: +public: + cv_config mode_config_; + std::string model_; + std::unique_ptr cv_; + std::string response_format_; + std::vector inputs_; + std::vector image_data_; + bool enoutput_; + bool enstream_; + static int ax_init_flage_; + task_callback_t out_callback_; + + bool parse_config(const nlohmann::json &config_body) + { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } else + throw std::string("error"); + } catch (...) { + SLOGE("setup config_body error"); + return true; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + nlohmann::json file_body; + std::list config_file_paths; + config_file_paths.push_back(std::string("./") + model_ + ".json"); + config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + CONFIG_AUTO_SET(file_body["mode_param"], cv_model); + CONFIG_AUTO_SET(file_body["mode_param"], img_h); + CONFIG_AUTO_SET(file_body["mode_param"], img_w); + CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); + mode_config_.cv_model = base_model + mode_config_.cv_model; + cv_ = std::make_unique(); + if (0 != cv_->Init(mode_config_.cv_model.c_str())) { + SLOGE("Init cv_model model failed!\n"); + return -5; + } + } catch (...) { + SLOGE("config false"); + return -6; + } + return 0; + } + + std::string format_float(double value, int decimal_places) + { + std::ostringstream out; + out << std::fixed << std::setprecision(decimal_places) << value; + return out.str(); + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + void inference(const std::string &msg) + { + try { + if ((image_data_.empty())) return; + cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR); + if (src.empty()) return; + + std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); + common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); + int ret = -1; + cv_->SetInput(image.data(), 0); + if (0 != cv_->RunSync()) { + SLOGE("Run cv model failed!\n"); + throw std::string("cv_ RunSync error"); + } + std::vector objects; + cv_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, + mode_config_.pron_threshold, mode_config_.nms_threshold, objects); + std::vector yolo_output; + for (size_t i = 0; i < objects.size(); i++) { + const detection::Object &obj = objects[i]; + nlohmann::json output; + output["class"] = obj.label; + output["confidence"] = format_float(obj.prob * 100, 2); + output["bbox"] = nlohmann::json::array(); + output["bbox"].push_back(format_float(obj.rect.x, 0)); + output["bbox"].push_back(format_float(obj.rect.y, 0)); + output["bbox"].push_back(format_float(obj.rect.x + obj.rect.width, 0)); + output["bbox"].push_back(format_float(obj.rect.y + obj.rect.height, 0)); + yolo_output.push_back(output); + if (out_callback_) out_callback_(yolo_output, false); + } + if (out_callback_) out_callback_(yolo_output, true); + } catch (...) { + SLOGW("CV_->Run have error!"); + } + } + + void _ax_init() + { + if (!ax_init_flage_) { + int ret = AX_SYS_Init(); + if (0 != ret) { + fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret); + } + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); + ret = AX_ENGINE_Init(&npu_attr); + if (0 != ret) { + fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret); + } + } + ax_init_flage_++; + } + + void _ax_deinit() + { + if (ax_init_flage_ > 0) { + --ax_init_flage_; + if (!ax_init_flage_) { + AX_ENGINE_Deinit(); + AX_SYS_Deinit(); + } + } + } + + llm_task(const std::string &workid) + { + _ax_init(); + } + + ~llm_task() + { + _ax_deinit(); + } +}; +int llm_task::ax_init_flage_ = 0; +#undef CONFIG_AUTO_SET + +class llm_cv : public StackFlow { +private: + int task_count_; + std::unordered_map> llm_task_; + int _load_config() + { + if (base_model_path_.empty()) { + base_model_path_ = sys_sql_select("config_base_mode_path"); + } + if (base_model_config_path_.empty()) { + base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); + } + if (base_model_path_.empty() || base_model_config_path_.empty()) { + return -1; + } else { + SLOGI("llm_cv::_load_config success"); + return 0; + } + } + +public: + llm_cv() : StackFlow("cv") + { + task_count_ = 1; + repeat_event(1000, std::bind(&llm_cv::_load_config, this)); + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::vector &data, + bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + for (const auto &jsonObj : data) { + data_body["delta"].push_back(jsonObj); + } + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + SLOGI("send utf-8"); + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (data.empty() || (data == "None")) return; + nlohmann::json error_body; + const std::string *next_data = &data; + bool enbase64 = (object.find("base64") == std::string::npos) ? false : true; + bool enstream = (object.find("stream") == std::string::npos) ? false : true; + int ret; + std::string tmp_msg1; + if (enstream) { + static std::unordered_map stream_buff; + if (decode_stream(data, tmp_msg1, stream_buff)) return; + next_data = &tmp_msg1; + } + std::string tmp_msg2; + if (enbase64) { + ret = decode_base64((*next_data), tmp_msg2); + if (!ret) { + return; + } + next_data = &tmp_msg2; + } + if (object.find("jpeg") != std::string::npos) { + llm_task_obj->image_data_.assign(next_data->begin(), next_data->end()); + llm_task_obj->inference((*next_data)); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, "cv"); + return -1; + } + + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + SLOGE("setup json format error."); + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, "cv"); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + + llm_task_obj->set_output(std::bind(&llm_cv::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + + for (const auto input : llm_task_obj->inputs_) { + if (input.find("cv") != std::string::npos) { + llm_channel->subscriber_work_id( + "", std::bind(&llm_cv::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + } + } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + SLOGE("load_mode Failed"); + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, "cv"); + return -1; + } + } + + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_cv::link:%s", data.c_str()); + int ret = 1; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + if (data.find("cv") != std::string::npos) { + ret = llm_channel->subscriber_work_id( + data, + std::bind(&llm_cv::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->inputs_.push_back(data); + } + if (ret) { + error_body["code"] = -20; + error_body["message"] = "link false"; + send("None", "None", error_body, work_id); + return; + } else { + send("None", "None", LLM_NO_ERROR, work_id); + } + } + + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_cv::unlink:%s", data.c_str()); + int ret = 0; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + llm_channel->stop_subscriber_work_id(data); + auto llm_task_obj = llm_task_[work_id_num]; + for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { + if (*it == data) { + it = llm_task_obj->inputs_.erase(it); + } else { + ++it; + } + } + send("None", "None", LLM_NO_ERROR, work_id); + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_cv::taskinfo:%s", data.c_str()); + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("cv.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs_"] = llm_task_obj->inputs_; + send("cv.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_cv::exit:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_cv() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main() +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_cv llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp new file mode 100644 index 00000000..f199ee09 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp @@ -0,0 +1,311 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ +#include "EngineWrapper.hpp" +#include "utils/io.hpp" +#include +#include "base/detection.hpp" + +static const char *strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; + +/// @brief npu type +typedef enum axNPU_TYPE_E { + AX_NPU_DEFAULT = 0, /* running under default NPU according to system */ + AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ + AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ + AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ + AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ + AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ +} AX_NPU_TYPE_E; + +static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { + AX_ENGINE_NPU_ATTR_T stNpuAttr; + memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); + + auto ret = AX_ENGINE_GetVNPUAttr(&stNpuAttr); + if (ret == 0) { + // VNPU DISABLE + if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_DISABLE) { + nNpuSet = 0x01; // NON-VNPU (0b111) + } + // STD VNPU + else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { + // 7.2T & 10.8T no allow + if (eModelType == AX_ENGINE_MODEL_TYPE1 + || eModelType == AX_ENGINE_MODEL_TYPE1) { + return -1; + } + + // default STD VNPU2 + if (nNpuType == 0) { + nNpuSet = 0x02; // VNPU2 (0b010) + } + else { + if (nNpuType & AX_STD_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + if (nNpuType & AX_STD_VNPU_2) { + nNpuSet |= 0x02; // VNPU2 (0b010) + } + if (nNpuType & AX_STD_VNPU_3) { + nNpuSet |= 0x04; // VNPU3 (0b100) + } + } + } + // BL VNPU + else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { + // 10.8T no allow + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + return -1; + } + + // default BL VNPU + if (nNpuType == 0) { + // 7.2T default BL VNPU1 + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + nNpuSet = 0x01; // VNPU1 (0b001) + } + // 3.6T default BL VNPU2 + else { + nNpuSet = 0x02; // VNPU2 (0b010) + } + } + else { + // 7.2T + if (eModelType == AX_ENGINE_MODEL_TYPE1) { + // no allow set to BL VNPU2 + if (nNpuType & AX_BL_VNPU_2) { + return -1; + } + if (nNpuType & AX_BL_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + } + // 3.6T + else { + if (nNpuType & AX_BL_VNPU_1) { + nNpuSet |= 0x01; // VNPU1 (0b001) + } + if (nNpuType & AX_BL_VNPU_2) { + nNpuSet |= 0x02; // VNPU2 (0b010) + } + } + } + } + } + else { + printf("AX_ENGINE_GetVNPUAttr fail ret = %x\n", ret); + } + + return ret; +} + +int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) +{ + AX_S32 ret = 0; + + // 1. load model + AX_BOOL bLoadModelUseCmm = AX_FALSE; + AX_CHAR *pModelBufferVirAddr = nullptr; + AX_U64 u64ModelBufferPhyAddr = 0; + AX_U32 nModelBufferSize = 0; + + std::vector model_buffer; + + if (bLoadModelUseCmm) { + if (!utils::read_file(strModelPath, (AX_VOID **)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { + printf("ALGO read model(%s) fail\n", strModelPath); + return -1; + } + } + else { + if (!utils::read_file(strModelPath, model_buffer)) { + printf("ALGO read model(%s) fail\n", strModelPath); + return -1; + } + + pModelBufferVirAddr = model_buffer.data(); + nModelBufferSize = model_buffer.size(); + } + + auto freeModelBuffer = [&]() { + if (bLoadModelUseCmm) { + if (u64ModelBufferPhyAddr != 0) { + AX_SYS_MemFree(u64ModelBufferPhyAddr, &pModelBufferVirAddr); + } + } + else { + std::vector().swap(model_buffer); + } + return; + }; + + // 1.1 Get Model Type + AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; + ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); + if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { + printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); + freeModelBuffer(); + return -1; + } + + // 1.2 Check VNPU + AX_ENGINE_NPU_SET_T nNpuSet = 0; + ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); + if (0 != ret) { + printf("ALGO CheckModelVNpu fail\n"); + freeModelBuffer(); + return -1; + } + + // 2. create handle + AX_ENGINE_HANDLE handle = nullptr; + ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); + auto deinit_handle = [&handle]() { + if (handle) { + AX_ENGINE_DestroyHandle(handle); + } + return -1; + }; + + freeModelBuffer(); + + if (0 != ret || !handle) { + printf("ALGO Create model(%s) handle fail\n", strModelPath); + + return deinit_handle(); + } + + // 3. create context + ret = AX_ENGINE_CreateContext(handle); + if (0 != ret) { + return deinit_handle(); + } + + // 4. set io + m_io_info = nullptr; + ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); + if (0 != ret) { + return deinit_handle(); + } + m_input_num = m_io_info->nInputSize; + m_output_num = m_io_info->nOutputSize; + + // 4.1 query io +// AX_IMG_FORMAT_E eDtype; +// ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. +// if (0 != ret) { +// printf("model(%s) query model input size fail\n", strModelPath.c_str()); +// return deinit_handle(); +// } + +// if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || +// eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { +// printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); +// return deinit_handle(); +// } + + // 4.2 brief io +#ifdef __DEBUG__ + printf("brief_io_info\n"); + utils::brief_io_info(strModelPath, m_io_info); +#endif + + //5. Config VNPU + // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); + // will do nothing for using create handle v2 api + + // 6. prepare io + // AX_U32 nIoDepth = (stCtx.vecOutputBufferFlag.size() == 0) ? 1 : stCtx.vecOutputBufferFlag.size(); + ret = utils::prepare_io(strModelPath, m_io_info, m_io, utils::IO_BUFFER_STRATEGY_DEFAULT); + if (0 != ret) { + printf("prepare io failed!\n"); + utils::free_io(m_io); + return deinit_handle(); + } + + m_handle = handle; + m_hasInit = true; + + return 0; +} + +int EngineWrapper::SetInput(void* pInput, int index) { + return utils::push_io_input(pInput, index, m_io); +} + +int EngineWrapper::RunSync() +{ + if (!m_hasInit) + return -1; + + // 7.3 run & benchmark + auto ret = AX_ENGINE_RunSync(m_handle, &m_io); + if (0 != ret) { + printf("AX_ENGINE_RunSync failed. ret=0x%x\n", ret); + return ret; + } + + return 0; +} + +const char* CLASS_NAMES[] = { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", + "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", + "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", + "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", + "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", + "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", + "hair drier", "toothbrush"}; + +void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) +{ + // std::vector objects; + std::vector proposals; + for (int i = 0; i < 3; ++i) + { + auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, cls_num); + } + detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + fprintf(stdout, "detection num: %zu\n", objects.size()); + + detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); +} + +int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects) +{ + post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects); + return 0; +} + +int EngineWrapper::GetOutput(void* pOutput, int index) { + return utils::push_io_output(pOutput, index, m_io); +} + +int EngineWrapper::GetInputSize(int index) { + return m_io.pInputs[index].nSize; +} + +int EngineWrapper::GetOutputSize(int index) { + return m_io.pOutputs[index].nSize; +} + +int EngineWrapper::Release() +{ + if (m_handle) { + utils::free_io(m_io); + AX_ENGINE_DestroyHandle(m_handle); + m_handle = nullptr; + } + return 0; +} diff --git a/projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp b/projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp new file mode 100644 index 00000000..7da45496 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp @@ -0,0 +1,49 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ + +#pragma once + +#include +#include +#include +#include "ax_engine_api.h" + +class EngineWrapper { +public: + EngineWrapper() : + m_hasInit(false), + m_handle(nullptr) {} + + ~EngineWrapper() { + Release(); + } + + int Init(const char* strModelPath, uint32_t nNpuType = 0); + + int SetInput(void* pInput, int index); + + int RunSync(); + + int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects); + + int GetOutput(void* pOutput, int index); + + int GetInputSize(int index); + int GetOutputSize(int index); + + int Release(); + +protected: + bool m_hasInit; + AX_ENGINE_HANDLE m_handle; + AX_ENGINE_IO_INFO_T *m_io_info{}; + AX_ENGINE_IO_T m_io{}; + int m_input_num{}, m_output_num{}; +}; \ No newline at end of file diff --git a/projects/llm_framework/main_cv/src/runner/base/common.hpp b/projects/llm_framework/main_cv/src/runner/base/common.hpp new file mode 100644 index 00000000..cc5ab2a3 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/common.hpp @@ -0,0 +1,151 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: ls.wang + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace common +{ + // opencv mat(h, w) + // resize cv::Size(dstw, dsth) + void get_input_data_no_letterbox(const cv::Mat& mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) + { + cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); + cv::resize(mat, img_new, cv::Size(model_w, model_h)); + if (bgr2rgb) + { + cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); + } + } + + void get_input_data_letterbox(cv::Mat mat, std::vector& image, int letterbox_rows, int letterbox_cols, bool bgr2rgb = false) + { + /* letterbox process to support different letterbox size */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / mat.rows) < (letterbox_cols * 1.0 / mat.cols)) + { + scale_letterbox = (float)letterbox_rows * 1.0f / (float)mat.rows; + } + else + { + scale_letterbox = (float)letterbox_cols * 1.0f / (float)mat.cols; + } + resize_cols = int(scale_letterbox * (float)mat.cols); + resize_rows = int(scale_letterbox * (float)mat.rows); + + cv::Mat img_new(letterbox_rows, letterbox_cols, CV_8UC3, image.data()); + + cv::resize(mat, mat, cv::Size(resize_cols, resize_rows)); + + int top = (letterbox_rows - resize_rows) / 2; + int bot = (letterbox_rows - resize_rows + 1) / 2; + int left = (letterbox_cols - resize_cols) / 2; + int right = (letterbox_cols - resize_cols + 1) / 2; + + // Letterbox filling + cv::copyMakeBorder(mat, img_new, top, bot, left, right, cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); + if (bgr2rgb) + { + cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); + } + } + + void get_input_data_centercrop(cv::Mat mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) + { + /* letterbox process to support different letterbox size */ + + /* C2C BGR */ + if (mat.channels() == 4) + { + cv::cvtColor(mat, mat, cv::COLOR_BGRA2BGR); + } + + if (mat.channels() == 1) + { + cv::cvtColor(mat, mat, cv::COLOR_GRAY2BGR); + } + + /* Center */ + int h0; + int w0; + if (mat.rows < mat.cols) + { + h0 = 256; + w0 = int(mat.cols * (256.0 / mat.rows)); + } + else + { + h0 = int(mat.rows * (256.0 / mat.cols)); + w0 = 256; + } + int center_h = int(h0 / 2); + int center_w = int(w0 / 2); + + cv::resize(mat, mat, cv::Size(w0, h0)); + + // cv::imwrite("center.jpg", mat); + + /* Crop */ + cv::Rect crop_box(center_w - int(model_w / 2), center_h - int(model_h / 2), model_w, model_h); + cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); + + cv::Mat mat_crop = mat(crop_box).clone(); + + // cv::imwrite("mat_crop.jpg", mat_crop); + mat_crop.copyTo(img_new); + + // cv::imwrite("img_new.jpg", img_new); + + /* SwapRB*/ + if (bgr2rgb) + { + cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); + } + } + + bool read_file(const char* fn, std::vector& data) + { + FILE* fp = fopen(fn, "r"); + if (fp != nullptr) + { + fseek(fp, 0L, SEEK_END); + auto len = ftell(fp); + fseek(fp, 0, SEEK_SET); + data.clear(); + size_t read_size = 0; + if (len > 0) + { + data.resize(len); + read_size = fread(data.data(), 1, len, fp); + } + fclose(fp); + return read_size == (size_t)len; + } + return false; + } +} // namespace common \ No newline at end of file diff --git a/projects/llm_framework/main_cv/src/runner/base/detection.h b/projects/llm_framework/main_cv/src/runner/base/detection.h new file mode 100644 index 00000000..cebeba13 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/detection.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace detection +{ + + typedef struct Object + { + cv::Rect_ rect; + int label; + float prob; + cv::Point2f landmark[5]; + /* for yolov5-seg */ + cv::Mat mask; + std::vector mask_feat; + std::vector kps_feat; + /* for yolov8-obb */ + float angle; + } Object; + +} \ No newline at end of file diff --git a/projects/llm_framework/main_cv/src/runner/base/detection.hpp b/projects/llm_framework/main_cv/src/runner/base/detection.hpp new file mode 100644 index 00000000..775eed05 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/detection.hpp @@ -0,0 +1,3122 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: ls.wang + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +namespace detection +{ + typedef struct + { + int grid0; + int grid1; + int stride; + } GridAndStride; + + // typedef struct Object + // { + // cv::Rect_ rect; + // int label; + // float prob; + // cv::Point2f landmark[5]; + // /* for yolov5-seg */ + // cv::Mat mask; + // std::vector mask_feat; + // std::vector kps_feat; + // /* for yolov8-obb */ + // float angle; + // } Object; + + /* for palm detection */ + typedef struct PalmObject + { + cv::Rect_ rect; + float prob; + cv::Point2f vertices[4]; + cv::Point2f landmarks[7]; + cv::Mat affine_trans_mat; + cv::Mat affine_trans_mat_inv; + } PalmObject; + + static inline float sigmoid(float x) + { + return static_cast(1.f / (1.f + exp(-x))); + } + + static float softmax(const float* src, float* dst, int length) + { + const float alpha = *std::max_element(src, src + length); + float denominator = 0; + float dis_sum = 0; + for (int i = 0; i < length; ++i) + { + dst[i] = exp(src[i] - alpha); + denominator += dst[i]; + } + for (int i = 0; i < length; ++i) + { + dst[i] /= denominator; + dis_sum += i * dst[i]; + } + return dis_sum; + } + + template + static inline float intersection_area(const T& a, const T& b) + { + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); + } + + template + static void qsort_descent_inplace(std::vector& faceobjects, int left, int right) + { + int i = left; + int j = right; + float p = faceobjects[(left + right) / 2].prob; + + while (i <= j) + { + while (faceobjects[i].prob > p) + i++; + + while (faceobjects[j].prob < p) + j--; + + if (i <= j) + { + // swap + std::swap(faceobjects[i], faceobjects[j]); + + i++; + j--; + } + } +#pragma omp parallel sections + { +#pragma omp section + { + if (left < j) qsort_descent_inplace(faceobjects, left, j); + } +#pragma omp section + { + if (i < right) qsort_descent_inplace(faceobjects, i, right); + } + } + } + + template + static void qsort_descent_inplace(std::vector& faceobjects) + { + if (faceobjects.empty()) + return; + + qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1); + } + + template + static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold) + { + picked.clear(); + + const int n = faceobjects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) + { + areas[i] = faceobjects[i].rect.area(); + } + + for (int i = 0; i < n; i++) + { + const T& a = faceobjects[i]; + + int keep = 1; + for (int j = 0; j < (int)picked.size(); j++) + { + const T& b = faceobjects[picked[j]]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } + } + + static void generate_grids_and_stride(const int target_w, const int target_h, std::vector& strides, std::vector& grid_strides) + { + for (auto stride : strides) + { + int num_grid_w = target_w / stride; + int num_grid_h = target_h / stride; + for (int g1 = 0; g1 < num_grid_h; g1++) + { + for (int g0 = 0; g0 < num_grid_w; g0++) + { + GridAndStride gs; + gs.grid0 = g0; + gs.grid1 = g1; + gs.stride = stride; + grid_strides.push_back(gs); + } + } + } + } + + static void generate_proposals_scrfd(int feat_stride, const float* score_blob, + const float* bbox_blob, const float* kps_blob, + float prob_threshold, std::vector& faceobjects, int letterbox_cols, int letterbox_rows) + { + static float anchors[] = {-8.f, -8.f, 8.f, 8.f, -16.f, -16.f, 16.f, 16.f, -32.f, -32.f, 32.f, 32.f, -64.f, -64.f, 64.f, 64.f, -128.f, -128.f, 128.f, 128.f, -256.f, -256.f, 256.f, 256.f}; + int feat_w = letterbox_cols / feat_stride; + int feat_h = letterbox_rows / feat_stride; + int feat_size = feat_w * feat_h; + int anchor_group = 1; + if (feat_stride == 8) + anchor_group = 1; + if (feat_stride == 16) + anchor_group = 2; + if (feat_stride == 32) + anchor_group = 3; + + // generate face proposal from bbox deltas and shifted anchors + const int num_anchors = 2; + + for (int q = 0; q < num_anchors; q++) + { + // shifted anchor + float anchor_y = anchors[(anchor_group - 1) * 8 + q * 4 + 1]; + + float anchor_w = anchors[(anchor_group - 1) * 8 + q * 4 + 2] - anchors[(anchor_group - 1) * 8 + q * 4 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 8 + q * 4 + 3] - anchors[(anchor_group - 1) * 8 + q * 4 + 1]; + + for (int i = 0; i < feat_h; i++) + { + float anchor_x = anchors[(anchor_group - 1) * 8 + q * 4 + 0]; + + for (int j = 0; j < feat_w; j++) + { + int index = i * feat_w + j; + + float prob = sigmoid(score_blob[q * feat_size + index]); + + if (prob >= prob_threshold) + { + // insightface/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py _get_bboxes_single() + float dx = bbox_blob[(q * 4 + 0) * feat_size + index] * feat_stride; + float dy = bbox_blob[(q * 4 + 1) * feat_size + index] * feat_stride; + float dw = bbox_blob[(q * 4 + 2) * feat_size + index] * feat_stride; + float dh = bbox_blob[(q * 4 + 3) * feat_size + index] * feat_stride; + // insightface/detection/scrfd/mmdet/core/bbox/transforms.py distance2bbox() + float cx = anchor_x + anchor_w * 0.5f; + float cy = anchor_y + anchor_h * 0.5f; + + float x0 = cx - dx; + float y0 = cy - dy; + float x1 = cx + dw; + float y1 = cy + dh; + + Object obj; + obj.label = 0; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0 + 1; + obj.rect.height = y1 - y0 + 1; + obj.prob = prob; + + if (kps_blob != 0) + { + obj.landmark[0].x = cx + kps_blob[index] * feat_stride; + obj.landmark[0].y = cy + kps_blob[1 * feat_h * feat_w + index] * feat_stride; + obj.landmark[1].x = cx + kps_blob[2 * feat_h * feat_w + index] * feat_stride; + obj.landmark[1].y = cy + kps_blob[3 * feat_h * feat_w + index] * feat_stride; + obj.landmark[2].x = cx + kps_blob[4 * feat_h * feat_w + index] * feat_stride; + obj.landmark[2].y = cy + kps_blob[5 * feat_h * feat_w + index] * feat_stride; + obj.landmark[3].x = cx + kps_blob[6 * feat_h * feat_w + index] * feat_stride; + obj.landmark[3].y = cy + kps_blob[7 * feat_h * feat_w + index] * feat_stride; + obj.landmark[4].x = cx + kps_blob[8 * feat_h * feat_w + index] * feat_stride; + obj.landmark[4].y = cy + kps_blob[9 * feat_h * feat_w + index] * feat_stride; + } + + faceobjects.push_back(obj); + } + + anchor_x += feat_stride; + } + + anchor_y += feat_stride; + } + } + } + + static void generate_proposals_mobilenet_ssd(const float* score, const float* boxes, const int head_count, const int* feature_map_size, const int* anchor_size, const int cls_num, + float prob_threshold, const float* strides, const float center_val, const float scale_val, const float* anchor_info, std::vector& objects) + { + auto ptr_score = score; + auto ptr_boxes = boxes; + auto ptr_anchor_info = anchor_info; + for (int head = 0; head < head_count; ++head) + { + for (int fea_h = 0; fea_h < feature_map_size[head]; ++fea_h) + { + for (int fea_w = 0; fea_w < feature_map_size[head]; ++fea_w) + { + for (int anchor_i = 0; anchor_i < anchor_size[head]; ++anchor_i) + { + float softmax_sum = 0; + float class_score = -FLT_MAX; + int class_index = 0; + for (int s = 0; s < cls_num + 1; s++) + { + softmax_sum += std::exp(ptr_score[s]); + } + for (int i = 0; i < cls_num + 1; ++i) + { + float temp = std::exp(ptr_score[i]) / softmax_sum; + // if (temp > class_score) + // { + // class_index = i; + // class_score = temp; + // } + + class_index = i; + class_score = temp; + + if (temp >= prob_threshold and class_index != 0) + { + // fprintf(stderr, "class_score: %f %d \n", class_score, i); + + float pred_x = (((float)fea_w + 0.5f) / (300.0f / strides[head]) + ptr_boxes[0] * center_val * ptr_anchor_info[anchor_i * 2] / 300.0f); + float pred_y = (((float)fea_h + 0.5f) / (300.0f / strides[head]) + ptr_boxes[1] * center_val * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f); + float pred_w = std::exp(ptr_boxes[2] * scale_val) * ptr_anchor_info[anchor_i * 2] / 300.0f; + float pred_h = std::exp(ptr_boxes[3] * scale_val) * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f; + + float x0 = (pred_x - pred_w * 0.5f) * 300.0f; + float y0 = (pred_y - pred_h * 0.5f) * 300.0f; + float x1 = (pred_x + pred_w * 0.5f) * 300.0f; + float y1 = (pred_y + pred_h * 0.5f) * 300.0f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = class_score; + + objects.push_back(obj); + } + } + + ptr_score += cls_num + 1; + ptr_boxes += 4; + } + } + } + ptr_anchor_info += anchor_size[head] * 2; + } + } + + static void generate_proposals_yolox(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + + auto feat_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + float box_objectness = feat_ptr[4]; + if (box_objectness < prob_threshold) + { + feat_ptr += cls_num + 5; + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feat_ptr[s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = box_objectness * class_score; + + if (box_prob > prob_threshold) + { + float x_center = (feat_ptr[0] + w) * stride; + float y_center = (feat_ptr[1] + h) * stride; + float w = exp(feat_ptr[2]) * stride; + float h = exp(feat_ptr[3]) * stride; + float x0 = x_center - w * 0.5f; + float y0 = y_center - h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = w; + obj.rect.height = h; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_ptr += cls_num + 5; + } + } + } + + static void generate_proposals_yolov7(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + + auto feat_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a_index = 0; a_index < 3; ++a_index) + { + float box_objectness = feat_ptr[4]; + if (box_objectness < prob_threshold) + { + feat_ptr += cls_num + 5; + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feat_ptr[s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = box_objectness * class_score; + + if (box_prob > prob_threshold) + { + float x_center = (feat_ptr[0] * 2 - 0.5f + (float)w) * (float)stride; + float y_center = (feat_ptr[1] * 2 - 0.5f + (float)h) * (float)stride; + float box_w = (feat_ptr[2] * 2) * (feat_ptr[2] * 2) * anchors[a_index * 2]; + float box_h = (feat_ptr[3] * 2) * (feat_ptr[3] * 2) * anchors[a_index * 2 + 1]; + float x0 = x_center - box_w * 0.5f; + float y0 = y_center - box_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = box_w; + obj.rect.height = box_h; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_ptr += cls_num + 5; + } + } + } + } + static void generate_proposals_yolov5_face(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int cls_num = 1; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5 + 10); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5 + 10]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + + const float* landmark_ptr = feature_ptr + 5; + for (int l = 0; l < 5; l++) + { + float lx = landmark_ptr[l * 2 + 0]; + float ly = landmark_ptr[l * 2 + 1]; + lx = lx * anchor_w + w * stride; + ly = ly * anchor_h + h * stride; + obj.landmark[l] = cv::Point2f(lx, ly); + } + + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5 + 10); + } + } + } + } + + static void generate_proposals_yolov5_license_plate(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int cls_num = 1; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5 + 8); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5 + 8]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + + const float* landmark_ptr = feature_ptr + 5; + for (int l = 0; l < 4; l++) + { + float lx = landmark_ptr[l * 2 + 0]; + float ly = landmark_ptr[l * 2 + 1]; + lx = lx * anchor_w + w * stride; + ly = ly * anchor_h + h * stride; + obj.landmark[l] = cv::Point2f(lx, ly); + } + + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5 + 8); + } + } + } + } + + static void generate_proposals_yolov5(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5); + } + } + } + } + + static void generate_proposals_yolov5_seg(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80, int mask_proto_dim = 32) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5 + mask_proto_dim); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + obj.mask_feat.resize(mask_proto_dim); + for (int k = 0; k < mask_proto_dim; k++) + { + obj.mask_feat[k] = feature_ptr[cls_num + 5 + k]; + } + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5 + mask_proto_dim); + } + } + } + } + + static void generate_proposals_yolov5_visdrone(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 10) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5); + } + } + } + } + + static void generate_proposals_yolov6(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + + auto feat_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feat_ptr[s + 4]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = class_score; + + if (box_prob > prob_threshold) + { + float x0 = (w + 0.5f - feat_ptr[0]) * stride; + float y0 = (h + 0.5f - feat_ptr[1]) * stride; + float x1 = (w + 0.5f + feat_ptr[2]) * stride; + float y1 = (h + 0.5f + feat_ptr[3]) * stride; + + float w = x1 - x0; + float h = y1 - y0; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = w; + obj.rect.height = h; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_ptr += 84; + } + } + } + + static void generate_proposals_yolov7_face(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int cls_num = 1; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5 + 15); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5 + 15]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + + const float* landmark_ptr = feature_ptr + 6; + for (int l = 0; l < 5; l++) + { + float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; + float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; + //float score = sigmoid(landmark_ptr[3 * l + 2]); + obj.landmark[l] = cv::Point2f(lx, ly); + } + + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5 + 15); + } + } + } + } + + static void generate_proposals_yolov7_palm(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int cls_num = 1; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + const int landmark_sort[7] = {0, 3, 4, 5, 6, 1, 2}; + auto feature_ptr = feat; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + if (feature_ptr[4] < prob_threshold_unsigmoid) + { + feature_ptr += (cls_num + 5 + 21); + continue; + } + + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feature_ptr[s + 5 + 21]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feature_ptr[4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + float dx = sigmoid(feature_ptr[0]); + float dy = sigmoid(feature_ptr[1]); + float dw = sigmoid(feature_ptr[2]); + float dh = sigmoid(feature_ptr[3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + PalmObject obj; + obj.rect.x = x0 / (float)letterbox_cols; + obj.rect.y = y0 / (float)letterbox_rows; + obj.rect.width = (x1 - x0) / (float)letterbox_cols; + obj.rect.height = (y1 - y0) / (float)letterbox_rows; + obj.prob = final_score; + + const float* landmark_ptr = feature_ptr + 6; + std::vector tmp(7); + float min_x = FLT_MAX, min_y = FLT_MAX, max_x = 0, max_y = 0; + for (int l = 0; l < 7; l++) + { + float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; + float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; + lx /= (float)letterbox_cols; + ly /= (float)letterbox_rows; + + tmp[l] = cv::Point2f(lx, ly); + min_x = lx < min_x ? lx : min_x; + min_y = ly < min_y ? ly : min_y; + max_x = lx > max_x ? lx : max_x; + max_y = ly > max_y ? ly : max_y; + } + float w = max_x - min_x; + float h = max_y - min_y; + float long_side = h > w ? h : w; + long_side *= 1.1f; + obj.rect.x = min_x + w * 0.5f - long_side * 0.5f; + obj.rect.y = min_y + h * 0.5f - long_side * 0.5f; + obj.rect.width = long_side; + obj.rect.height = long_side; + for (int l = 0; l < 7; l++) + { + obj.landmarks[l] = tmp[landmark_sort[l]]; + } + + objects.push_back(obj); + } + + feature_ptr += (cls_num + 5 + 21); + } + } + } + } + + static void generate_proposals_yolov8(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto dfl_ptr = dfl_feat; + auto cls_ptr = cls_feat; + auto cls_idx_ptr = cls_idx; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + //process cls score + int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); + float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; + + float box_prob = sigmoid(class_score); + + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + dfl_ptr += (4 * reg_max); + } + } + } + + static void generate_proposals_yolov8_seg(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto dfl_ptr = dfl_feat; + auto cls_ptr = cls_feat; + auto cls_idx_ptr = cls_idx; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + //process cls score + int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); + float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + obj.mask_feat.resize(mask_proto_dim); + for (int k = 0; k < mask_proto_dim; k++) + { + obj.mask_feat[k] = dfl_ptr[4 * reg_max + k]; + } + objects.push_back(obj); + } + + dfl_ptr += (4 * reg_max + mask_proto_dim); + } + } + } + + static void generate_proposals_yolov8_pose(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const int num_point = 17) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + //process cls score + auto scores = feat; + auto bboxes = feat + 1; + auto kps = feat + 1 + 4 * reg_max; + + float box_prob = sigmoid(*scores); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(bboxes + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = 0; + obj.prob = box_prob; + obj.kps_feat.clear(); + for (int k = 0; k < num_point; k++) + { + float kps_x = (kps[k * 3] * 2.f + w) * stride; + float kps_y = (kps[k * 3 + 1] * 2.f + h) * stride; + float kps_s = sigmoid(kps[k * 3 + 2]); + obj.kps_feat.push_back(kps_x); + obj.kps_feat.push_back(kps_y); + obj.kps_feat.push_back(kps_s); + } + objects.push_back(obj); + } + feat += (1 + 4 * reg_max + 3 * num_point); + } + } + } + + static void generate_proposals_yolov9(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto feat_ptr = feat; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_ptr[s + 4 * reg_max]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_ptr += (cls_num + 4 * reg_max); + } + } + } + + static void generate_proposals_yolov8_native(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto feat_ptr = feat; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_ptr[s + 4 * reg_max]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_ptr += (cls_num + 4 * reg_max); + } + } + } + + static void generate_proposals_yolov8_seg_native(int stride, const float* feat, const float* feat_seg, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto feat_ptr = feat; + auto feat_seg_ptr = feat_seg; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_ptr[s + 4 * reg_max]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + obj.mask_feat.resize(mask_proto_dim); + memcpy(obj.mask_feat.data(), feat_seg_ptr, sizeof(float) * mask_proto_dim); + // for (int k = 0; k < mask_proto_dim; k++) + // { + // obj.mask_feat[k] = feat_seg_ptr[k]; + // } + objects.push_back(obj); + } + + feat_ptr += cls_num + 4 * reg_max; + feat_seg_ptr += mask_proto_dim; + } + } + } + + static void generate_proposals_yolov8_pose_native(int stride, const float* feat, const float* feat_kps, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const int num_point = 17, int cls_num = 1) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto feat_ptr = feat; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_ptr[s + 4 * reg_max]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + obj.kps_feat.clear(); + for (int k = 0; k < num_point; k++) + { + float kps_x = (feat_kps[k * 3] * 2.f + w) * stride; + float kps_y = (feat_kps[k * 3 + 1] * 2.f + h) * stride; + float kps_s = sigmoid(feat_kps[k * 3 + 2]); + obj.kps_feat.push_back(kps_x); + obj.kps_feat.push_back(kps_y); + obj.kps_feat.push_back(kps_s); + } + objects.push_back(obj); + } + feat_ptr += (cls_num + 4 * reg_max); + feat_kps += 3 * num_point; + } + } + } + + static void generate_proposals_yolo_world(int stride, const float* feat_cls, const float* feat_reg, float exp, float bias, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int reg_max = 16; + + auto feat_cls_ptr = feat_cls; + auto feat_reg_ptr = feat_reg; + + std::vector dis_after_sm(reg_max, 0.f); + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_cls_ptr[s]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + class_score = class_score * exp + bias; + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_reg_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * stride; + } + + float pb_cx = (w + 0.5f) * stride; + float pb_cy = (h + 0.5f) * stride; + + float x0 = pb_cx - pred_ltrb[0]; + float y0 = pb_cy - pred_ltrb[1]; + float x1 = pb_cx + pred_ltrb[2]; + float y1 = pb_cy + pred_ltrb[3]; + + x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = box_prob; + + objects.push_back(obj); + } + + feat_cls_ptr += cls_num; + feat_reg_ptr += 4 * reg_max; + } + } + } + + static void generate_proposals(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) + { + int anchor_num = 3; + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + int anchor_group; + if (stride == 8) + anchor_group = 1; + if (stride == 16) + anchor_group = 2; + if (stride == 32) + anchor_group = 3; + + int w_stride = (cls_num + 5); + int h_stride = feat_w * w_stride; + int a_stride = feat_h * h_stride; + + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + for (int a = 0; a <= anchor_num - 1; a++) + { + //process cls score + int class_index = 0; + float class_score = -FLT_MAX; + int offset = a * a_stride + h * h_stride + w * w_stride; + for (int s = 0; s <= cls_num - 1; s++) + { + float score = feat[offset + s + 5]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + //process box score + float box_score = feat[offset + 4]; + float final_score = sigmoid(box_score) * sigmoid(class_score); + + if (final_score >= prob_threshold) + { + int loc_idx = offset; + float dx = sigmoid(feat[loc_idx + 0]); + float dy = sigmoid(feat[loc_idx + 1]); + float dw = sigmoid(feat[loc_idx + 2]); + float dh = sigmoid(feat[loc_idx + 3]); + float pred_cx = (dx * 2.0f - 0.5f + w) * stride; + float pred_cy = (dy * 2.0f - 0.5f + h) * stride; + float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; + float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; + float pred_w = dw * dw * 4.0f * anchor_w; + float pred_h = dh * dh * 4.0f * anchor_h; + float x0 = pred_cx - pred_w * 0.5f; + float y0 = pred_cy - pred_h * 0.5f; + float x1 = pred_cx + pred_w * 0.5f; + float y1 = pred_cy + pred_h * 0.5f; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = final_score; + objects.push_back(obj); + } + } + } + } + } + + static void generate_proposals_palm(std::vector& region_list, float score_thresh, int input_img_w, int input_img_h, float* scores_ptr, float* bboxes_ptr, int head_count, const int* strides, const int* anchor_size, const float* anchor_offset, const int* feature_map_size, float prob_threshold_unsigmoid) + { + int idx = 0; + for (int i = 0; i < head_count; i++) + { + for (int y = 0; y < feature_map_size[i]; y++) + { + for (int x = 0; x < feature_map_size[i]; x++) + { + for (int k = 0; k < anchor_size[i]; k++) + { + if (scores_ptr[idx] < prob_threshold_unsigmoid) + { + idx++; + continue; + } + + const float x_center = (x + anchor_offset[i]) * 1.0f / feature_map_size[i]; + const float y_center = (y + anchor_offset[i]) * 1.0f / feature_map_size[i]; + float score = sigmoid(scores_ptr[idx]); + + if (score > score_thresh) + { + float* p = bboxes_ptr + (idx * 18); + + float cx = p[0] / input_img_w + x_center; + float cy = p[1] / input_img_h + y_center; + float w = p[2] / input_img_w; + float h = p[3] / input_img_h; + + float x0 = cx - w * 0.5f; + float y0 = cy - h * 0.5f; + float x1 = cx + w * 0.5f; + float y1 = cy + h * 0.5f; + + PalmObject region; + region.prob = score; + region.rect.x = x0; + region.rect.y = y0; + region.rect.width = x1 - x0; + region.rect.height = y1 - y0; + + for (int j = 0; j < 7; j++) + { + float lx = p[4 + (2 * j) + 0]; + float ly = p[4 + (2 * j) + 1]; + lx += x_center * input_img_w; + ly += y_center * input_img_h; + lx /= (float)input_img_w; + ly /= (float)input_img_h; + + region.landmarks[j].x = lx; + region.landmarks[j].y = ly; + } + region_list.push_back(region); + } + idx++; + } + } + } + } + } + + static void draw_objects(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name,double fontScale = 0.5, int thickness = 1) + { + static const std::vector COCO_COLORS = { + {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; + cv::Mat image = bgr.clone(); + + for (size_t i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, + obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); + + cv::rectangle(image, obj.rect, COCO_COLORS[obj.label], thickness); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > image.cols) + x = image.cols - label_size.width; + + cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(0, 0, 0), -1); + + cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, fontScale, + cv::Scalar(255, 255, 255), thickness); + } + + cv::imwrite(std::string(output_name) + ".jpg", image); + } + + static void draw_keypoints(const cv::Mat& bgr, const std::vector& objects, + const std::vector >& kps_colors, + const std::vector >& limb_colors, + const std::vector >& skeleton, + const char* output_name) + { + cv::Mat image = bgr.clone(); + + for (size_t i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + + fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], person\n", obj.label, obj.prob * 100, obj.rect.x, + obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); + + cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); + + char text[256]; + sprintf(text, "person %.1f%%", obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > image.cols) + x = image.cols - label_size.width; + + cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, + cv::Scalar(0, 0, 0)); + + const int num_point = obj.kps_feat.size() / 3; + for (int j = 0; j < num_point + 2; j++) + { + // draw circle + if (j < num_point) + { + int kps_x = std::round(obj.kps_feat[j * 3]); + int kps_y = std::round(obj.kps_feat[j * 3 + 1]); + float kps_s = obj.kps_feat[j * 3 + 2]; + if (kps_s > 0.5f) + { + auto kps_color = cv::Scalar(kps_colors[j][0], kps_colors[j][1], kps_colors[j][2]); + cv::circle(image, {kps_x, kps_y}, 5, kps_color, -1); + } + } + // draw line + auto& ske = skeleton[j]; + int pos1_x = obj.kps_feat[(ske[0] - 1) * 3]; + int pos1_y = obj.kps_feat[(ske[0] - 1) * 3 + 1]; + + int pos2_x = obj.kps_feat[(ske[1] - 1) * 3]; + int pos2_y = obj.kps_feat[(ske[1] - 1) * 3 + 1]; + + float pos1_s = obj.kps_feat[(ske[0] - 1) * 3 + 2]; + float pos2_s = obj.kps_feat[(ske[1] - 1) * 3 + 2]; + + if (pos1_s > 0.5f && pos2_s > 0.5f) + { + auto limb_color = cv::Scalar(limb_colors[j][0], limb_colors[j][1], limb_colors[j][2]); + cv::line(image, {pos1_x, pos1_y}, {pos2_x, pos2_y}, limb_color, 2); + } + } + } + cv::imwrite(std::string(output_name) + ".jpg", image); + } + + static void draw_objects_mask(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const std::vector >& colors, const char* output_name) + { + cv::Mat image = bgr.clone(); + cv::Mat mask = bgr.clone(); + int color_index = 0; + for (size_t i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + + const auto& color = colors[color_index % 80]; + color_index++; + + fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, + obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); + + mask(cv::Rect((int)obj.rect.x, (int)obj.rect.y, (int)objects[i].rect.width, (int)objects[i].rect.height)).setTo(color, objects[i].mask); + + cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > image.cols) + x = image.cols - label_size.width; + + cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, + cv::Scalar(0, 0, 0)); + } + float blended_alpha = 0.5; + image = (1 - blended_alpha) * mask + blended_alpha * image; + + cv::imwrite(std::string(output_name) + ".jpg", image); + } + + static void draw_objects_palm(const cv::Mat& bgr, const std::vector& objects, const char* output_name) + { + cv::Mat image = bgr.clone(); + for (size_t i = 0; i < objects.size(); i++) + { + const PalmObject& obj = objects[i]; + //fprintf(stdout, "prob:%.2f, x0:%.2f, y0:%.2f, x1:%.2f, y1:%.2f, x2:%.2f, y2:%.2f, x3:%.2f, y3:%.2f\n", obj.prob, + // obj.vertices[0].x, obj.vertices[0].y, obj.vertices[1].x, obj.vertices[1].y, obj.vertices[2].x, + // obj.vertices[2].y, obj.vertices[3].x, obj.vertices[3].y); + cv::line(image, obj.vertices[0], obj.vertices[1], cv::Scalar(0, 0, 255), 2, 8, 0); + cv::line(image, obj.vertices[1], obj.vertices[2], cv::Scalar(0, 0, 255), 2, 8, 0); + cv::line(image, obj.vertices[2], obj.vertices[3], cv::Scalar(0, 0, 255), 2, 8, 0); + cv::line(image, obj.vertices[3], obj.vertices[0], cv::Scalar(0, 0, 255), 2, 8, 0); + for (auto ld : obj.landmarks) + { + cv::circle(image, ld, 2, cv::Scalar(0, 255, 0), -1, 8); + } + } + cv::imwrite(std::string(output_name) + ".jpg", image); + } + + static void draw_objects_yolopv2(const cv::Mat& bgr, const std::vector& objects, const cv::Mat& da_seg_mask, const cv::Mat& ll_seg_mask, const char* output_name) + { + cv::Mat image = bgr.clone(); + cv::Mat mask = bgr.clone(); + for (size_t i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + + fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f]\n", obj.label, obj.prob * 100, obj.rect.x, + obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); + + cv::rectangle(image, obj.rect, cv::Scalar(0, 255, 255), 2, 8, 0); + } + + mask.setTo(cv::Scalar(0, 255, 0), da_seg_mask); + mask.setTo(cv::Scalar(0, 0, 255), ll_seg_mask); + float blended_alpha = 0.5; + image = (1 - blended_alpha) * mask + blended_alpha * image; + + cv::imwrite(std::string(output_name) + ".jpg", image); + } + + void reverse_letterbox(std::vector& proposal, std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = proposal.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposal[i]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + } + + void get_out_bbox_no_letterbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int model_h, int model_w, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov5 draw the result */ + float ratio_x = (float)src_cols / (float)model_w; + float ratio_y = (float)src_rows / (float)model_h; + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0)*ratio_x; + y0 = (y0)*ratio_y; + x1 = (x1)*ratio_x; + y1 = (y1)*ratio_y; + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + } + + void get_out_bbox(std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + /* yolov5 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = objects.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + for (int l = 0; l < 5; l++) + { + auto lx = objects[i].landmark[l].x; + auto ly = objects[i].landmark[l].y; + objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); + } + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + } + + void get_out_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov5 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + for (int l = 0; l < 5; l++) + { + auto lx = objects[i].landmark[l].x; + auto ly = objects[i].landmark[l].y; + objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); + } + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + } + + void get_out_bbox_mask(std::vector& proposals, std::vector& objects, const float* mask_proto, int mask_proto_dim, int mask_stride, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov5 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int mask_proto_h = int(letterbox_rows / mask_stride); + int mask_proto_w = int(letterbox_cols / mask_stride); + + int count = picked.size(); + objects.resize(count); + + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + /* naive RoiAlign by opencv */ + int hstart = std::floor(objects[i].rect.y / mask_stride); + int hend = std::ceil(objects[i].rect.y / mask_stride + objects[i].rect.height / mask_stride); + int wstart = std::floor(objects[i].rect.x / mask_stride); + int wend = std::ceil(objects[i].rect.x / mask_stride + objects[i].rect.width / mask_stride); + + hstart = std::min(std::max(hstart, 0), mask_proto_h); + wstart = std::min(std::max(wstart, 0), mask_proto_w); + hend = std::min(std::max(hend, 0), mask_proto_h); + wend = std::min(std::max(wend, 0), mask_proto_w); + + int mask_w = wend - wstart; + int mask_h = hend - hstart; + + cv::Mat mask = cv::Mat(mask_h, mask_w, CV_32FC1); + if (mask_w > 0 && mask_h > 0) + { + std::vector roi_ranges; + roi_ranges.push_back(cv::Range(0, 1)); + roi_ranges.push_back(cv::Range::all()); + roi_ranges.push_back(cv::Range(hstart, hend)); + roi_ranges.push_back(cv::Range(wstart, wend)); + + cv::Mat mask_protos = cv::Mat(mask_proto_dim, mask_proto_h * mask_proto_w, CV_32FC1, (float*)mask_proto); + int sz[] = {1, mask_proto_dim, mask_proto_h, mask_proto_w}; + cv::Mat mask_protos_reshape = mask_protos.reshape(1, 4, sz); + cv::Mat protos = mask_protos_reshape(roi_ranges).clone().reshape(0, {mask_proto_dim, mask_w * mask_h}); + cv::Mat mask_proposals = cv::Mat(1, mask_proto_dim, CV_32FC1, (float*)objects[i].mask_feat.data()); + cv::Mat masks_feature = (mask_proposals * protos); + /* sigmoid */ + cv::exp(-masks_feature.reshape(1, {mask_h, mask_w}), mask); + mask = 1.0 / (1.0 + mask); + } + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + cv::resize(mask, mask, cv::Size((int)objects[i].rect.width, (int)objects[i].rect.height)); + objects[i].mask = mask > 0.5; + } + } + + void get_out_bbox_kps(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov8 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + + for (int j = 0; j < objects[i].kps_feat.size() / 3; j++) + { + objects[i].kps_feat[j * 3] = std::max( + std::min((objects[i].kps_feat[j * 3] - tmp_w) * ratio_x, (float)(src_cols - 1)), 0.f); + objects[i].kps_feat[j * 3 + 1] = std::max( + std::min((objects[i].kps_feat[j * 3 + 1] - tmp_h) * ratio_y, (float)(src_rows - 1)), 0.f); + } + } + } + + static void transform_rects_palm(PalmObject& object) + { + float x0 = object.landmarks[0].x; + float y0 = object.landmarks[0].y; + float x1 = object.landmarks[2].x; + float y1 = object.landmarks[2].y; + float rotation = M_PI * 0.5f - std::atan2(-(y1 - y0), x1 - x0); + + float hand_cx; + float hand_cy; + float shift_x = 0.0f; + float shift_y = -0.5f; + if (rotation == 0) + { + hand_cx = object.rect.x + object.rect.width * 0.5f + (object.rect.width * shift_x); + hand_cy = object.rect.y + object.rect.height * 0.5f + (object.rect.height * shift_y); + } + else + { + float dx = (object.rect.width * shift_x) * std::cos(rotation) - (object.rect.height * shift_y) * std::sin(rotation); + float dy = (object.rect.width * shift_x) * std::sin(rotation) + (object.rect.height * shift_y) * std::cos(rotation); + hand_cx = object.rect.x + object.rect.width * 0.5f + dx; + hand_cy = object.rect.y + object.rect.height * 0.5f + dy; + } + + float long_side = (std::max)(object.rect.width, object.rect.height); + float dx = long_side * 1.3f; + float dy = long_side * 1.3f; + + object.vertices[0].x = -dx; + object.vertices[0].y = -dy; + object.vertices[1].x = +dx; + object.vertices[1].y = -dy; + object.vertices[2].x = +dx; + object.vertices[2].y = +dy; + object.vertices[3].x = -dx; + object.vertices[3].y = +dy; + + for (int i = 0; i < 4; i++) + { + float sx = object.vertices[i].x; + float sy = object.vertices[i].y; + object.vertices[i].x = sx * std::cos(rotation) - sy * std::sin(rotation); + object.vertices[i].y = sx * std::sin(rotation) + sy * std::cos(rotation); + object.vertices[i].x += hand_cx; + object.vertices[i].y += hand_cy; + } + } + + static void get_out_bbox_palm(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + transform_rects_palm(objects[i]); + } + + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_cols / resize_cols; + float ratio_y = (float)src_rows / resize_rows; + + for (auto& object : objects) + { + for (auto& vertice : object.vertices) + { + vertice.x = (vertice.x * letterbox_cols - tmp_w) * ratio_x; + vertice.y = (vertice.y * letterbox_rows - tmp_h) * ratio_y; + } + + for (auto& ld : object.landmarks) + { + ld.x = (ld.x * letterbox_cols - tmp_w) * ratio_x; + ld.y = (ld.y * letterbox_rows - tmp_h) * ratio_y; + } + // get warpaffine transform mat to landmark detect + cv::Point2f src_pts[4]; + src_pts[0] = object.vertices[0]; + src_pts[1] = object.vertices[1]; + src_pts[2] = object.vertices[2]; + src_pts[3] = object.vertices[3]; + + cv::Point2f dst_pts[4]; + dst_pts[0] = cv::Point2f(0, 0); + dst_pts[1] = cv::Point2f(224, 0); + dst_pts[2] = cv::Point2f(224, 224); + dst_pts[3] = cv::Point2f(0, 224); + + object.affine_trans_mat = cv::getAffineTransform(src_pts, dst_pts); + cv::invertAffineTransform(object.affine_trans_mat, object.affine_trans_mat_inv); + } + } + + void get_out_bbox_yolopv2(std::vector& proposals, std::vector& objects, const float* da_ptr, const float* ll_ptr, cv::Mat& ll_seg_mask, cv::Mat& da_seg_mask, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov5 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + float x0 = (objects[i].rect.x); + float y0 = (objects[i].rect.y); + float x1 = (objects[i].rect.x + objects[i].rect.width); + float y1 = (objects[i].rect.y + objects[i].rect.height); + + x0 = (x0 - tmp_w) * ratio_x; + y0 = (y0 - tmp_h) * ratio_y; + x1 = (x1 - tmp_w) * ratio_x; + y1 = (y1 - tmp_h) * ratio_y; + + x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + cv::Mat ll = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)ll_ptr); + ll = ll > 0.5; + cv::resize(ll(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), ll_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_LINEAR); + + cv::Mat da = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)da_ptr); + da = da > 0; + cv::resize(da(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), da_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_NEAREST); + } + + namespace mmyolo + { + inline static float clamp( + float val, + float min = 0.f, + float max = 1536.f) + { + return val > min ? (val < max ? val : max) : min; + } + + inline float fast_exp(const float& x) + { + union + { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; + } + + inline float fast_sigmoid(const float& x) + { + return 1.0f / (1.0f + fast_exp(-x)); + } + + inline static float fast_softmax( + const float* src, + float* dst, + int length) + { + const float alpha = *std::max_element(src, src + length); + float denominator = 0; + float dis_sum = 0; + for (int i = 0; i < length; ++i) + { + dst[i] = fast_exp(src[i] - alpha); + denominator += dst[i]; + } + for (int i = 0; i < length; ++i) + { + dst[i] /= denominator; + dis_sum += i * dst[i]; + } + return dis_sum; + } + + static void generate_proposals_ppyoloeplus( + int stride, + const float* cls_feat, + const float* box_feat, + float prob_threshold, + std::vector& objects, + int letterbox_cols, + int letterbox_rows, + int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + auto cls_ptr = cls_feat; + auto boxes_ptr = box_feat; + int reg_max = 17; + float dis_after_sm[reg_max]; + + for (int h = 0; h < feat_h; h++) + { + for (int w = 0; w < feat_w; w++) + { + auto max = std::max_element(cls_ptr, cls_ptr + cls_num); + float box_prob = fast_sigmoid(*max); + + if (box_prob > prob_threshold) + { + float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); + float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); + float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); + float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); + + x0 *= stride; + y0 *= stride; + x1 *= stride; + y1 *= stride; + + x0 = clamp(x0, 0.f, letterbox_cols - 1); + y0 = clamp(y0, 0.f, letterbox_rows - 1); + x1 = clamp(x1, 0.f, letterbox_cols - 1); + y1 = clamp(y1, 0.f, letterbox_rows - 1); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = max - cls_ptr; + obj.prob = box_prob; + + objects.push_back(obj); + } + cls_ptr += cls_num; + boxes_ptr += 4 * reg_max; + } + } + } + + static void generate_proposals_yolox( + int stride, + const float* cls_feat, + const float* box_feat, + const float* conf_feat, + float prob_threshold, + std::vector& objects, + int letterbox_cols, + int letterbox_rows, + int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + auto cls_ptr = cls_feat; + auto boxes_ptr = box_feat; + auto conf_ptr = conf_feat; + + for (int h = 0; h < feat_h; h++) + { + for (int w = 0; w < feat_w; w++) + { + //process cls score + auto max = std::max_element(cls_ptr, cls_ptr + cls_num); + float box_prob = fast_sigmoid(*max) * fast_sigmoid(*conf_ptr); + + if (box_prob > prob_threshold) + { + float x = (w + boxes_ptr[0]) * stride; + float y = (h + boxes_ptr[1]) * stride; + + float width = fast_exp(boxes_ptr[2]) * stride; + float height = fast_exp(boxes_ptr[3]) * stride; + + float x0 = x - width * 0.5f; + float y0 = y - height * 0.5f; + + x0 = clamp(x0, 0.f, letterbox_cols - 1); + y0 = clamp(y0, 0.f, letterbox_rows - 1); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = width; + obj.rect.height = height; + obj.label = max - cls_ptr; + obj.prob = box_prob; + + objects.push_back(obj); + } + cls_ptr += cls_num; + boxes_ptr += 4; + conf_ptr++; + } + } + } + + static void generate_proposals_yolov6( + int stride, + const float* cls_feat, + const float* box_feat, + float prob_threshold, + std::vector& objects, + int letterbox_cols, + int letterbox_rows, + int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + auto cls_ptr = cls_feat; + auto boxes_ptr = box_feat; + + for (int h = 0; h < feat_h; h++) + { + for (int w = 0; w < feat_w; w++) + { + //process cls score + auto max = std::max_element(cls_ptr, cls_ptr + cls_num); + float box_prob = fast_sigmoid(*max); + + if (box_prob > prob_threshold) + { + float x0 = (w + 0.5f - boxes_ptr[0]) * stride; + float y0 = (h + 0.5f - boxes_ptr[1]) * stride; + float x1 = (w + 0.5f + boxes_ptr[2]) * stride; + float y1 = (h + 0.5f + boxes_ptr[3]) * stride; + + x0 = clamp(x0, 0.f, letterbox_cols - 1); + y0 = clamp(y0, 0.f, letterbox_rows - 1); + x1 = clamp(x1, 0.f, letterbox_cols - 1); + y1 = clamp(y1, 0.f, letterbox_rows - 1); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = max - cls_ptr; + obj.prob = box_prob; + + objects.push_back(obj); + } + cls_ptr += cls_num; + boxes_ptr += 4; + } + } + } + + static void generate_proposals_yolov8( + int stride, + const float* cls_feat, + const float* box_feat, + float prob_threshold, + std::vector& objects, + int letterbox_cols, + int letterbox_rows, + int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + auto cls_ptr = cls_feat; + auto boxes_ptr = box_feat; + int reg_max = 16; + float dis_after_sm[reg_max]; + + for (int h = 0; h < feat_h; h++) + { + for (int w = 0; w < feat_w; w++) + { + auto max = std::max_element(cls_ptr, cls_ptr + cls_num); + float box_prob = fast_sigmoid(*max); + + if (box_prob > prob_threshold) + { + float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); + float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); + float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); + float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); + + x0 *= stride; + y0 *= stride; + x1 *= stride; + y1 *= stride; + + x0 = clamp(x0, 0.f, letterbox_cols - 1); + y0 = clamp(y0, 0.f, letterbox_rows - 1); + x1 = clamp(x1, 0.f, letterbox_cols - 1); + y1 = clamp(y1, 0.f, letterbox_rows - 1); + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = max - cls_ptr; + obj.prob = box_prob; + + objects.push_back(obj); + } + cls_ptr += cls_num; + boxes_ptr += 4 * reg_max; + } + } + } + } // namespace mmyolo + + namespace obb + { + static inline float fast_exp(const float& x) + { + union + { + uint32_t i; + float f; + } v; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; + } + + static inline float clamp_(float v, float min = 0.f, float max = std::numeric_limits::infinity()) + { + return std::max(std::min(v, max), min); + } + static inline void get_covariance_matrix( + const std::vector& objects, + std::vector& covar_maxtirx) + { + int n = objects.size(); + for (size_t i = 0; i < n; ++i) { + + float a = objects[i].rect.width * objects[i].rect.width * 0.0833333333f; + float b = objects[i].rect.height * objects[i].rect.height * 0.0833333333f; + float c = objects[i].angle; + + float c_sin = std::sin(c); + float c_cos = std::cos(c); + + float x = a * c_cos * c_cos + b * c_sin * c_sin; + float y = a * c_sin * c_sin + b * c_cos * c_cos; + float z = a * c_cos * c_sin - b * c_sin * c_cos; + covar_maxtirx.push_back(cv::Point3f(x, y, z)); + } + } + + static inline float fast_log2 (float x) + { + union { + float f; + int i; + } v; + v.f = x; + int* const exp_ptr = &v.i; + int m = *exp_ptr; + const int log_2 = ((m >> 23) & 255) - 128; + m &= ~(255 << 23); + m += 127 << 23; + *exp_ptr = m; + return (((-1.f / 3) * v.f + 2) * v.f - 2.f / 3 + log_2); + } + + static inline float fast_log (const float x) + { + return 0.69314718f * fast_log2 (x); + } + + static inline float probiou( + const Object& obj1, + const Object& obj2, + const cv::Point3f& covar1, + const cv::Point3f& covar2) + { + + float v_x1_x2 = covar1.x + covar2.x; + float v_y1_y2 = covar1.y + covar2.y; + float x1_x2 = obj1.rect.x - obj2.rect.x; + float y1_y2 = obj1.rect.y - obj2.rect.y; + float dem1 = clamp_(covar1.x * covar1.y - covar1.z * covar1.z); + float dem2 = clamp_(covar2.x * covar2.y - covar2.z * covar2.z); + float dem_num = v_x1_x2 * v_y1_y2 - (covar1.z + covar2.z) * (covar1.z + covar2.z); + + float t1 = (v_x1_x2 * y1_y2 * y1_y2 + v_y1_y2 * x1_x2 * x1_x2) / (dem_num + 1e-7) * 0.25f; + float t2 = ((covar1.z + covar2.z) * (-x1_x2) * y1_y2) / (dem_num + 1e-7) * 0.5f; + float t3 = fast_log(0.25f * dem_num / (std::sqrt(dem1 * dem2) + 1e-7) + 1e-7) * 0.5f; + + float bd = t1 + t2 + t3; + float iou = fast_exp(-clamp_(bd, 1e-7, 100.f)); + return iou; + } + + static inline void nms_rotated_sorted_bboxes( + const std::vector& objects, + std::vector& picked, + float nms_threshold) + { + std::vector covar_maxtrix; + get_covariance_matrix(objects, covar_maxtrix); + + float nms_threshold_ = (1.f - nms_threshold) * (1.f - nms_threshold); + int n = objects.size(); + + for (size_t i = 0; i < n; ++i) { + float max_iou = 0.f; + const Object& obj1 = objects[i]; + + for (size_t j = 0; j < i; ++j) { + const Object& obj2 = objects[j]; + + float iou = probiou(obj1, obj2, covar_maxtrix[i], covar_maxtrix[j]); + if (iou > max_iou){ + max_iou = iou; + } + + } + if (max_iou < nms_threshold_) + picked.push_back(i); + } + } + static void get_out_obb_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) + { + qsort_descent_inplace(proposals); + std::vector picked; + obb::nms_rotated_sorted_bboxes(proposals, picked, nms_threshold); + + /* yolov5 draw the result */ + float scale_letterbox; + int resize_rows; + int resize_cols; + if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) + { + scale_letterbox = letterbox_rows * 1.0 / src_rows; + } + else + { + scale_letterbox = letterbox_cols * 1.0 / src_cols; + } + resize_cols = int(scale_letterbox * src_cols); + resize_rows = int(scale_letterbox * src_rows); + + int tmp_h = (letterbox_rows - resize_rows) / 2; + int tmp_w = (letterbox_cols - resize_cols) / 2; + + float ratio_x = (float)src_rows / resize_rows; + float ratio_y = (float)src_cols / resize_cols; + + int count = picked.size(); + double pi = M_PI; + double pi_2 = M_PI_2; + objects.resize(count); + for (int i = 0; i < count; i++) { + objects[i] = proposals[picked[i]]; + + float w_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.width : objects[i].rect.height; + float h_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.height : objects[i].rect.width; + float a_ = (float)std::fmod((objects[i].rect.width > objects[i].rect.height ? objects[i].angle : objects[i].angle + pi_2), pi); + + float xc = (objects[i].rect.x - tmp_w) * ratio_x; + float yc = (objects[i].rect.y - tmp_h) * ratio_y; + float w = w_ * ratio_x; + float h = h_ * ratio_y; + + // clip + xc = std::max(std::min(xc, (float)(src_cols - 1)), 0.f); + yc = std::max(std::min(yc, (float)(src_rows - 1)), 0.f); + w = std::max(std::min(w, (float)(src_cols - 1)), 0.f); + h = std::max(std::min(h, (float)(src_rows - 1)), 0.f); + + + objects[i].rect.x = xc; + objects[i].rect.y = yc; + objects[i].rect.width = w; + objects[i].rect.height = h; + objects[i].angle = a_; + } + } + static void generate_proposals_yolov8_obb_native(const std::vector& grid_strides, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 15) + { + const int num_points = grid_strides.size(); + int reg_max = 16; + auto feat_ptr = feat; + std::vector dis_after_sm(reg_max, 0.f); + for (int i = 0; i < num_points; i++) + { + // process cls score + int class_index = 0; + float class_score = -FLT_MAX; + for (int s = 0; s < cls_num; s++) + { + float score = feat_ptr[s + 4 * reg_max]; + if (score > class_score) + { + class_index = s; + class_score = score; + } + } + + float box_prob = sigmoid(class_score); + if (box_prob > prob_threshold) + { + float pred_ltrb[4]; + for (int k = 0; k < 4; k++) + { + float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); + pred_ltrb[k] = dis * grid_strides[i].stride; + } + + float angle = feat_ptr[4 * reg_max + cls_num]; + + float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; + float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; + + float cos = std::cos(angle); + float sin = std::sin(angle); + + float x = (pred_ltrb[2] - pred_ltrb[0]) * 0.5f; + float y = (pred_ltrb[3] - pred_ltrb[1]) * 0.5f; + float xc = x * cos - y * sin + pb_cx; + float yc = x * sin + y * cos + pb_cy; + float w = pred_ltrb[2] + pred_ltrb[0]; + float h = pred_ltrb[3] + pred_ltrb[1]; + + Object obj; + obj.rect.x = xc; //center x + obj.rect.y = yc; //center y + obj.rect.width = w; + obj.rect.height = h; + obj.label = class_index; + obj.prob = box_prob; + obj.angle = angle; + + objects.push_back(obj); + } + feat_ptr += (cls_num + 4 * reg_max + 1); + } + } + static void draw_objects_obb(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name, int thickness = 1) + { + static const std::vector COCO_COLORS = { + {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; + cv::Mat image = bgr.clone(); + + for (size_t i = 0; i < objects.size(); i++){ + const Object& obj = objects[i]; + + fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, + obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); + { + float xc = obj.rect.x; + float yc = obj.rect.y; + float w = obj.rect.width; + float h = obj.rect.height; + float ag = obj.angle; + float wx = w / 2 * std::cos(ag); + float wy = w / 2 * std::sin(ag); + float hx = -h / 2 * std::sin(ag); + float hy = h / 2 * std::cos(ag); + cv::Point2f p1{ xc - wx - hx, yc - wy - hy }; + cv::Point2f p2{ xc + wx - hx, yc + wy - hy }; + cv::Point2f p3{ xc + wx + hx, yc + wy + hy }; + cv::Point2f p4{ xc - wx + hx, yc - wy + hy }; + std::vector points = { p1, p2, p3, p4, p1 }; + std::vector> contours= { points }; + cv::polylines(image, contours, true, COCO_COLORS[obj.label], thickness, cv::LINE_AA); + } + } + + cv::imwrite(std::string(output_name) + ".jpg", image); + } + } // namespace obb + + static void generate_proposals_yolov10(int stride, const float* feat, float prob_threshold, std::vector& objects, + int letterbox_cols, int letterbox_rows, int cls_num = 80) + { + int feat_w = letterbox_cols / stride; + int feat_h = letterbox_rows / stride; + auto feat_ptr = feat; + float dis[16]; + for (int h = 0; h <= feat_h - 1; h++) + { + for (int w = 0; w <= feat_w - 1; w++) + { + int c_index = 0; + float c_score = 0; + + for (int c = 0; c < cls_num; c++) + { + float score = feat_ptr[c]; + if (score > c_score) + { + c_index = c; + c_score = score; + } + } + + if(c_score >= prob_threshold) + { + float x0 = w + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 0 * 16,dis, 16); + float y0 = h + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 1 * 16,dis, 16); + float x1 = w + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 2 * 16,dis, 16); + float y1 = h + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 3 * 16,dis, 16); + + x0 *= stride; + y0 *= stride; + x1 *= stride; + y1 *= stride; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0;; + obj.rect.height = y1 - y0; + obj.label = c_index; + obj.prob = c_score; + objects.push_back(obj); + } + feat_ptr += (cls_num + 4 * 16); + } + } + } + +} // namespace detection diff --git a/projects/llm_framework/main_cv/src/runner/base/pose.hpp b/projects/llm_framework/main_cv/src/runner/base/pose.hpp new file mode 100644 index 00000000..b06badfc --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/pose.hpp @@ -0,0 +1,432 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: hebing + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace pose +{ + typedef struct + { + float x; + float y; + float score; + } ai_point_t; + + struct skeleton + { + int connection[2]; + int left_right_neutral; + }; + + std::vector pairs = {{15, 13, 0}, + {13, 11, 0}, + {16, 14, 0}, + {14, 12, 0}, + {11, 12, 0}, + {5, 11, 0}, + {6, 12, 0}, + {5, 6, 0}, + {5, 7, 0}, + {6, 8, 0}, + {7, 9, 0}, + {8, 10, 0}, + {1, 2, 0}, + {0, 1, 0}, + {0, 2, 0}, + {1, 3, 0}, + {2, 4, 0}, + {0, 5, 0}, + {0, 6, 0}}; + std::vector hand_pairs = {{0, 1, 0}, + {1, 2, 0}, + {2, 3, 0}, + {3, 4, 0}, + {0, 5, 1}, + {5, 6, 1}, + {6, 7, 1}, + {7, 8, 1}, + {0, 9, 2}, + {9, 10, 2}, + {10, 11, 2}, + {11, 12, 2}, + {0, 13, 3}, + {13, 14, 3}, + {14, 15, 3}, + {15, 16, 3}, + {0, 17, 4}, + {17, 18, 4}, + {18, 19, 4}, + {19, 20, 4}}; + std::vector animal_pairs = {{19, 15, 0}, + {18, 14, 0}, + {17, 13, 0}, + {16, 12, 0}, + {15, 11, 0}, + {14, 10, 0}, + {13, 9, 0}, + {12, 8, 0}, + {11, 6, 0}, + {10, 6, 0}, + {9, 7, 0}, + {8, 7, 0}, + {6, 7, 0}, + {7, 5, 0}, + {5, 4, 0}, + {0, 2, 0}, + {1, 3, 0}, + {0, 1, 0}, + {0, 4, 0}, + {1, 4, 0}}; + + typedef struct ai_body_parts_s + { + std::vector keypoints; + int32_t img_width = 0; + int32_t img_heigh = 0; + uint64_t timestamp = 0; + } ai_body_parts_s; + + typedef struct ai_hand_parts_s + { + std::vector keypoints; + int32_t hand_side = 0; //0-left hand,1-right hand + int32_t img_width = 0; + int32_t img_heigh = 0; + uint64_t timestamp = 0; + } ai_hand_parts_s; + + typedef struct ai_animal_parts_s + { + std::vector keypoints; + int32_t img_width = 0; + int32_t img_heigh = 0; + uint64_t timestamp = 0; + } ai_animal_parts_s; + + static inline void find_max_2d(float* buf, int width, int height, int* max_idx_width, int* max_idx_height, float* max_value, int c) + { + float* ptr = buf; + *max_value = -10.f; + *max_idx_width = 0; + *max_idx_height = 0; + for (int h = 0; h < height; h++) + { + for (int w = 0; w < width; w++) + { + float score = ptr[c * height * width + h * width + w]; + if (score > *max_value) + { + *max_value = score; + *max_idx_height = h; + *max_idx_width = w; + } + } + } + } + + static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h) + { + for (int i = 0; i < joints_num; i++) + { + int x = (int)(pose.keypoints[i].x * img.cols); + int y = (int)(pose.keypoints[i].y * img.rows); + + x = std::max(std::min(x, (img.cols - 1)), 0); + y = std::max(std::min(y, (img.rows - 1)), 0); + + cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); + } + + cv::Scalar color; + cv::Point pt1; + cv::Point pt2; + for (auto& element : pairs) + { + switch (element.left_right_neutral) + { + case 0: + color = cv::Scalar(255, 0, 0); + break; + case 1: + color = cv::Scalar(0, 0, 255); + break; + default: + color = cv::Scalar(0, 255, 0); + } + + int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); + int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); + int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); + int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); + + x1 = std::max(std::min(x1, (img.cols - 1)), 0); + y1 = std::max(std::min(y1, (img.rows - 1)), 0); + x2 = std::max(std::min(x2, (img.cols - 1)), 0); + y2 = std::max(std::min(y2, (img.rows - 1)), 0); + + pt1 = cv::Point(x1, y1); + pt2 = cv::Point(x2, y2); + cv::line(img, pt1, pt2, color, 2); + } + } + + static inline void draw_animal_result(cv::Mat img, ai_animal_parts_s& pose, int joints_num, int model_w, int model_h) + { + for (int i = 0; i < joints_num; i++) + { + int x = (int)(pose.keypoints[i].x * img.cols); + int y = (int)(pose.keypoints[i].y * img.rows); + + x = std::max(std::min(x, (img.cols - 1)), 0); + y = std::max(std::min(y, (img.rows - 1)), 0); + + cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); + } + + cv::Scalar color; + cv::Point pt1; + cv::Point pt2; + for (auto& element : animal_pairs) + { + switch (element.left_right_neutral) + { + case 0: + color = cv::Scalar(255, 0, 0); + break; + case 1: + color = cv::Scalar(0, 0, 255); + break; + default: + color = cv::Scalar(0, 255, 0); + } + + int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); + int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); + int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); + int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); + + x1 = std::max(std::min(x1, (img.cols - 1)), 0); + y1 = std::max(std::min(y1, (img.rows - 1)), 0); + x2 = std::max(std::min(x2, (img.cols - 1)), 0); + y2 = std::max(std::min(y2, (img.rows - 1)), 0); + + pt1 = cv::Point(x1, y1); + pt2 = cv::Point(x2, y2); + cv::line(img, pt1, pt2, color, 2); + } + } + + static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h, const detection::Object& obj) + { + for (int i = 0; i < joints_num; i++) + { + int x = (int)(pose.keypoints[i].x); + int y = (int)(pose.keypoints[i].y); + x = std::max(std::min(x, (img.cols - 1)), 0); + y = std::max(std::min(y, (img.rows - 1)), 0); + + cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); + } + + cv::Scalar color; + cv::Point pt1; + cv::Point pt2; + for (auto& element : pairs) + { + switch (element.left_right_neutral) + { + case 0: + color = cv::Scalar(255, 0, 0); + break; + case 1: + color = cv::Scalar(0, 0, 255); + break; + default: + color = cv::Scalar(0, 255, 0); + } + + int x1 = (int)(pose.keypoints[element.connection[0]].x); + int y1 = (int)(pose.keypoints[element.connection[0]].y); + int x2 = (int)(pose.keypoints[element.connection[1]].x); + int y2 = (int)(pose.keypoints[element.connection[1]].y); + + x1 = std::max(std::min(x1, (img.cols - 1)), 0); + y1 = std::max(std::min(y1, (img.rows - 1)), 0); + x2 = std::max(std::min(x2, (img.cols - 1)), 0); + y2 = std::max(std::min(y2, (img.rows - 1)), 0); + + pt1 = cv::Point(x1, y1); + pt2 = cv::Point(x2, y2); + cv::line(img, pt1, pt2, color, 2); + } + // 画框 + cv::rectangle(img, obj.rect, cv::Scalar(255, 0, 0)); + char text[256]; + sprintf(text, "%s %.1f%%", "person", obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > img.cols) + x = img.cols - label_size.width; + + cv::rectangle(img, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(img, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, + cv::Scalar(0, 0, 0)); + cv::imwrite("./pose_ppl_out.png", img); + } + + static inline void draw_result_hand(cv::Mat img, ai_hand_parts_s& pose, int joints_num) + { + for (int i = 0; i < joints_num; i++) + { + int x = (int)(pose.keypoints[i].x * img.cols); + int y = (int)(pose.keypoints[i].y * img.rows); + + x = std::max(std::min(x, (img.cols - 1)), 0); + y = std::max(std::min(y, (img.rows - 1)), 0); + + cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 0, 255), cv::FILLED); + } + + cv::Scalar color; + cv::Point pt1; + cv::Point pt2; + for (auto& element : hand_pairs) + { + switch (element.left_right_neutral) + { + case 0: + color = cv::Scalar(10, 215, 255); + break; + case 1: + color = cv::Scalar(255, 115, 55); + break; + case 2: + color = cv::Scalar(5, 255, 55); + break; + case 3: + color = cv::Scalar(25, 15, 255); + break; + default: + color = cv::Scalar(225, 15, 55); + } + + int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); + int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); + int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); + int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); + + x1 = std::max(std::min(x1, (img.cols - 1)), 0); + y1 = std::max(std::min(y1, (img.rows - 1)), 0); + x2 = std::max(std::min(x2, (img.cols - 1)), 0); + y2 = std::max(std::min(y2, (img.rows - 1)), 0); + + pt1 = cv::Point(x1, y1); + pt2 = cv::Point(x2, y2); + cv::line(img, pt1, pt2, color, 2); + } + + cv::imwrite("./hand_pose_out.png", img); + } + + static inline void post_process(float* data, ai_body_parts_s& pose, int joint_num, int img_h, int img_w) + { + int heatmap_width = img_w / 4; + int heatmap_height = img_h / 4; + int max_idx_width, max_idx_height; + float max_score; + + ai_point_t kp; + for (int c = 0; c < joint_num; ++c) + { + find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); + kp.x = (float)max_idx_width / (float)heatmap_width; + kp.y = (float)max_idx_height / (float)heatmap_height; + kp.score = max_score; + pose.keypoints.push_back(kp); + + // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " + // << pose.keypoints[c].score << std::endl; + } + } + + static inline void animal_post_process(float* data, ai_animal_parts_s& pose, int joint_num, int img_h, int img_w) + { + int heatmap_width = img_w / 4; + int heatmap_height = img_h / 4; + int max_idx_width, max_idx_height; + float max_score; + + ai_point_t kp; + for (int c = 0; c < joint_num; ++c) + { + find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); + kp.x = (float)max_idx_width / (float)heatmap_width; + kp.y = (float)max_idx_height / (float)heatmap_height; + kp.score = max_score; + pose.keypoints.push_back(kp); + + // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " + // << pose.keypoints[c].score << std::endl; + } + } + + static inline void ppl_pose_post_process(float* data1, float* data2, ai_body_parts_s& pose, int joint_num, int img_h, int img_w, int offset_top, int offset_left, int offset_x, int offset_y, float ratio) + { + ai_point_t kp; + + for (int c = 0; c < joint_num; ++c) + { + kp.x = (data1[c] / 2 - offset_left) / ratio + offset_x; + kp.y = (data2[c] / 2 - offset_top) / ratio + offset_y; + std::cout << "x1: " << kp.x << ", y1: " << kp.y << std::endl; + pose.keypoints.push_back(kp); + } + } + + static inline void post_process_hand(float* point_data, float* score_data, ai_hand_parts_s& pose, int joint_num, int img_h, int img_w) + { + ai_point_t kp; + for (int c = 0; c < joint_num; ++c) + { + kp.x = (float)point_data[c * 3] / img_w; + kp.y = (float)point_data[c * 3 + 1] / img_h; + pose.keypoints.push_back(kp); + } + if (score_data[0] > 0.5) + pose.hand_side = 1; + } + +} // namespace pose \ No newline at end of file diff --git a/projects/llm_framework/main_cv/src/runner/base/score.hpp b/projects/llm_framework/main_cv/src/runner/base/score.hpp new file mode 100644 index 00000000..080815d6 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/score.hpp @@ -0,0 +1,33 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: ls.wang + */ + +#pragma once + +#include + + +namespace classification +{ + typedef struct + { + uint32_t id; + float score; + } score; +} diff --git a/projects/llm_framework/main_cv/src/runner/base/topk.hpp b/projects/llm_framework/main_cv/src/runner/base/topk.hpp new file mode 100644 index 00000000..50d43144 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/topk.hpp @@ -0,0 +1,52 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: ls.wang + */ + +#pragma once + +#include +#include +#include + +#include "base/score.hpp" + + +namespace classification +{ + void sort_score(std::vector& array, bool reverse = false) + { + auto compare_func = [](const score& a, const score& b) -> bool + { + return a.score > b.score; + }; + + std::sort(array.begin(), array.end(), compare_func); + + if (reverse) std::reverse(array.begin(), array.end()); + } + + + void print_score(const std::vector& array, const size_t& n) + { + for (size_t i = 0; i < n; i++) + { + fprintf(stdout, "%.4f, %d\n", array[i].score, array[i].id); + } + } +} diff --git a/projects/llm_framework/main_cv/src/runner/base/transform.hpp b/projects/llm_framework/main_cv/src/runner/base/transform.hpp new file mode 100644 index 00000000..edee13a7 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/transform.hpp @@ -0,0 +1,47 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: hebing + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +namespace transform +{ + static void nhwc2nchw(const float* input, float* output, int h, int w, int c) + { + int output_index = 0; + for (int i = 0; i < c; ++i) + { + for (int j = 0; j < h; ++j) + { + for (int k = 0; k < w; ++k) + { + int input_index = j * w * c + k * c + i; + output[output_index++] = input[input_index]; + } + } + } + } + +} // namespace transform diff --git a/projects/llm_framework/main_cv/src/runner/base/yolo.hpp b/projects/llm_framework/main_cv/src/runner/base/yolo.hpp new file mode 100644 index 00000000..023cd545 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/base/yolo.hpp @@ -0,0 +1,578 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: hebing + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace yolo +{ + enum + { + YOLOV3 = 0, + YOLOV3_TINY = 1, + YOLOV4 = 2, + YOLOV4_TINY = 3, + YOLO_FASTEST = 4, + YOLO_FASTEST_XL = 5, + YOLO_FASTEST_BODY = 6, + YOLOV4_TINY_3L = 7 + }; + + struct BBoxRect + { + float score; + float xmin; + float ymin; + float xmax; + float ymax; + float area; + int label; + }; + + static inline float sigmoid(float x) + { + return (float)(1.f / (1.f + std::exp(-x))); + } + static inline float intersection_area(const BBoxRect& a, const BBoxRect& b) + { + if (a.xmin > b.xmax || a.xmax < b.xmin || a.ymin > b.ymax || a.ymax < b.ymin) + { + // no intersection + return 0.f; + } + + float inter_width = std::min(a.xmax, b.xmax) - std::max(a.xmin, b.xmin); + float inter_height = std::min(a.ymax, b.ymax) - std::max(a.ymin, b.ymin); + + return inter_width * inter_height; + } + static void qsort_descent_inplace(std::vector& datas, int left, int right) + { + int i = left; + int j = right; + float p = datas[(left + right) / 2].score; + + while (i <= j) + { + while (datas[i].score > p) + i++; + + while (datas[j].score < p) + j--; + + if (i <= j) + { + // swap + std::swap(datas[i], datas[j]); + + i++; + j--; + } + } + + if (left < j) + qsort_descent_inplace(datas, left, j); + + if (i < right) + qsort_descent_inplace(datas, i, right); + } + + static void qsort_descent_inplace(std::vector& datas) + { + if (datas.empty()) + return; + + qsort_descent_inplace(datas, 0, (int)(datas.size() - 1)); + } + + static void nms_sorted_bboxes(std::vector& bboxes, std::vector& picked, float nms_threshold) + { + picked.clear(); + + const size_t n = bboxes.size(); + + for (size_t i = 0; i < n; i++) + { + const BBoxRect& a = bboxes[i]; + + int keep = 1; + for (unsigned int j : picked) + { + const BBoxRect& b = bboxes[j]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = a.area + b.area - inter_area; + // float IoU = inter_area / union_area + if (inter_area > nms_threshold * union_area) + { + keep = 0; + break; + } + } + + if (keep) + picked.push_back(i); + } + } + + struct TMat + { + operator const float*() const + { + return (const float*)data; + } + + float* row(int row) const + { + return (float*)data + w * row; + } + + TMat channel_range(int start, int chn_num) const + { + TMat mat = {0}; + + mat.batch = 1; + mat.c = chn_num; + mat.h = h; + mat.w = w; + mat.data = (float*)data + start * h * w; + + return mat; + } + + TMat channel(int channel) const + { + return channel_range(channel, 1); + } + + int batch, c, h, w; + void* data; + }; + + class YoloDetectionOutput + { + public: + int init(int version, float nms_threshold = 0.45f, float confidence_threshold = 0.48f, int class_num = 80); + int forward(const std::vector& bottom_blobs, std::vector& top_blobs); + int forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs); + + private: + int m_num_box; + int m_num_class; + int m_anchors_scale[32]; + float m_biases[32]; + int m_mask[32]; + float m_confidence_threshold; + float m_confidence_threshold_unsigmoid; + float m_nms_threshold; + }; + + int YoloDetectionOutput::init(int version, float nms_threshold, float confidence_threshold, int class_num) + { + memset(this, 0, sizeof(*this)); + m_num_box = 3; + m_num_class = class_num; + fprintf(stderr, "YoloDetectionOutput init param[%d]\n", version); + + if (version == YOLOV3) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + m_anchors_scale[2] = 8; + + float bias[] = {10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326}; + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 6; + m_mask[1] = 7; + m_mask[2] = 8; + + m_mask[3] = 3; + m_mask[4] = 4; + m_mask[5] = 5; + + m_mask[6] = 0; + m_mask[7] = 1; + m_mask[8] = 2; + } + else if (version == YOLOV3_TINY || version == YOLOV4_TINY) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + + float bias[] = {10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319}; + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 3; + m_mask[1] = 4; + m_mask[2] = 5; + + m_mask[3] = 0; + m_mask[4] = 1; + m_mask[5] = 2; + } + else if (version == YOLOV4) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + m_anchors_scale[2] = 8; + + float bias[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 6; + m_mask[1] = 7; + m_mask[2] = 8; + + m_mask[3] = 3; + m_mask[4] = 4; + m_mask[5] = 5; + + m_mask[6] = 0; + m_mask[7] = 1; + m_mask[8] = 2; + } + else if (version == YOLO_FASTEST || version == YOLO_FASTEST_XL) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + + float bias[] = {12, 18, 37, 49, 52, 132, 115, 73, 119, 199, 242, 238}; + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 3; + m_mask[1] = 4; + m_mask[2] = 5; + + m_mask[3] = 0; + m_mask[4] = 1; + m_mask[5] = 2; + } + else if (version == YOLO_FASTEST_BODY) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + + float bias[] = {7, 17, 20, 50, 45, 99, 64, 187, 123, 211, 227, 264}; + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 3; + m_mask[1] = 4; + m_mask[2] = 5; + + m_mask[3] = 0; + m_mask[4] = 1; + m_mask[5] = 2; + } + else if (version == YOLOV4_TINY_3L) + { + m_anchors_scale[0] = 32; + m_anchors_scale[1] = 16; + m_anchors_scale[2] = 8; + + //float bias_official[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; + float bias[] = {10, 14, 23, 27, 37, 58, 36, 75, 76, 55, 72, 146, 81, 82, 135, 169, 344, 319}; + + memcpy(m_biases, bias, sizeof(bias)); + + m_mask[0] = 6; + m_mask[1] = 7; + m_mask[2] = 8; + + m_mask[3] = 3; + m_mask[4] = 4; + m_mask[5] = 5; + + m_mask[6] = 0; + m_mask[7] = 1; + m_mask[8] = 2; + } + + m_confidence_threshold = confidence_threshold; + m_nms_threshold = nms_threshold; + m_confidence_threshold_unsigmoid = -1.0f * (float)std::log((1.0f / m_confidence_threshold) - 1.0f); + + return 0; + } + + int YoloDetectionOutput::forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs) + { + // gather all box + std::vector all_bbox_rects; + for (size_t b = 0; b < bottom_blobs.size(); b++) + { + const TMat& bottom_top_blobs = bottom_blobs[b]; + + int w = bottom_top_blobs.w; + int h = bottom_top_blobs.h; + size_t mask_offset = b * m_num_box; + int net_w = (int)(m_anchors_scale[b] * w); + int net_h = (int)(m_anchors_scale[b] * h); + + auto feature_ptr = (float*)bottom_top_blobs.data; + for (int i = 0; i < h; i++) + { + for (int j = 0; j < w; j++) + { + for (int box = 0; box < m_num_box; ++box) + { + if (feature_ptr[4] < m_confidence_threshold_unsigmoid) + { + feature_ptr += (m_num_class + 5); + continue; + } + + int class_index = 0; + float class_score = -FLT_MAX; + for (int k = 5; k < m_num_class + 5; ++k) + { + if (class_score < feature_ptr[k]) + { + class_score = feature_ptr[k]; + class_index = k - 5; + } + } + + //sigmoid(box_score) * sigmoid(class_score) + float confidence_1 = 1.0f / ((1.f + std::exp(-feature_ptr[4])) * (1.f + std::exp(-class_score))); + if (confidence_1 >= m_confidence_threshold) + { + int biases_index = (int)(m_mask[box + mask_offset]); + const float bias_w = m_biases[biases_index * 2]; + const float bias_h = m_biases[biases_index * 2 + 1]; + + // region box + // fprintf(stderr, "%f %f %d \n", class_score, feature_ptr[4], class_index); + float bbox_cx = ((float)j + sigmoid(feature_ptr[0])) / (float)w; + float bbox_cy = ((float)i + sigmoid(feature_ptr[1])) / (float)h; + auto bbox_w = (float)(std::exp(feature_ptr[2]) * bias_w / (float)net_w); + auto bbox_h = (float)(std::exp(feature_ptr[3]) * bias_h / (float)net_h); + + float bbox_xmin = bbox_cx - bbox_w * 0.5f; + float bbox_ymin = bbox_cy - bbox_h * 0.5f; + float bbox_xmax = bbox_cx + bbox_w * 0.5f; + float bbox_ymax = bbox_cy + bbox_h * 0.5f; + + float area = bbox_w * bbox_h; + + BBoxRect c = {confidence_1, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; + all_bbox_rects.push_back(c); + } + + feature_ptr += (m_num_class + 5); + } + } + } + } + + // global sort inplace + qsort_descent_inplace(all_bbox_rects); + + // apply nms + std::vector picked; + nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); + + // select + std::vector bbox_rects; + + for (unsigned int z : picked) + { + bbox_rects.push_back(all_bbox_rects[z]); + } + + // fill result + int num_detected = (int)(bbox_rects.size()); + if (num_detected == 0) + { + top_blobs[0].h = 0; + return 0; + } + + TMat& top_blob = top_blobs[0]; + + for (int i = 0; i < num_detected; i++) + { + const BBoxRect& r = bbox_rects[i]; + float score = r.score; + float* outptr = top_blob.row(i); + + outptr[0] = (float)r.label; // +1 for prepend background class + outptr[1] = score; + outptr[2] = r.xmin; + outptr[3] = r.ymin; + outptr[4] = r.xmax; + outptr[5] = r.ymax; + } + top_blob.h = num_detected; + return 0; + } + + int YoloDetectionOutput::forward(const std::vector& bottom_blobs, std::vector& top_blobs) + { + // gather all box + std::vector all_bbox_rects; + + for (size_t b = 0; b < bottom_blobs.size(); b++) + { + std::vector > all_box_bbox_rects; + all_box_bbox_rects.resize(m_num_box); + const TMat& bottom_top_blobs = bottom_blobs[b]; + + int w = bottom_top_blobs.w; + int h = bottom_top_blobs.h; + int channels = bottom_top_blobs.c; + //printf("%d %d %d\n", w, h, channels); + const int channels_per_box = channels / m_num_box; + + // anchor coord + box score + num_class + if (channels_per_box != 4 + 1 + m_num_class) + return -1; + size_t mask_offset = b * m_num_box; + int net_w = (int)(m_anchors_scale[b] * w); + int net_h = (int)(m_anchors_scale[b] * h); + //printf("%d %d\n", net_w, net_h); + + //printf("%d %d %d\n", w, h, channels); + for (int pp = 0; pp < m_num_box; pp++) + { + int p = pp * channels_per_box; + int biases_index = (int)(m_mask[pp + mask_offset]); + //printf("%d\n", biases_index); + const float bias_w = m_biases[biases_index * 2]; + const float bias_h = m_biases[biases_index * 2 + 1]; + //printf("%f %f\n", bias_w, bias_h); + const float* xptr = bottom_top_blobs.channel(p); + const float* yptr = bottom_top_blobs.channel(p + 1); + const float* wptr = bottom_top_blobs.channel(p + 2); + const float* hptr = bottom_top_blobs.channel(p + 3); + + const float* box_score_ptr = bottom_top_blobs.channel(p + 4); + + // softmax class scores + TMat scores = bottom_top_blobs.channel_range(p + 5, m_num_class); + //softmax->forward_inplace(scores, opt); + + for (int i = 0; i < h; i++) + { + for (int j = 0; j < w; j++) + { + // find class index with max class score + int class_index = 0; + float class_score = -FLT_MAX; + for (int q = 0; q < m_num_class; q++) + { + float score = scores.channel(q).row(i)[j]; + if (score > class_score) + { + class_index = q; + class_score = score; + } + } + + //sigmoid(box_score) * sigmoid(class_score) + float confidence = (float)1.f / ((1.f + std::exp(-box_score_ptr[0]) * (1.f + std::exp(-class_score)))); + if (confidence >= m_confidence_threshold) + { + // fprintf(stderr, "%f %d \n", class_score, class_index); + // region box + float bbox_cx = ((float)j + sigmoid(xptr[0])) / (float)w; + float bbox_cy = ((float)i + sigmoid(yptr[0])) / (float)h; + auto bbox_w = (float)(std::exp(wptr[0]) * bias_w / (float)net_w); + auto bbox_h = (float)(std::exp(hptr[0]) * bias_h / (float)net_h); + + float bbox_xmin = bbox_cx - bbox_w * 0.5f; + float bbox_ymin = bbox_cy - bbox_h * 0.5f; + float bbox_xmax = bbox_cx + bbox_w * 0.5f; + float bbox_ymax = bbox_cy + bbox_h * 0.5f; + + float area = bbox_w * bbox_h; + + BBoxRect c = {confidence, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; + all_box_bbox_rects[pp].push_back(c); + } + + xptr++; + yptr++; + wptr++; + hptr++; + + box_score_ptr++; + } + } + } + + for (int i = 0; i < m_num_box; i++) + { + const std::vector& box_bbox_rects = all_box_bbox_rects[i]; + + all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end()); + } + } + + // global sort inplace + qsort_descent_inplace(all_bbox_rects); + + // apply nms + std::vector picked; + nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); + + // select + std::vector bbox_rects; + + for (unsigned int z : picked) + { + bbox_rects.push_back(all_bbox_rects[z]); + } + + // fill result + int num_detected = (int)(bbox_rects.size()); + if (num_detected == 0) + { + top_blobs[0].h = 0; + return 0; + } + + TMat& top_blob = top_blobs[0]; + + for (int i = 0; i < num_detected; i++) + { + const BBoxRect& r = bbox_rects[i]; + float score = r.score; + float* outptr = top_blob.row(i); + + outptr[0] = (float)r.label; // +1 for prepend background class + outptr[1] = score; + outptr[2] = r.xmin; + outptr[3] = r.ymin; + outptr[4] = r.xmax; + outptr[5] = r.ymax; + } + top_blob.h = num_detected; + + return 0; + } + +} // namespace yolo diff --git a/projects/llm_framework/main_cv/src/runner/utils/checker.h b/projects/llm_framework/main_cv/src/runner/utils/checker.h new file mode 100644 index 00000000..9b7e288e --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/utils/checker.h @@ -0,0 +1,32 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ + +#ifndef AX_CHECKER_H +#define AX_CHECKER_H + +#include "utils/logger.h" + +#define CHECK_PTR(p) \ + do { \ + if (!p) { \ + ALOGE("%s nil pointer\n", #p); \ + return -1; \ + } \ + } while (0) + +#define CHECK_INITED(p) \ + do { \ + if (!p->HasInit()) { \ + ALOGE("%s has not init\n", #p); \ + return -1; \ + } \ + } while (0) + +#endif //AX_CHECKER_H diff --git a/projects/llm_framework/main_cv/src/runner/utils/io.hpp b/projects/llm_framework/main_cv/src/runner/utils/io.hpp new file mode 100644 index 00000000..64db991a --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/utils/io.hpp @@ -0,0 +1,601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/checker.h" +#include "ax_sys_api.h" +#include "ax_engine_type.h" + + +#define IO_CMM_ALIGN_SIZE 128 + +namespace utils { + typedef enum { + IO_BUFFER_STRATEGY_DEFAULT, + IO_BUFFER_STRATEGY_CACHED + } IO_BUFFER_STRATEGY_T; + + static inline AX_S32 query_model_input_size(const AX_ENGINE_IO_INFO_T* io_info, std::array &input_size, + AX_IMG_FORMAT_E &eDtype) { + int height = 0; + int width = 0; + int size = 0; + int channel = 0; + int data_type_size = 0; + auto& input = io_info->pInputs[0]; + + switch (input.eLayout) { + case AX_ENGINE_TENSOR_LAYOUT_NHWC: + height = input.pShape[1]; + width = input.pShape[2]; + channel = input.pShape[3]; + size = input.nSize; + break; + case AX_ENGINE_TENSOR_LAYOUT_NCHW: + channel = input.pShape[1]; + height = input.pShape[2]; + width = input.pShape[3]; + size = input.nSize; + break; + default: // NHWC + height = input.pShape[1]; + width = input.pShape[2]; + channel = input.pShape[3]; + size = input.nSize; + break; + } + + switch (input.eDataType) { + case AX_ENGINE_DT_UINT8: + case AX_ENGINE_DT_SINT8: + data_type_size = 1; + break; + case AX_ENGINE_DT_UINT16: + case AX_ENGINE_DT_SINT16: + data_type_size = 2; + break; + case AX_ENGINE_DT_FLOAT32: + data_type_size = 4; + break; + case AX_ENGINE_DT_SINT32: + case AX_ENGINE_DT_UINT32: + data_type_size = 4; + break; + case AX_ENGINE_DT_FLOAT64: + data_type_size = 8; + break; + default: + data_type_size = 1; + break; + } + + if (channel == 0 || height == 0 || width == 0 || size == 0 || data_type_size == 0) { + return -1; + } + + if (input.pExtraMeta) { + switch (input.pExtraMeta->eColorSpace) { + case AX_ENGINE_CS_BGR: + input_size[0] = height; + input_size[1] = width; + eDtype = AX_FORMAT_BGR888; + break; + case AX_ENGINE_CS_RGB: + input_size[0] = height; + input_size[1] = width; + eDtype = AX_FORMAT_RGB888; + break; + case AX_ENGINE_CS_NV12: + input_size[0] = height * 2 / 3; + input_size[1] = width; + eDtype = AX_FORMAT_YUV420_SEMIPLANAR; + break; + case AX_ENGINE_CS_NV21: + input_size[0] = height * 2 / 3; + input_size[1] = width; + eDtype = AX_FORMAT_YUV420_SEMIPLANAR_VU; + break; + default: // AX_ENGINE_CS_NV12 + input_size[0] = height * 2 / 3; + input_size[1] = width; + eDtype = AX_FORMAT_YUV420_SEMIPLANAR; + break; + } + } + else { + input_size[0] = height * 2 / 3; + input_size[1] = width; + eDtype = AX_FORMAT_YUV420_SEMIPLANAR; + } + + ALOGD("eLayout:%d, eDataType:%d, channel:%d, height:%d, width:%d, size:%d, data_type_size:%d", + input.eLayout, input.eDataType, channel, input_size[0], input_size[1], size, data_type_size); + + return 0; + } + + static inline void brief_io_info(std::string strModel, const AX_ENGINE_IO_INFO_T* io_info) { + auto describe_shape_type = [](AX_ENGINE_TENSOR_LAYOUT_T type) -> const char* { + switch (type) { + case AX_ENGINE_TENSOR_LAYOUT_NHWC: + return "NHWC"; + case AX_ENGINE_TENSOR_LAYOUT_NCHW: + return "NCHW"; + default: + return "unknown"; + } + }; + auto describe_data_type = [](AX_ENGINE_DATA_TYPE_T type) -> const char* { + switch (type) { + case AX_ENGINE_DT_UINT8: + return "uint8"; + case AX_ENGINE_DT_UINT16: + return "uint16"; + case AX_ENGINE_DT_FLOAT32: + return "float32"; + case AX_ENGINE_DT_SINT16: + return "sint16"; + case AX_ENGINE_DT_SINT8: + return "sint8"; + case AX_ENGINE_DT_SINT32: + return "sint32"; + case AX_ENGINE_DT_UINT32: + return "uint32"; + case AX_ENGINE_DT_FLOAT64: + return "float64"; + case AX_ENGINE_DT_UINT10_PACKED: + return "uint10_packed"; + case AX_ENGINE_DT_UINT12_PACKED: + return "uint12_packed"; + case AX_ENGINE_DT_UINT14_PACKED: + return "uint14_packed"; + case AX_ENGINE_DT_UINT16_PACKED: + return "uint16_packed"; + default: + return "unknown"; + } + }; + auto describe_memory_type = [](AX_ENGINE_MEMORY_TYPE_T type) -> const char* { + switch (type) { + case AX_ENGINE_MT_PHYSICAL: + return "Physical"; + case AX_ENGINE_MT_VIRTUAL: + return "Virtual"; + default: + return "unknown"; + } + }; + auto describe_color_space = [](AX_ENGINE_COLOR_SPACE_T cs) -> const char* { + switch (cs) { + case AX_ENGINE_CS_FEATUREMAP: + return "FeatureMap"; + case AX_ENGINE_CS_BGR: + return "BGR"; + case AX_ENGINE_CS_RGB: + return "RGB"; + case AX_ENGINE_CS_RGBA: + return "RGBA"; + case AX_ENGINE_CS_GRAY: + return "GRAY"; + case AX_ENGINE_CS_NV12: + return "NV12"; + case AX_ENGINE_CS_NV21: + return "NV21"; + case AX_ENGINE_CS_YUV444: + return "YUV444"; + case AX_ENGINE_CS_RAW8: + return "RAW8"; + case AX_ENGINE_CS_RAW10: + return "RAW10"; + case AX_ENGINE_CS_RAW12: + return "RAW12"; + case AX_ENGINE_CS_RAW14: + return "RAW14"; + case AX_ENGINE_CS_RAW16: + return "RAW16"; + default: + return "unknown"; + } + }; + printf("Model Name: %s\n", strModel.c_str()); + printf("Max Batch Size %d\n", io_info->nMaxBatchSize); + printf("Support Dynamic Batch? %s\n", io_info->bDynamicBatchSize == AX_TRUE ? "Yes" : "No"); + + for (uint32_t i = 0; i < io_info->nInputSize; ++i) { + auto& input = io_info->pInputs[i]; + printf("Input[%d]: %s\n", i, input.pName); + printf(" Shape ["); + for (uint32_t j = 0; j < input.nShapeSize; ++j) { + printf("%d", (int)input.pShape[j]); + if (j + 1 < input.nShapeSize) printf(", "); + } + printf("] %s %s %s %s\n", describe_shape_type(input.eLayout), describe_data_type(input.eDataType), + input.pExtraMeta ? describe_color_space(input.pExtraMeta->eColorSpace) : "", + input.nQuantizationValue > 0 ? ("Q=" + std::to_string(input.nQuantizationValue)).c_str() : ""); + printf(" Memory %s\n", describe_memory_type(input.eMemoryType)); + printf(" Size %u\n", input.nSize); + } + for (uint32_t i = 0; i < io_info->nOutputSize; ++i) { + auto& output = io_info->pOutputs[i]; + printf("Output[%d]: %s\n", i, output.pName); + printf(" Shape ["); + for (uint32_t j = 0; j < output.nShapeSize; ++j) { + printf("%d", (int)output.pShape[j]); + if (j + 1 < output.nShapeSize) printf(", "); + } + printf("] %s %s %s\n", describe_shape_type(output.eLayout), describe_data_type(output.eDataType), + output.nQuantizationValue > 0 ? ("Q=" + std::to_string(output.nQuantizationValue)).c_str() : ""); + printf(" Memory %s\n", describe_memory_type(output.eMemoryType)); + printf(" Size %u\n", output.nSize); + } + } + + static inline AX_S32 alloc_engine_buffer(const std::string& token, const std::string& appendix, size_t index, const AX_ENGINE_IOMETA_T* pMeta, AX_ENGINE_IO_BUFFER_T* pBuf, IO_BUFFER_STRATEGY_T eStrategy = IO_BUFFER_STRATEGY_DEFAULT) { + AX_S32 ret = -1; + if (eStrategy != IO_BUFFER_STRATEGY_DEFAULT && eStrategy != IO_BUFFER_STRATEGY_CACHED) { + fprintf(stderr, "strategy %d not supported\n", (int)eStrategy); + return -1; + } + memset(pBuf, 0, sizeof(AX_ENGINE_IO_BUFFER_T)); + pBuf->nSize = pMeta->nSize; + + const std::string token_name = "skel_" + token + appendix + std::to_string(index); + + if (eStrategy == IO_BUFFER_STRATEGY_CACHED) { + ret = AX_SYS_MemAllocCached((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); + } + else { + ret = AX_SYS_MemAlloc((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); + } + + return ret; + } + + static inline AX_S32 free_engine_buffer(AX_ENGINE_IO_BUFFER_T* pBuf) { + if (pBuf->phyAddr == 0) { + delete[] reinterpret_cast(pBuf->pVirAddr); + } + else { + AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); + } + pBuf->phyAddr = 0; + pBuf->pVirAddr = nullptr; + + return 0; + } + + static inline void free_io_index(AX_ENGINE_IO_BUFFER_T* io_buf, size_t index) { + AX_ENGINE_IO_BUFFER_T* pBuf = io_buf + index; + free_engine_buffer(pBuf); + } + + static inline void free_io(AX_ENGINE_IO_T &io) { + for (size_t j = 0; j < io.nInputSize; ++j) + { + AX_ENGINE_IO_BUFFER_T *pBuf = io.pInputs + j; + AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); + } + for (size_t j = 0; j < io.nOutputSize; ++j) + { + AX_ENGINE_IO_BUFFER_T *pBuf = io.pOutputs + j; + AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); + } + delete[] io.pInputs; + delete[] io.pOutputs; + } + + static inline void free_io(AX_ENGINE_IO_T &io, std::vector> &vecOutputBuffer) { + if (io.pInputs) { + delete[] io.pInputs; + io.pInputs = nullptr; + } + + if (io.pOutputs) { + for (size_t index = 0; index < vecOutputBuffer.size(); ++index) { + AX_ENGINE_IO_BUFFER_T *pOutputs = &vecOutputBuffer[index][0]; + for (size_t j = 0; j < io.nOutputSize; ++j) { + free_io_index(pOutputs, j); + } + } + + delete[] io.pOutputs; + io.pOutputs = nullptr; + } + } + + static inline int prepare_io(const std::string& token, const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, IO_BUFFER_STRATEGY_T strategy) { + auto ret = 0; + + memset(&io, 0, sizeof(io)); + + io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; + + if (!io.pInputs) { + goto EXIT; + } + + memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); + io.nInputSize = info->nInputSize; + for (AX_U32 i = 0; i < info->nInputSize; ++i) + { + auto meta = info->pInputs[i]; + auto buffer = &io.pInputs[i]; + ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); + if (ret != 0) + { + free_io_index(io.pInputs, i); + return ret; + } + } + + io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; + + if (!io.pOutputs) { + goto EXIT; + } + + memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); + io.nOutputSize = info->nOutputSize; + + for (size_t i = 0; i < info->nOutputSize; ++i) { + auto meta = info->pOutputs[i]; + auto buffer = &io.pOutputs[i]; + ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); + if (ret != 0) { + goto EXIT; + } + } + + EXIT: + if (ret != 0) { + free_io(io); + return -1; + } + + return 0; + } + + static inline int prepare_io(const std::string& token, + const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, + std::vector &vecOutputBuffer, + const IO_BUFFER_STRATEGY_T &strategy) { + AX_S32 ret = 0; + memset(&io, 0, sizeof(io)); + + std::vector outputBuffer; + + if (1 != info->nInputSize) { + fprintf(stderr, "[ERR]: Only single input was accepted(got %u).\n", info->nInputSize); + return -1; + } + + io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; + + if (!io.pInputs) { + goto EXIT; + } + + memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); + io.nInputSize = info->nInputSize; + + for (AX_U32 i = 0; i < info->nInputSize; ++i) + { + auto meta = info->pInputs[i]; + auto buffer = &io.pInputs[i]; + ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); + if (ret != 0) + { + free_io_index(io.pInputs, i); + return ret; + } + } + + io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; + + if (!io.pOutputs) { + goto EXIT; + } + + for (size_t i = 0; i < info->nOutputSize; ++i) { + auto meta = info->pOutputs[i]; + auto buffer = &io.pOutputs[i]; + ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); + + if (ret != 0) { + goto EXIT; + } + + vecOutputBuffer.push_back(*buffer); + } + + memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); + io.nOutputSize = info->nOutputSize; + + for (size_t i = 0; i < info->nOutputSize; ++i) { + auto buffer = &io.pOutputs[i]; + *buffer = vecOutputBuffer[i]; + } + + EXIT: + if (ret != 0) { + free_io(io); + return -1; + } + + return 0; + } + + static inline AX_S32 push_io_output(const AX_ENGINE_IO_INFO_T* info, + AX_ENGINE_IO_T& io, + std::vector &outputBuffer) { + for (size_t i = 0; i < info->nOutputSize; ++i) { + auto buffer = &io.pOutputs[i]; + *buffer = outputBuffer[i]; + } + + return 0; + } + + static inline AX_S32 push_io_input(void* input, int index, AX_ENGINE_IO_T& io) { + // img ranks_depth ranks_feat ranks_bev, n_points + AX_ENGINE_IO_BUFFER_T* pImg = &io.pInputs[index]; + memcpy(pImg->pVirAddr, input, pImg->nSize); + return 0; + } + + static inline AX_S32 cache_io_flush(const AX_ENGINE_IO_BUFFER_T *io_buf) { + if (io_buf->phyAddr != 0) { + AX_SYS_MflushCache(io_buf->phyAddr, io_buf->pVirAddr, io_buf->nSize); + } + + return 0; + } + + static inline AX_S32 push_io_output(void* output, + int index, + AX_ENGINE_IO_T& io) { + AX_ENGINE_IO_BUFFER_T* pImg = &io.pOutputs[index]; + cache_io_flush(pImg); + memcpy(output, pImg->pVirAddr, pImg->nSize); + return 0; + } + + static inline AX_S32 cpu_copy(AX_U64 nPhyAddrSrc, AX_U64 nPhyAddrDst, AX_U32 nLen) { + if (nPhyAddrSrc != 0 && nPhyAddrDst != 0 && nLen > 0) { + AX_VOID* pSrcVirAddr = AX_SYS_MmapCache(nPhyAddrSrc, nLen); + AX_VOID* pDstVirAddr = AX_SYS_MmapCache(nPhyAddrDst, nLen); + + memcpy((AX_VOID*)pDstVirAddr, (AX_VOID*)pSrcVirAddr, nLen); + + AX_SYS_Munmap(pSrcVirAddr, nLen); + AX_SYS_Munmap(pDstVirAddr, nLen); + + return 0; + } + + return -1; + } + + static inline AX_S32 inc_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { + if (stFrame.u32BlkId[0] > 0) { + AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[0]); + } + if (stFrame.u32BlkId[1] > 0) { + AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[1]); + } + if (stFrame.u32BlkId[2] > 0) { + AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[2]); + } + + return 0; + } + + static inline AX_S32 dec_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { + if (stFrame.u32BlkId[0] > 0) { + AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[0]); + } + if (stFrame.u32BlkId[1] > 0) { + AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[1]); + } + if (stFrame.u32BlkId[2] > 0) { + AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[2]); + } + + return 0; + } + + static inline bool read_file(const char* path, std::vector& data) { + std::fstream fs(path, std::ios::in | std::ios::binary); + + if (!fs.is_open()) { + return false; + } + + fs.seekg(std::ios::end); + auto fs_end = fs.tellg(); + fs.seekg(std::ios::beg); + auto fs_beg = fs.tellg(); + + auto file_size = static_cast(fs_end - fs_beg); + auto vector_size = data.size(); + + data.reserve(vector_size + file_size); + data.insert(data.end(), std::istreambuf_iterator(fs), std::istreambuf_iterator()); + + fs.close(); + + return true; + } + + static inline bool read_file(const char* path, AX_VOID **pModelBufferVirAddr, + AX_U64 &u64ModelBufferPhyAddr, AX_U32 &nModelBufferSize) { + std::fstream fs(path, std::ios::in | std::ios::binary); + + if (!fs.is_open()) { + return false; + } + + fs.seekg(0, std::ios::end); + int file_size = fs.tellg(); + fs.seekg(0, std::ios::beg); + + nModelBufferSize = (AX_U32)file_size; + + AX_SYS_MemAlloc(&u64ModelBufferPhyAddr, pModelBufferVirAddr, nModelBufferSize, 0x100, (AX_S8 *)"SKEL-CV"); + + if (!pModelBufferVirAddr || (u64ModelBufferPhyAddr == 0)) { + return false; + } + + fs.read((AX_CHAR *)*pModelBufferVirAddr, nModelBufferSize); + + fs.close(); + + return true; + } + + static inline void dequant(float** pptrOutput, const AX_ENGINE_IOMETA_T& ptrIoInfo, const AX_ENGINE_IO_BUFFER_T& ioBuf, float zp, float scale) + { + if (ptrIoInfo.eDataType == AX_ENGINE_DT_FLOAT32) + { + *pptrOutput = (float*)ioBuf.pVirAddr; + return; + } + + *pptrOutput = (float*)malloc(ptrIoInfo.nSize * sizeof(float)); + uint8_t *pBuf = (uint8_t*)ioBuf.pVirAddr; + float* pOutput = *pptrOutput; + // float inv_scale = 1.0f / scale; + for (int i = 0; i < ptrIoInfo.nSize; i++) + { + pOutput[i] = ((float)pBuf[i] - zp) * scale; + } + } +} + diff --git a/projects/llm_framework/main_cv/src/runner/utils/logger.h b/projects/llm_framework/main_cv/src/runner/utils/logger.h new file mode 100644 index 00000000..83a2e691 --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/utils/logger.h @@ -0,0 +1,82 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor (Ningbo) Co., Ltd. + * + **************************************************************************************************/ + +#ifndef SKEL_LOGGER_H +#define SKEL_LOGGER_H + +#include "ax_global_type.h" +#include "ax_sys_log.h" + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +//#define SKEL_LOG_TAG "SKEL" +// +//#define ALOGE(fmt, ...) AX_LOG_ERR_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) +//#define ALOGW(fmt, ...) AX_LOG_WARN_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) +//#define ALOGI(fmt, ...) AX_LOG_INFO_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) +//#define ALOGD(fmt, ...) AX_LOG_DBG_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) +//#define ALOGN(fmt, ...) AX_LOG_NOTICE_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) + + +typedef enum { + SKEL_LOG_MIN = -1, + SKEL_LOG_EMERGENCY = 0, + SKEL_LOG_ALERT = 1, + SKEL_LOG_CRITICAL = 2, + SKEL_LOG_ERROR = 3, + SKEL_LOG_WARN = 4, + SKEL_LOG_NOTICE = 5, + SKEL_LOG_INFO = 6, + SKEL_LOG_DEBUG = 7, + SKEL_LOG_MAX +} SKEL_LOG_LEVEL_E; + +static SKEL_LOG_LEVEL_E log_level = SKEL_LOG_DEBUG; + +#if 1 +#define MACRO_BLACK "\033[1;30;30m" +#define MACRO_RED "\033[1;30;31m" +#define MACRO_GREEN "\033[1;30;32m" +#define MACRO_YELLOW "\033[1;30;33m" +#define MACRO_BLUE "\033[1;30;34m" +#define MACRO_PURPLE "\033[1;30;35m" +#define MACRO_WHITE "\033[1;30;37m" +#define MACRO_END "\033[0m" +#else +#define MACRO_BLACK +#define MACRO_RED +#define MACRO_GREEN +#define MACRO_YELLOW +#define MACRO_BLUE +#define MACRO_PURPLE +#define MACRO_WHITE +#define MACRO_END +#endif + +#define ALOGE(fmt, ...) printf(MACRO_RED "[E][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGW(fmt, ...) if (log_level >= SKEL_LOG_WARN) \ + printf(MACRO_YELLOW "[W][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGI(fmt, ...) if (log_level >= SKEL_LOG_INFO) \ + printf(MACRO_GREEN "[I][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGD(fmt, ...) if (log_level >= SKEL_LOG_DEBUG) \ + printf(MACRO_WHITE "[D][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGN(fmt, ...) if (log_level >= SKEL_LOG_NOTICE) \ + printf(MACRO_PURPLE "[N][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) + +#ifdef __cplusplus +} +#endif + +#endif //SKEL_LOGGER_H diff --git a/projects/llm_framework/main_cv/src/runner/utils/timer.hpp b/projects/llm_framework/main_cv/src/runner/utils/timer.hpp new file mode 100644 index 00000000..e753192c --- /dev/null +++ b/projects/llm_framework/main_cv/src/runner/utils/timer.hpp @@ -0,0 +1,61 @@ +/* + * AXERA is pleased to support the open source community by making ax-samples available. + * + * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Author: ls.wang + */ + +#pragma once + +#include + +class timer +{ +private: + std::chrono::system_clock::time_point start_time, end_time; + +public: + timer() + { + start(); + } + + void start() + { + stop(); + this->start_time = this->end_time; + } + + void stop() + { +#ifdef _MSC_VER + this->end_time = std::chrono::system_clock::now(); +#else + this->end_time = std::chrono::high_resolution_clock::now(); +#endif + } + + float cost() + { + if (this->end_time <= this->start_time) + { + this->stop(); + } + + auto ms = std::chrono::duration_cast(this->end_time - this->start_time).count(); + return static_cast(ms) / 1000.f; + } +}; diff --git a/projects/llm_framework/main_cv/yolo11s.json b/projects/llm_framework/main_cv/yolo11s.json new file mode 100644 index 00000000..742976b2 --- /dev/null +++ b/projects/llm_framework/main_cv/yolo11s.json @@ -0,0 +1,23 @@ +{ + "mode":"yolo11s", + "type":"cv", + "capabilities":[ + "object detect" + ], + "input_type":[ + "cv.jpeg.base64" + ], + "output_type":[ + "cv.yolobox" + ], + "mode_param":{ + "cv_model":"yolo11s.axmodel" + }, + "mode_param_bak":{ + "model_config.img_h":640, + "model_config.img_w":640, + "model_config.cls_num":80, + "model_config.pron_threshold":0.45, + "model_config.nms_threshold":0.45 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index f04bbddf..cd487f17 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -202,13 +202,8 @@ class llm_task { std::string out = lLaMa_->Run(prompt_data_); if (out_callback_) out_callback_(out, true); } else { - if (image_data_.size() > 2 && image_data_[0] == 0xFF && image_data_[1] == 0xD8 && - image_data_[image_data_.size() - 2] == 0xFF && image_data_[image_data_.size() - 1] == 0xD9) { - } else { - SLOGE("Invalid JPEG data."); - return; - } cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR); + if (src.empty()) return; image_data_.clear(); lLaMa_->Encode(src, img_embed); lLaMa_->Encode(img_embed, prompt_data_, prompt_complete(msg)); From 4937e3697646a411ca894e2a534d08c373580c8c Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 15:50:14 +0800 Subject: [PATCH 10/44] [add] ax-samples --- ext_components/ax-samples/Kconfig | 6 +++ ext_components/ax-samples/SConstruct | 58 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 ext_components/ax-samples/Kconfig create mode 100644 ext_components/ax-samples/SConstruct diff --git a/ext_components/ax-samples/Kconfig b/ext_components/ax-samples/Kconfig new file mode 100644 index 00000000..d21f5ac8 --- /dev/null +++ b/ext_components/ax-samples/Kconfig @@ -0,0 +1,6 @@ +menuconfig AX_SAMPLES_ENABLED + bool "Enable ax-samples support" + default n + depends on AX_MSP_ENABLED + help + enable axera ax-samples support diff --git a/ext_components/ax-samples/SConstruct b/ext_components/ax-samples/SConstruct new file mode 100644 index 00000000..1408b20b --- /dev/null +++ b/ext_components/ax-samples/SConstruct @@ -0,0 +1,58 @@ +# component2/SConscript +Import("env") +import os +from pathlib import Path + +with open(env["PROJECT_TOOL_S"]) as f: + exec(f.read()) + +_SDK_PATH = os.path.normpath( + os.environ.get("SDK_PATH", str(Path(os.getcwd()) / ".." / "..")) +) +env["GIT_REPO_LISTS"]["ax-samples"] = { + "url": "https://github.com/AXERA-TECH/ax-samples.git", + "commit": "146154ca03929a0e119ca53650e3b05fa40c3e7a", + "path": str(Path(_SDK_PATH) / "github_source" / "ax-samples"), +} + +if "CONFIG_AX_SAMPLES_ENABLED" in os.environ: + check_component("ax-samples") + SRCS = [] + INCLUDE = [] + PRIVATE_INCLUDE = [] + REQUIREMENTS = [] + STATIC_LIB = [] + DYNAMIC_LIB = [] + DEFINITIONS = [] + DEFINITIONS_PRIVATE = [] + LDFLAGS = [] + LINK_SEARCH_PATH = [] + + INCLUDE += [ + os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/base"), + os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/utilities"), + ] + if "CONFIG_AX_620E_MSP_ENABLED" in os.environ: + INCLUDE += [ + os.path.join( + env["GIT_REPO_LISTS"]["ax-samples"]["path"], + "examples/ax620e/middleware", + ) + ] + + env["COMPONENTS"].append( + { + "target": os.path.basename(env["component_dir"]), + "SRCS": SRCS, + "INCLUDE": INCLUDE, + "PRIVATE_INCLUDE": PRIVATE_INCLUDE, + "REQUIREMENTS": REQUIREMENTS, + "STATIC_LIB": STATIC_LIB, + "DYNAMIC_LIB": DYNAMIC_LIB, + "DEFINITIONS": DEFINITIONS, + "DEFINITIONS_PRIVATE": DEFINITIONS_PRIVATE, + "LDFLAGS": LDFLAGS, + "LINK_SEARCH_PATH": LINK_SEARCH_PATH, + "REGISTER": "static", + } + ) From a02304cba2931193b33a4dd70b724aee0ab1c11c Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 15:50:49 +0800 Subject: [PATCH 11/44] [change] cv to yolo --- doc/projects_llm_framework_doc/llm_yolo_zh.md | 144 ++++++++++++++++++ projects/llm_framework/config_defaults.mk | 1 + projects/llm_framework/main_cv/SConstruct | 63 -------- .../{main_cv => main_yolo}/Kconfig | 0 projects/llm_framework/main_yolo/SConstruct | 44 ++++++ .../{main_cv => main_yolo}/src/main.cpp | 72 ++++----- .../src/runner/EngineWrapper.cpp | 0 .../src/runner/EngineWrapper.hpp | 0 .../src/runner/base/common.hpp | 0 .../src/runner/base/detection.h | 0 .../src/runner/base/detection.hpp | 0 .../src/runner/base/pose.hpp | 0 .../src/runner/base/score.hpp | 0 .../src/runner/base/topk.hpp | 0 .../src/runner/base/transform.hpp | 0 .../src/runner/base/yolo.hpp | 0 .../src/runner/utils/checker.h | 0 .../src/runner/utils/io.hpp | 0 .../src/runner/utils/logger.h | 0 .../src/runner/utils/timer.hpp | 0 .../{main_cv => main_yolo}/yolo11s.json | 0 21 files changed, 225 insertions(+), 99 deletions(-) create mode 100644 doc/projects_llm_framework_doc/llm_yolo_zh.md delete mode 100644 projects/llm_framework/main_cv/SConstruct rename projects/llm_framework/{main_cv => main_yolo}/Kconfig (100%) create mode 100644 projects/llm_framework/main_yolo/SConstruct rename projects/llm_framework/{main_cv => main_yolo}/src/main.cpp (88%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/EngineWrapper.cpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/EngineWrapper.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/common.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/detection.h (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/detection.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/pose.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/score.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/topk.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/transform.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/base/yolo.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/utils/checker.h (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/utils/io.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/utils/logger.h (100%) rename projects/llm_framework/{main_cv => main_yolo}/src/runner/utils/timer.hpp (100%) rename projects/llm_framework/{main_cv => main_yolo}/yolo11s.json (100%) diff --git a/doc/projects_llm_framework_doc/llm_yolo_zh.md b/doc/projects_llm_framework_doc/llm_yolo_zh.md new file mode 100644 index 00000000..66c0f6d0 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_yolo_zh.md @@ -0,0 +1,144 @@ +# llm-yolo +yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo 模型。 + +## setup +配置单元工作。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "setup", + "object": "yolo.setup", + "data": { + "model": "yolov5", + "response_format": "yolo.box", + "input": "yolo.jpg.base64", + "enoutput": true + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `yolo`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `yolo.setup`。 +- model：使用的模型为 `yolov5` 模型。 +- response_format：返回结果为 `yolo.box`。 +- input：输入的为 `yolo.jpg.base64`,代表的是从用户输入，数据类型为 jpg, base64编码。 +- enoutput：是否起用用户结果输出。 + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"yolo.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "tts.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"tts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["yolo.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"yolo.tasklist", + "request_id":"2", + "work_id":"yolo" +} +``` + +获取任务运行参数。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "yolo.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["yolo.jpg.base64"], + "model":"yolov5", + "response_format":"yolo.box" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"tts.taskinfo", + "request_id":"2", + "work_id":"yolo.1003" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/projects/llm_framework/config_defaults.mk b/projects/llm_framework/config_defaults.mk index e09f3056..5ea858fb 100644 --- a/projects/llm_framework/config_defaults.mk +++ b/projects/llm_framework/config_defaults.mk @@ -21,3 +21,4 @@ CONFIG_EVENTPP_ENABLED=y CONFIG_STACKFLOW_ENABLED=y CONFIG_UTILITIES_BASE64_ENABLED=y CONFIG_SINGLE_HEADER_LIBS_ENABLED=y +CONFIG_AX_SAMPLES_ENABLED=y diff --git a/projects/llm_framework/main_cv/SConstruct b/projects/llm_framework/main_cv/SConstruct deleted file mode 100644 index b98b2017..00000000 --- a/projects/llm_framework/main_cv/SConstruct +++ /dev/null @@ -1,63 +0,0 @@ -import os - -Import('env') -with open(env['PROJECT_TOOL_S']) as f: - exec(f.read()) - -SRCS = append_srcs_dir(ADir('src')) -INCLUDE = [ADir('include'), ADir('.')] -PRIVATE_INCLUDE = [] -REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow'] -STATIC_LIB = [] -DYNAMIC_LIB = [] -DEFINITIONS = [] -DEFINITIONS_PRIVATE = [] -LDFLAGS = [] -LINK_SEARCH_PATH = [] -STATIC_FILES = [] - -DEFINITIONS += ['-std=c++17'] -LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] -LINK_SEARCH_PATH += [ADir('../static_lib')] -REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] - -INCLUDE += [ADir('src/runner')] -INCLUDE += [ADir('../include/opencv4')] -# LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] -# LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', -# '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', -# '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] - -# STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), -# AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] -static_file = Glob('../static_lib/module-llm/libabsl_*') -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] -STATIC_LIB += static_file * 4 - -env['COMPONENTS'].append({'target':'llm_cv', - 'SRCS':SRCS, - 'INCLUDE':INCLUDE, - 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, - 'REQUIREMENTS':REQUIREMENTS, - 'STATIC_LIB':STATIC_LIB, - 'DYNAMIC_LIB':DYNAMIC_LIB, - 'DEFINITIONS':DEFINITIONS, - 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, - 'LDFLAGS':LDFLAGS, - 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, - 'STATIC_FILES':STATIC_FILES, - 'REGISTER':'project' - }) diff --git a/projects/llm_framework/main_cv/Kconfig b/projects/llm_framework/main_yolo/Kconfig similarity index 100% rename from projects/llm_framework/main_cv/Kconfig rename to projects/llm_framework/main_yolo/Kconfig diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct new file mode 100644 index 00000000..50258ca9 --- /dev/null +++ b/projects/llm_framework/main_yolo/SConstruct @@ -0,0 +1,44 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = append_srcs_dir(ADir('src')) +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'ax-samples'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] + +INCLUDE += [ADir('src/runner')] +INCLUDE += [ADir('../include/opencv4')] + +static_file = Glob('../static_lib/module-llm/libabsl_*') +static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') +STATIC_LIB += static_file * 2 + +env['COMPONENTS'].append({'target':'llm_yolo', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_cv/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp similarity index 88% rename from projects/llm_framework/main_cv/src/main.cpp rename to projects/llm_framework/main_yolo/src/main.cpp index 7df50864..ab9e3b3c 100644 --- a/projects/llm_framework/main_cv/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -18,7 +18,7 @@ using namespace StackFlows; int main_exit_flage = 0; static void __sigint(int iSigNo) { - SLOGW("llm_cv will be exit!"); + SLOGW("llm_yolo will be exit!"); main_exit_flage = 1; } @@ -26,13 +26,13 @@ static std::string base_model_path_; static std::string base_model_config_path_; typedef struct { - std::string cv_model; + std::string yolo_model; int img_h = 640; int img_w = 640; int cls_num = 80; float pron_threshold = 0.45f; float nms_threshold = 0.45; -} cv_config; +} yolo_config; typedef std::function &data, bool finish)> task_callback_t; @@ -45,9 +45,9 @@ typedef std::function &data, bool finish) class llm_task { private: public: - cv_config mode_config_; + yolo_config mode_config_; std::string model_; - std::unique_ptr cv_; + std::unique_ptr yolo_; std::string response_format_; std::vector inputs_; std::vector image_data_; @@ -106,15 +106,15 @@ class llm_task { } std::string base_model = base_model_path_ + model_ + "/"; SLOGI("base_model %s", base_model.c_str()); - CONFIG_AUTO_SET(file_body["mode_param"], cv_model); + CONFIG_AUTO_SET(file_body["mode_param"], yolo_model); CONFIG_AUTO_SET(file_body["mode_param"], img_h); CONFIG_AUTO_SET(file_body["mode_param"], img_w); CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); - mode_config_.cv_model = base_model + mode_config_.cv_model; - cv_ = std::make_unique(); - if (0 != cv_->Init(mode_config_.cv_model.c_str())) { - SLOGE("Init cv_model model failed!\n"); + mode_config_.yolo_model = base_model + mode_config_.yolo_model; + yolo_ = std::make_unique(); + if (0 != yolo_->Init(mode_config_.yolo_model.c_str())) { + SLOGE("Init yolo_model model failed!\n"); return -5; } } catch (...) { @@ -146,13 +146,13 @@ class llm_task { std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); int ret = -1; - cv_->SetInput(image.data(), 0); - if (0 != cv_->RunSync()) { - SLOGE("Run cv model failed!\n"); - throw std::string("cv_ RunSync error"); + yolo_->SetInput(image.data(), 0); + if (0 != yolo_->RunSync()) { + SLOGE("Run yolo model failed!\n"); + throw std::string("yolo_ RunSync error"); } std::vector objects; - cv_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, + yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, mode_config_.pron_threshold, mode_config_.nms_threshold, objects); std::vector yolo_output; for (size_t i = 0; i < objects.size(); i++) { @@ -170,7 +170,7 @@ class llm_task { } if (out_callback_) out_callback_(yolo_output, true); } catch (...) { - SLOGW("CV_->Run have error!"); + SLOGW("yolo_->Run have error!"); } } @@ -215,7 +215,7 @@ class llm_task { int llm_task::ax_init_flage_ = 0; #undef CONFIG_AUTO_SET -class llm_cv : public StackFlow { +class llm_yolo : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; @@ -230,16 +230,16 @@ class llm_cv : public StackFlow { if (base_model_path_.empty() || base_model_config_path_.empty()) { return -1; } else { - SLOGI("llm_cv::_load_config success"); + SLOGI("llm_yolo::_load_config success"); return 0; } } public: - llm_cv() : StackFlow("cv") + llm_yolo() : StackFlow("yolo") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_cv::_load_config, this)); + repeat_event(1000, std::bind(&llm_yolo::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, @@ -312,7 +312,7 @@ class llm_cv : public StackFlow { if ((llm_task_channel_.size() - 1) == task_count_) { error_body["code"] = -21; error_body["message"] = "task full"; - send("None", "None", error_body, "cv"); + send("None", "None", error_body, unit_name_); return -1; } @@ -327,7 +327,7 @@ class llm_cv : public StackFlow { SLOGE("setup json format error."); error_body["code"] = -2; error_body["message"] = "json format error."; - send("None", "None", error_body, "cv"); + send("None", "None", error_body, unit_name_); return -2; } int ret = llm_task_obj->load_model(config_body); @@ -335,13 +335,13 @@ class llm_cv : public StackFlow { llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); - llm_task_obj->set_output(std::bind(&llm_cv::task_output, this, llm_task_obj, llm_channel, + llm_task_obj->set_output(std::bind(&llm_yolo::task_output, this, llm_task_obj, llm_channel, std::placeholders::_1, std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { - if (input.find("cv") != std::string::npos) { + if (input.find("yolo") != std::string::npos) { llm_channel->subscriber_work_id( - "", std::bind(&llm_cv::task_user_data, this, std::weak_ptr(llm_task_obj), + "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); } @@ -354,14 +354,14 @@ class llm_cv : public StackFlow { SLOGE("load_mode Failed"); error_body["code"] = -5; error_body["message"] = "Model loading failed."; - send("None", "None", error_body, "cv"); + send("None", "None", error_body, unit_name_); return -1; } } void link(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_cv::link:%s", data.c_str()); + SLOGI("llm_yolo::link:%s", data.c_str()); int ret = 1; nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -373,10 +373,10 @@ class llm_cv : public StackFlow { } auto llm_channel = get_channel(work_id); auto llm_task_obj = llm_task_[work_id_num]; - if (data.find("cv") != std::string::npos) { + if (data.find("yolo") != std::string::npos) { ret = llm_channel->subscriber_work_id( data, - std::bind(&llm_cv::task_user_data, this, std::weak_ptr(llm_task_obj), + std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } @@ -392,7 +392,7 @@ class llm_cv : public StackFlow { void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_cv::unlink:%s", data.c_str()); + SLOGI("llm_yolo::unlink:%s", data.c_str()); int ret = 0; nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -417,7 +417,7 @@ class llm_cv : public StackFlow { void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_cv::taskinfo:%s", data.c_str()); + SLOGI("llm_yolo::taskinfo:%s", data.c_str()); nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { @@ -425,7 +425,7 @@ class llm_cv : public StackFlow { std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), [](const auto task_channel) { return task_channel.second->work_id_; }); req_body = task_list; - send("cv.tasklist", req_body, LLM_NO_ERROR, work_id); + send("yolo.tasklist", req_body, LLM_NO_ERROR, work_id); } else { if (llm_task_.find(work_id_num) == llm_task_.end()) { req_body["code"] = -6; @@ -438,13 +438,13 @@ class llm_cv : public StackFlow { req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; req_body["inputs_"] = llm_task_obj->inputs_; - send("cv.taskinfo", req_body, LLM_NO_ERROR, work_id); + send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); } } int exit(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_cv::exit:%s", data.c_str()); + SLOGI("llm_yolo::exit:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -461,7 +461,7 @@ class llm_cv : public StackFlow { return 0; } - ~llm_cv() + ~llm_yolo() { while (1) { auto iteam = llm_task_.begin(); @@ -480,7 +480,7 @@ int main() signal(SIGTERM, __sigint); signal(SIGINT, __sigint); mkdir("/tmp/llm", 0777); - llm_cv llm; + llm_yolo llm; while (!main_exit_flage) { sleep(1); } diff --git a/projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/EngineWrapper.cpp rename to projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp diff --git a/projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp b/projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/EngineWrapper.hpp rename to projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/common.hpp b/projects/llm_framework/main_yolo/src/runner/base/common.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/common.hpp rename to projects/llm_framework/main_yolo/src/runner/base/common.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/detection.h b/projects/llm_framework/main_yolo/src/runner/base/detection.h similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/detection.h rename to projects/llm_framework/main_yolo/src/runner/base/detection.h diff --git a/projects/llm_framework/main_cv/src/runner/base/detection.hpp b/projects/llm_framework/main_yolo/src/runner/base/detection.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/detection.hpp rename to projects/llm_framework/main_yolo/src/runner/base/detection.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/pose.hpp b/projects/llm_framework/main_yolo/src/runner/base/pose.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/pose.hpp rename to projects/llm_framework/main_yolo/src/runner/base/pose.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/score.hpp b/projects/llm_framework/main_yolo/src/runner/base/score.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/score.hpp rename to projects/llm_framework/main_yolo/src/runner/base/score.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/topk.hpp b/projects/llm_framework/main_yolo/src/runner/base/topk.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/topk.hpp rename to projects/llm_framework/main_yolo/src/runner/base/topk.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/transform.hpp b/projects/llm_framework/main_yolo/src/runner/base/transform.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/transform.hpp rename to projects/llm_framework/main_yolo/src/runner/base/transform.hpp diff --git a/projects/llm_framework/main_cv/src/runner/base/yolo.hpp b/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/base/yolo.hpp rename to projects/llm_framework/main_yolo/src/runner/base/yolo.hpp diff --git a/projects/llm_framework/main_cv/src/runner/utils/checker.h b/projects/llm_framework/main_yolo/src/runner/utils/checker.h similarity index 100% rename from projects/llm_framework/main_cv/src/runner/utils/checker.h rename to projects/llm_framework/main_yolo/src/runner/utils/checker.h diff --git a/projects/llm_framework/main_cv/src/runner/utils/io.hpp b/projects/llm_framework/main_yolo/src/runner/utils/io.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/utils/io.hpp rename to projects/llm_framework/main_yolo/src/runner/utils/io.hpp diff --git a/projects/llm_framework/main_cv/src/runner/utils/logger.h b/projects/llm_framework/main_yolo/src/runner/utils/logger.h similarity index 100% rename from projects/llm_framework/main_cv/src/runner/utils/logger.h rename to projects/llm_framework/main_yolo/src/runner/utils/logger.h diff --git a/projects/llm_framework/main_cv/src/runner/utils/timer.hpp b/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp similarity index 100% rename from projects/llm_framework/main_cv/src/runner/utils/timer.hpp rename to projects/llm_framework/main_yolo/src/runner/utils/timer.hpp diff --git a/projects/llm_framework/main_cv/yolo11s.json b/projects/llm_framework/main_yolo/yolo11s.json similarity index 100% rename from projects/llm_framework/main_cv/yolo11s.json rename to projects/llm_framework/main_yolo/yolo11s.json From a4ab2ace0feb6fbe16d95227b3bfa7ccfa9ecd93 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 16:17:23 +0800 Subject: [PATCH 12/44] [clean] code --- .../src/runner => include}/utils/checker.h | 0 .../src/runner => include}/utils/io.hpp | 0 .../src/runner => include}/utils/logger.h | 0 .../llm_framework/main_melotts/SConstruct | 2 +- projects/llm_framework/main_yolo/SConstruct | 3 +- .../src/{runner => }/EngineWrapper.cpp | 187 +- .../src/{runner => }/EngineWrapper.hpp | 35 +- projects/llm_framework/main_yolo/src/main.cpp | 5 +- .../main_yolo/src/runner/base/common.hpp | 151 - .../main_yolo/src/runner/base/detection.h | 22 - .../main_yolo/src/runner/base/detection.hpp | 3122 ----------------- .../main_yolo/src/runner/base/pose.hpp | 432 --- .../main_yolo/src/runner/base/score.hpp | 33 - .../main_yolo/src/runner/base/topk.hpp | 52 - .../main_yolo/src/runner/base/transform.hpp | 47 - .../main_yolo/src/runner/base/yolo.hpp | 578 --- .../main_yolo/src/runner/utils/checker.h | 32 - .../main_yolo/src/runner/utils/io.hpp | 601 ---- .../main_yolo/src/runner/utils/logger.h | 82 - .../main_yolo/src/runner/utils/timer.hpp | 61 - 20 files changed, 127 insertions(+), 5318 deletions(-) rename projects/llm_framework/{main_melotts/src/runner => include}/utils/checker.h (100%) rename projects/llm_framework/{main_melotts/src/runner => include}/utils/io.hpp (100%) rename projects/llm_framework/{main_melotts/src/runner => include}/utils/logger.h (100%) rename projects/llm_framework/main_yolo/src/{runner => }/EngineWrapper.cpp (52%) rename projects/llm_framework/main_yolo/src/{runner => }/EngineWrapper.hpp (64%) delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/common.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/detection.h delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/detection.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/pose.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/score.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/topk.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/transform.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/base/yolo.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/utils/checker.h delete mode 100644 projects/llm_framework/main_yolo/src/runner/utils/io.hpp delete mode 100644 projects/llm_framework/main_yolo/src/runner/utils/logger.h delete mode 100644 projects/llm_framework/main_yolo/src/runner/utils/timer.hpp diff --git a/projects/llm_framework/main_melotts/src/runner/utils/checker.h b/projects/llm_framework/include/utils/checker.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/checker.h rename to projects/llm_framework/include/utils/checker.h diff --git a/projects/llm_framework/main_melotts/src/runner/utils/io.hpp b/projects/llm_framework/include/utils/io.hpp similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/io.hpp rename to projects/llm_framework/include/utils/io.hpp diff --git a/projects/llm_framework/main_melotts/src/runner/utils/logger.h b/projects/llm_framework/include/utils/logger.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/logger.h rename to projects/llm_framework/include/utils/logger.h diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index 21838316..6124c3e7 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -23,7 +23,7 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] REQUIREMENTS += ['onnxruntime', 'samplerate'] - +INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] STATIC_FILES += [AFile('melotts_zh-cn.json')] diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 50258ca9..7685db17 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -21,8 +21,7 @@ LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '- LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -INCLUDE += [ADir('src/runner')] -INCLUDE += [ADir('../include/opencv4')] +INCLUDE += [ADir('../include'), ADir('../include/opencv4')] static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') diff --git a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp similarity index 52% rename from projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp rename to projects/llm_framework/main_yolo/src/EngineWrapper.cpp index f199ee09..12bbb949 100644 --- a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -7,24 +7,27 @@ * written consent of Axera Semiconductor (Ningbo) Co., Ltd. * **************************************************************************************************/ +#include "detection.hpp" +#define UNUSE_STRUCT_OBJECT #include "EngineWrapper.hpp" #include "utils/io.hpp" #include -#include "base/detection.hpp" -static const char *strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; +static const char* strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; /// @brief npu type typedef enum axNPU_TYPE_E { AX_NPU_DEFAULT = 0, /* running under default NPU according to system */ - AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ - AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ - AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ - AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ - AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ + AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ + AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ + AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ + AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ + AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { +static AX_S32 CheckModelVNpu(const std::string& strModel, const AX_ENGINE_MODEL_TYPE_T& eModelType, + const AX_S32& nNpuType, AX_U32& nNpuSet) +{ AX_ENGINE_NPU_ATTR_T stNpuAttr; memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); @@ -32,29 +35,27 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ if (ret == 0) { // VNPU DISABLE if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_DISABLE) { - nNpuSet = 0x01; // NON-VNPU (0b111) + nNpuSet = 0x01; // NON-VNPU (0b111) } // STD VNPU else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { // 7.2T & 10.8T no allow - if (eModelType == AX_ENGINE_MODEL_TYPE1 - || eModelType == AX_ENGINE_MODEL_TYPE1) { + if (eModelType == AX_ENGINE_MODEL_TYPE1 || eModelType == AX_ENGINE_MODEL_TYPE1) { return -1; } // default STD VNPU2 if (nNpuType == 0) { - nNpuSet = 0x02; // VNPU2 (0b010) - } - else { + nNpuSet = 0x02; // VNPU2 (0b010) + } else { if (nNpuType & AX_STD_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } if (nNpuType & AX_STD_VNPU_2) { - nNpuSet |= 0x02; // VNPU2 (0b010) + nNpuSet |= 0x02; // VNPU2 (0b010) } if (nNpuType & AX_STD_VNPU_3) { - nNpuSet |= 0x04; // VNPU3 (0b100) + nNpuSet |= 0x04; // VNPU3 (0b100) } } } @@ -69,14 +70,13 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ if (nNpuType == 0) { // 7.2T default BL VNPU1 if (eModelType == AX_ENGINE_MODEL_TYPE1) { - nNpuSet = 0x01; // VNPU1 (0b001) + nNpuSet = 0x01; // VNPU1 (0b001) } // 3.6T default BL VNPU2 else { - nNpuSet = 0x02; // VNPU2 (0b010) + nNpuSet = 0x02; // VNPU2 (0b010) } - } - else { + } else { // 7.2T if (eModelType == AX_ENGINE_MODEL_TYPE1) { // no allow set to BL VNPU2 @@ -84,22 +84,21 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ return -1; } if (nNpuType & AX_BL_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } } // 3.6T else { if (nNpuType & AX_BL_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } if (nNpuType & AX_BL_VNPU_2) { - nNpuSet |= 0x02; // VNPU2 (0b010) + nNpuSet |= 0x02; // VNPU2 (0b010) } } } } - } - else { + } else { printf("AX_ENGINE_GetVNPUAttr fail ret = %x\n", ret); } @@ -111,27 +110,26 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) AX_S32 ret = 0; // 1. load model - AX_BOOL bLoadModelUseCmm = AX_FALSE; - AX_CHAR *pModelBufferVirAddr = nullptr; + AX_BOOL bLoadModelUseCmm = AX_FALSE; + AX_CHAR* pModelBufferVirAddr = nullptr; AX_U64 u64ModelBufferPhyAddr = 0; - AX_U32 nModelBufferSize = 0; + AX_U32 nModelBufferSize = 0; std::vector model_buffer; if (bLoadModelUseCmm) { - if (!utils::read_file(strModelPath, (AX_VOID **)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { + if (!utils::read_file(strModelPath, (AX_VOID**)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { printf("ALGO read model(%s) fail\n", strModelPath); return -1; } - } - else { + } else { if (!utils::read_file(strModelPath, model_buffer)) { printf("ALGO read model(%s) fail\n", strModelPath); return -1; } pModelBufferVirAddr = model_buffer.data(); - nModelBufferSize = model_buffer.size(); + nModelBufferSize = model_buffer.size(); } auto freeModelBuffer = [&]() { @@ -139,35 +137,34 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) if (u64ModelBufferPhyAddr != 0) { AX_SYS_MemFree(u64ModelBufferPhyAddr, &pModelBufferVirAddr); } - } - else { + } else { std::vector().swap(model_buffer); } return; }; // 1.1 Get Model Type - AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; - ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); - if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { - printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); - freeModelBuffer(); - return -1; - } + AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; + ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); + if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { + printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); + freeModelBuffer(); + return -1; + } // 1.2 Check VNPU - AX_ENGINE_NPU_SET_T nNpuSet = 0; - ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); - if (0 != ret) { - printf("ALGO CheckModelVNpu fail\n"); - freeModelBuffer(); - return -1; - } + AX_ENGINE_NPU_SET_T nNpuSet = 0; + ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); + if (0 != ret) { + printf("ALGO CheckModelVNpu fail\n"); + freeModelBuffer(); + return -1; + } // 2. create handle AX_ENGINE_HANDLE handle = nullptr; - ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); - auto deinit_handle = [&handle]() { + ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); + auto deinit_handle = [&handle]() { if (handle) { AX_ENGINE_DestroyHandle(handle); } @@ -190,26 +187,26 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) // 4. set io m_io_info = nullptr; - ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); + ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); if (0 != ret) { return deinit_handle(); } - m_input_num = m_io_info->nInputSize; + m_input_num = m_io_info->nInputSize; m_output_num = m_io_info->nOutputSize; // 4.1 query io -// AX_IMG_FORMAT_E eDtype; -// ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. -// if (0 != ret) { -// printf("model(%s) query model input size fail\n", strModelPath.c_str()); -// return deinit_handle(); -// } - -// if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || -// eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { -// printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); -// return deinit_handle(); -// } + // AX_IMG_FORMAT_E eDtype; + // ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. + // if (0 != ret) { + // printf("model(%s) query model input size fail\n", strModelPath.c_str()); + // return deinit_handle(); + // } + + // if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || + // eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { + // printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); + // return deinit_handle(); + // } // 4.2 brief io #ifdef __DEBUG__ @@ -217,9 +214,9 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) utils::brief_io_info(strModelPath, m_io_info); #endif - //5. Config VNPU - // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); - // will do nothing for using create handle v2 api + // 5. Config VNPU + // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); + // will do nothing for using create handle v2 api // 6. prepare io // AX_U32 nIoDepth = (stCtx.vecOutputBufferFlag.size() == 0) ? 1 : stCtx.vecOutputBufferFlag.size(); @@ -230,20 +227,20 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) return deinit_handle(); } - m_handle = handle; + m_handle = handle; m_hasInit = true; return 0; } -int EngineWrapper::SetInput(void* pInput, int index) { +int EngineWrapper::SetInput(void* pInput, int index) +{ return utils::push_io_input(pInput, index, m_io); } int EngineWrapper::RunSync() { - if (!m_hasInit) - return -1; + if (!m_hasInit) return -1; // 7.3 run & benchmark auto ret = AX_ENGINE_RunSync(m_handle, &m_io); @@ -256,25 +253,29 @@ int EngineWrapper::RunSync() } const char* CLASS_NAMES[] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", - "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", - "hair drier", "toothbrush"}; - -void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", + "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", + "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", + "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush"}; + +void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, + int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) { // std::vector objects; std::vector proposals; - for (int i = 0; i < 3; ++i) - { - auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; + for (int i = 0; i < 3; ++i) { + auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; int32_t stride = (1 << i) * 8; - detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, cls_num); + detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, + cls_num); } detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); fprintf(stdout, "detection num: %zu\n", objects.size()); @@ -282,21 +283,25 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); } -int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects) +int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, + float& nms_threshold, std::vector& objects) { post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects); return 0; } -int EngineWrapper::GetOutput(void* pOutput, int index) { +int EngineWrapper::GetOutput(void* pOutput, int index) +{ return utils::push_io_output(pOutput, index, m_io); } -int EngineWrapper::GetInputSize(int index) { +int EngineWrapper::GetInputSize(int index) +{ return m_io.pInputs[index].nSize; } -int EngineWrapper::GetOutputSize(int index) { +int EngineWrapper::GetOutputSize(int index) +{ return m_io.pOutputs[index].nSize; } diff --git a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp similarity index 64% rename from projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp rename to projects/llm_framework/main_yolo/src/EngineWrapper.hpp index 7da45496..4c018f4d 100644 --- a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp @@ -12,16 +12,34 @@ #include #include -#include #include "ax_engine_api.h" +#ifndef UNUSE_STRUCT_OBJECT +namespace detection { +typedef struct Object { + cv::Rect_ rect; + int label; + float prob; + cv::Point2f landmark[5]; + /* for yolov5-seg */ + cv::Mat mask; + std::vector mask_feat; + std::vector kps_feat; + /* for yolov8-obb */ + float angle; +} Object; + +} // namespace detection +#endif + class EngineWrapper { public: - EngineWrapper() : - m_hasInit(false), - m_handle(nullptr) {} + EngineWrapper() : m_hasInit(false), m_handle(nullptr) + { + } - ~EngineWrapper() { + ~EngineWrapper() + { Release(); } @@ -31,7 +49,8 @@ class EngineWrapper { int RunSync(); - int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects); + int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, + std::vector& objects); int GetOutput(void* pOutput, int index); @@ -43,7 +62,7 @@ class EngineWrapper { protected: bool m_hasInit; AX_ENGINE_HANDLE m_handle; - AX_ENGINE_IO_INFO_T *m_io_info{}; + AX_ENGINE_IO_INFO_T* m_io_info{}; AX_ENGINE_IO_T m_io{}; int m_input_num{}, m_output_num{}; -}; \ No newline at end of file +}; diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index ab9e3b3c..ef6fd1c8 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -5,8 +5,7 @@ */ #include "StackFlow.h" #include "EngineWrapper.hpp" -#include "base/common.hpp" -#include "base/detection.h" +#include "common.hpp" #include #include #include @@ -153,7 +152,7 @@ class llm_task { } std::vector objects; yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, - mode_config_.pron_threshold, mode_config_.nms_threshold, objects); + mode_config_.pron_threshold, mode_config_.nms_threshold, objects); std::vector yolo_output; for (size_t i = 0; i < objects.size(); i++) { const detection::Object &obj = objects[i]; diff --git a/projects/llm_framework/main_yolo/src/runner/base/common.hpp b/projects/llm_framework/main_yolo/src/runner/base/common.hpp deleted file mode 100644 index cc5ab2a3..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/common.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace common -{ - // opencv mat(h, w) - // resize cv::Size(dstw, dsth) - void get_input_data_no_letterbox(const cv::Mat& mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) - { - cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); - cv::resize(mat, img_new, cv::Size(model_w, model_h)); - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - void get_input_data_letterbox(cv::Mat mat, std::vector& image, int letterbox_rows, int letterbox_cols, bool bgr2rgb = false) - { - /* letterbox process to support different letterbox size */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / mat.rows) < (letterbox_cols * 1.0 / mat.cols)) - { - scale_letterbox = (float)letterbox_rows * 1.0f / (float)mat.rows; - } - else - { - scale_letterbox = (float)letterbox_cols * 1.0f / (float)mat.cols; - } - resize_cols = int(scale_letterbox * (float)mat.cols); - resize_rows = int(scale_letterbox * (float)mat.rows); - - cv::Mat img_new(letterbox_rows, letterbox_cols, CV_8UC3, image.data()); - - cv::resize(mat, mat, cv::Size(resize_cols, resize_rows)); - - int top = (letterbox_rows - resize_rows) / 2; - int bot = (letterbox_rows - resize_rows + 1) / 2; - int left = (letterbox_cols - resize_cols) / 2; - int right = (letterbox_cols - resize_cols + 1) / 2; - - // Letterbox filling - cv::copyMakeBorder(mat, img_new, top, bot, left, right, cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - void get_input_data_centercrop(cv::Mat mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) - { - /* letterbox process to support different letterbox size */ - - /* C2C BGR */ - if (mat.channels() == 4) - { - cv::cvtColor(mat, mat, cv::COLOR_BGRA2BGR); - } - - if (mat.channels() == 1) - { - cv::cvtColor(mat, mat, cv::COLOR_GRAY2BGR); - } - - /* Center */ - int h0; - int w0; - if (mat.rows < mat.cols) - { - h0 = 256; - w0 = int(mat.cols * (256.0 / mat.rows)); - } - else - { - h0 = int(mat.rows * (256.0 / mat.cols)); - w0 = 256; - } - int center_h = int(h0 / 2); - int center_w = int(w0 / 2); - - cv::resize(mat, mat, cv::Size(w0, h0)); - - // cv::imwrite("center.jpg", mat); - - /* Crop */ - cv::Rect crop_box(center_w - int(model_w / 2), center_h - int(model_h / 2), model_w, model_h); - cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); - - cv::Mat mat_crop = mat(crop_box).clone(); - - // cv::imwrite("mat_crop.jpg", mat_crop); - mat_crop.copyTo(img_new); - - // cv::imwrite("img_new.jpg", img_new); - - /* SwapRB*/ - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - bool read_file(const char* fn, std::vector& data) - { - FILE* fp = fopen(fn, "r"); - if (fp != nullptr) - { - fseek(fp, 0L, SEEK_END); - auto len = ftell(fp); - fseek(fp, 0, SEEK_SET); - data.clear(); - size_t read_size = 0; - if (len > 0) - { - data.resize(len); - read_size = fread(data.data(), 1, len, fp); - } - fclose(fp); - return read_size == (size_t)len; - } - return false; - } -} // namespace common \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/detection.h b/projects/llm_framework/main_yolo/src/runner/base/detection.h deleted file mode 100644 index cebeba13..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/detection.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include - -namespace detection -{ - - typedef struct Object - { - cv::Rect_ rect; - int label; - float prob; - cv::Point2f landmark[5]; - /* for yolov5-seg */ - cv::Mat mask; - std::vector mask_feat; - std::vector kps_feat; - /* for yolov8-obb */ - float angle; - } Object; - -} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/detection.hpp b/projects/llm_framework/main_yolo/src/runner/base/detection.hpp deleted file mode 100644 index 775eed05..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/detection.hpp +++ /dev/null @@ -1,3122 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -namespace detection -{ - typedef struct - { - int grid0; - int grid1; - int stride; - } GridAndStride; - - // typedef struct Object - // { - // cv::Rect_ rect; - // int label; - // float prob; - // cv::Point2f landmark[5]; - // /* for yolov5-seg */ - // cv::Mat mask; - // std::vector mask_feat; - // std::vector kps_feat; - // /* for yolov8-obb */ - // float angle; - // } Object; - - /* for palm detection */ - typedef struct PalmObject - { - cv::Rect_ rect; - float prob; - cv::Point2f vertices[4]; - cv::Point2f landmarks[7]; - cv::Mat affine_trans_mat; - cv::Mat affine_trans_mat_inv; - } PalmObject; - - static inline float sigmoid(float x) - { - return static_cast(1.f / (1.f + exp(-x))); - } - - static float softmax(const float* src, float* dst, int length) - { - const float alpha = *std::max_element(src, src + length); - float denominator = 0; - float dis_sum = 0; - for (int i = 0; i < length; ++i) - { - dst[i] = exp(src[i] - alpha); - denominator += dst[i]; - } - for (int i = 0; i < length; ++i) - { - dst[i] /= denominator; - dis_sum += i * dst[i]; - } - return dis_sum; - } - - template - static inline float intersection_area(const T& a, const T& b) - { - cv::Rect_ inter = a.rect & b.rect; - return inter.area(); - } - - template - static void qsort_descent_inplace(std::vector& faceobjects, int left, int right) - { - int i = left; - int j = right; - float p = faceobjects[(left + right) / 2].prob; - - while (i <= j) - { - while (faceobjects[i].prob > p) - i++; - - while (faceobjects[j].prob < p) - j--; - - if (i <= j) - { - // swap - std::swap(faceobjects[i], faceobjects[j]); - - i++; - j--; - } - } -#pragma omp parallel sections - { -#pragma omp section - { - if (left < j) qsort_descent_inplace(faceobjects, left, j); - } -#pragma omp section - { - if (i < right) qsort_descent_inplace(faceobjects, i, right); - } - } - } - - template - static void qsort_descent_inplace(std::vector& faceobjects) - { - if (faceobjects.empty()) - return; - - qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1); - } - - template - static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold) - { - picked.clear(); - - const int n = faceobjects.size(); - - std::vector areas(n); - for (int i = 0; i < n; i++) - { - areas[i] = faceobjects[i].rect.area(); - } - - for (int i = 0; i < n; i++) - { - const T& a = faceobjects[i]; - - int keep = 1; - for (int j = 0; j < (int)picked.size(); j++) - { - const T& b = faceobjects[picked[j]]; - - // intersection over union - float inter_area = intersection_area(a, b); - float union_area = areas[i] + areas[picked[j]] - inter_area; - // float IoU = inter_area / union_area - if (inter_area / union_area > nms_threshold) - keep = 0; - } - - if (keep) - picked.push_back(i); - } - } - - static void generate_grids_and_stride(const int target_w, const int target_h, std::vector& strides, std::vector& grid_strides) - { - for (auto stride : strides) - { - int num_grid_w = target_w / stride; - int num_grid_h = target_h / stride; - for (int g1 = 0; g1 < num_grid_h; g1++) - { - for (int g0 = 0; g0 < num_grid_w; g0++) - { - GridAndStride gs; - gs.grid0 = g0; - gs.grid1 = g1; - gs.stride = stride; - grid_strides.push_back(gs); - } - } - } - } - - static void generate_proposals_scrfd(int feat_stride, const float* score_blob, - const float* bbox_blob, const float* kps_blob, - float prob_threshold, std::vector& faceobjects, int letterbox_cols, int letterbox_rows) - { - static float anchors[] = {-8.f, -8.f, 8.f, 8.f, -16.f, -16.f, 16.f, 16.f, -32.f, -32.f, 32.f, 32.f, -64.f, -64.f, 64.f, 64.f, -128.f, -128.f, 128.f, 128.f, -256.f, -256.f, 256.f, 256.f}; - int feat_w = letterbox_cols / feat_stride; - int feat_h = letterbox_rows / feat_stride; - int feat_size = feat_w * feat_h; - int anchor_group = 1; - if (feat_stride == 8) - anchor_group = 1; - if (feat_stride == 16) - anchor_group = 2; - if (feat_stride == 32) - anchor_group = 3; - - // generate face proposal from bbox deltas and shifted anchors - const int num_anchors = 2; - - for (int q = 0; q < num_anchors; q++) - { - // shifted anchor - float anchor_y = anchors[(anchor_group - 1) * 8 + q * 4 + 1]; - - float anchor_w = anchors[(anchor_group - 1) * 8 + q * 4 + 2] - anchors[(anchor_group - 1) * 8 + q * 4 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 8 + q * 4 + 3] - anchors[(anchor_group - 1) * 8 + q * 4 + 1]; - - for (int i = 0; i < feat_h; i++) - { - float anchor_x = anchors[(anchor_group - 1) * 8 + q * 4 + 0]; - - for (int j = 0; j < feat_w; j++) - { - int index = i * feat_w + j; - - float prob = sigmoid(score_blob[q * feat_size + index]); - - if (prob >= prob_threshold) - { - // insightface/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py _get_bboxes_single() - float dx = bbox_blob[(q * 4 + 0) * feat_size + index] * feat_stride; - float dy = bbox_blob[(q * 4 + 1) * feat_size + index] * feat_stride; - float dw = bbox_blob[(q * 4 + 2) * feat_size + index] * feat_stride; - float dh = bbox_blob[(q * 4 + 3) * feat_size + index] * feat_stride; - // insightface/detection/scrfd/mmdet/core/bbox/transforms.py distance2bbox() - float cx = anchor_x + anchor_w * 0.5f; - float cy = anchor_y + anchor_h * 0.5f; - - float x0 = cx - dx; - float y0 = cy - dy; - float x1 = cx + dw; - float y1 = cy + dh; - - Object obj; - obj.label = 0; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0 + 1; - obj.rect.height = y1 - y0 + 1; - obj.prob = prob; - - if (kps_blob != 0) - { - obj.landmark[0].x = cx + kps_blob[index] * feat_stride; - obj.landmark[0].y = cy + kps_blob[1 * feat_h * feat_w + index] * feat_stride; - obj.landmark[1].x = cx + kps_blob[2 * feat_h * feat_w + index] * feat_stride; - obj.landmark[1].y = cy + kps_blob[3 * feat_h * feat_w + index] * feat_stride; - obj.landmark[2].x = cx + kps_blob[4 * feat_h * feat_w + index] * feat_stride; - obj.landmark[2].y = cy + kps_blob[5 * feat_h * feat_w + index] * feat_stride; - obj.landmark[3].x = cx + kps_blob[6 * feat_h * feat_w + index] * feat_stride; - obj.landmark[3].y = cy + kps_blob[7 * feat_h * feat_w + index] * feat_stride; - obj.landmark[4].x = cx + kps_blob[8 * feat_h * feat_w + index] * feat_stride; - obj.landmark[4].y = cy + kps_blob[9 * feat_h * feat_w + index] * feat_stride; - } - - faceobjects.push_back(obj); - } - - anchor_x += feat_stride; - } - - anchor_y += feat_stride; - } - } - } - - static void generate_proposals_mobilenet_ssd(const float* score, const float* boxes, const int head_count, const int* feature_map_size, const int* anchor_size, const int cls_num, - float prob_threshold, const float* strides, const float center_val, const float scale_val, const float* anchor_info, std::vector& objects) - { - auto ptr_score = score; - auto ptr_boxes = boxes; - auto ptr_anchor_info = anchor_info; - for (int head = 0; head < head_count; ++head) - { - for (int fea_h = 0; fea_h < feature_map_size[head]; ++fea_h) - { - for (int fea_w = 0; fea_w < feature_map_size[head]; ++fea_w) - { - for (int anchor_i = 0; anchor_i < anchor_size[head]; ++anchor_i) - { - float softmax_sum = 0; - float class_score = -FLT_MAX; - int class_index = 0; - for (int s = 0; s < cls_num + 1; s++) - { - softmax_sum += std::exp(ptr_score[s]); - } - for (int i = 0; i < cls_num + 1; ++i) - { - float temp = std::exp(ptr_score[i]) / softmax_sum; - // if (temp > class_score) - // { - // class_index = i; - // class_score = temp; - // } - - class_index = i; - class_score = temp; - - if (temp >= prob_threshold and class_index != 0) - { - // fprintf(stderr, "class_score: %f %d \n", class_score, i); - - float pred_x = (((float)fea_w + 0.5f) / (300.0f / strides[head]) + ptr_boxes[0] * center_val * ptr_anchor_info[anchor_i * 2] / 300.0f); - float pred_y = (((float)fea_h + 0.5f) / (300.0f / strides[head]) + ptr_boxes[1] * center_val * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f); - float pred_w = std::exp(ptr_boxes[2] * scale_val) * ptr_anchor_info[anchor_i * 2] / 300.0f; - float pred_h = std::exp(ptr_boxes[3] * scale_val) * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f; - - float x0 = (pred_x - pred_w * 0.5f) * 300.0f; - float y0 = (pred_y - pred_h * 0.5f) * 300.0f; - float x1 = (pred_x + pred_w * 0.5f) * 300.0f; - float y1 = (pred_y + pred_h * 0.5f) * 300.0f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = class_score; - - objects.push_back(obj); - } - } - - ptr_score += cls_num + 1; - ptr_boxes += 4; - } - } - } - ptr_anchor_info += anchor_size[head] * 2; - } - } - - static void generate_proposals_yolox(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - float box_objectness = feat_ptr[4]; - if (box_objectness < prob_threshold) - { - feat_ptr += cls_num + 5; - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = box_objectness * class_score; - - if (box_prob > prob_threshold) - { - float x_center = (feat_ptr[0] + w) * stride; - float y_center = (feat_ptr[1] + h) * stride; - float w = exp(feat_ptr[2]) * stride; - float h = exp(feat_ptr[3]) * stride; - float x0 = x_center - w * 0.5f; - float y0 = y_center - h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += cls_num + 5; - } - } - } - - static void generate_proposals_yolov7(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a_index = 0; a_index < 3; ++a_index) - { - float box_objectness = feat_ptr[4]; - if (box_objectness < prob_threshold) - { - feat_ptr += cls_num + 5; - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = box_objectness * class_score; - - if (box_prob > prob_threshold) - { - float x_center = (feat_ptr[0] * 2 - 0.5f + (float)w) * (float)stride; - float y_center = (feat_ptr[1] * 2 - 0.5f + (float)h) * (float)stride; - float box_w = (feat_ptr[2] * 2) * (feat_ptr[2] * 2) * anchors[a_index * 2]; - float box_h = (feat_ptr[3] * 2) * (feat_ptr[3] * 2) * anchors[a_index * 2 + 1]; - float x0 = x_center - box_w * 0.5f; - float y0 = y_center - box_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = box_w; - obj.rect.height = box_h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += cls_num + 5; - } - } - } - } - static void generate_proposals_yolov5_face(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 10); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 10]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 5; - for (int l = 0; l < 5; l++) - { - float lx = landmark_ptr[l * 2 + 0]; - float ly = landmark_ptr[l * 2 + 1]; - lx = lx * anchor_w + w * stride; - ly = ly * anchor_h + h * stride; - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 10); - } - } - } - } - - static void generate_proposals_yolov5_license_plate(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 8); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 8]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 5; - for (int l = 0; l < 4; l++) - { - float lx = landmark_ptr[l * 2 + 0]; - float ly = landmark_ptr[l * 2 + 1]; - lx = lx * anchor_w + w * stride; - ly = ly * anchor_h + h * stride; - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 8); - } - } - } - } - - static void generate_proposals_yolov5(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5); - } - } - } - } - - static void generate_proposals_yolov5_seg(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80, int mask_proto_dim = 32) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + mask_proto_dim); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - obj.mask_feat.resize(mask_proto_dim); - for (int k = 0; k < mask_proto_dim; k++) - { - obj.mask_feat[k] = feature_ptr[cls_num + 5 + k]; - } - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + mask_proto_dim); - } - } - } - } - - static void generate_proposals_yolov5_visdrone(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 10) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5); - } - } - } - } - - static void generate_proposals_yolov6(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 4]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = class_score; - - if (box_prob > prob_threshold) - { - float x0 = (w + 0.5f - feat_ptr[0]) * stride; - float y0 = (h + 0.5f - feat_ptr[1]) * stride; - float x1 = (w + 0.5f + feat_ptr[2]) * stride; - float y1 = (h + 0.5f + feat_ptr[3]) * stride; - - float w = x1 - x0; - float h = y1 - y0; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += 84; - } - } - } - - static void generate_proposals_yolov7_face(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 15); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 15]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 6; - for (int l = 0; l < 5; l++) - { - float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; - float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; - //float score = sigmoid(landmark_ptr[3 * l + 2]); - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 15); - } - } - } - } - - static void generate_proposals_yolov7_palm(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - const int landmark_sort[7] = {0, 3, 4, 5, 6, 1, 2}; - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 21); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 21]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - PalmObject obj; - obj.rect.x = x0 / (float)letterbox_cols; - obj.rect.y = y0 / (float)letterbox_rows; - obj.rect.width = (x1 - x0) / (float)letterbox_cols; - obj.rect.height = (y1 - y0) / (float)letterbox_rows; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 6; - std::vector tmp(7); - float min_x = FLT_MAX, min_y = FLT_MAX, max_x = 0, max_y = 0; - for (int l = 0; l < 7; l++) - { - float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; - float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; - lx /= (float)letterbox_cols; - ly /= (float)letterbox_rows; - - tmp[l] = cv::Point2f(lx, ly); - min_x = lx < min_x ? lx : min_x; - min_y = ly < min_y ? ly : min_y; - max_x = lx > max_x ? lx : max_x; - max_y = ly > max_y ? ly : max_y; - } - float w = max_x - min_x; - float h = max_y - min_y; - float long_side = h > w ? h : w; - long_side *= 1.1f; - obj.rect.x = min_x + w * 0.5f - long_side * 0.5f; - obj.rect.y = min_y + h * 0.5f - long_side * 0.5f; - obj.rect.width = long_side; - obj.rect.height = long_side; - for (int l = 0; l < 7; l++) - { - obj.landmarks[l] = tmp[landmark_sort[l]]; - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 21); - } - } - } - } - - static void generate_proposals_yolov8(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto dfl_ptr = dfl_feat; - auto cls_ptr = cls_feat; - auto cls_idx_ptr = cls_idx; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); - float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; - - float box_prob = sigmoid(class_score); - - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - dfl_ptr += (4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_seg(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto dfl_ptr = dfl_feat; - auto cls_ptr = cls_feat; - auto cls_idx_ptr = cls_idx; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); - float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.mask_feat.resize(mask_proto_dim); - for (int k = 0; k < mask_proto_dim; k++) - { - obj.mask_feat[k] = dfl_ptr[4 * reg_max + k]; - } - objects.push_back(obj); - } - - dfl_ptr += (4 * reg_max + mask_proto_dim); - } - } - } - - static void generate_proposals_yolov8_pose(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const int num_point = 17) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - auto scores = feat; - auto bboxes = feat + 1; - auto kps = feat + 1 + 4 * reg_max; - - float box_prob = sigmoid(*scores); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(bboxes + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = 0; - obj.prob = box_prob; - obj.kps_feat.clear(); - for (int k = 0; k < num_point; k++) - { - float kps_x = (kps[k * 3] * 2.f + w) * stride; - float kps_y = (kps[k * 3 + 1] * 2.f + h) * stride; - float kps_s = sigmoid(kps[k * 3 + 2]); - obj.kps_feat.push_back(kps_x); - obj.kps_feat.push_back(kps_y); - obj.kps_feat.push_back(kps_s); - } - objects.push_back(obj); - } - feat += (1 + 4 * reg_max + 3 * num_point); - } - } - } - - static void generate_proposals_yolov9(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += (cls_num + 4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_native(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += (cls_num + 4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_seg_native(int stride, const float* feat, const float* feat_seg, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - auto feat_seg_ptr = feat_seg; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.mask_feat.resize(mask_proto_dim); - memcpy(obj.mask_feat.data(), feat_seg_ptr, sizeof(float) * mask_proto_dim); - // for (int k = 0; k < mask_proto_dim; k++) - // { - // obj.mask_feat[k] = feat_seg_ptr[k]; - // } - objects.push_back(obj); - } - - feat_ptr += cls_num + 4 * reg_max; - feat_seg_ptr += mask_proto_dim; - } - } - } - - static void generate_proposals_yolov8_pose_native(int stride, const float* feat, const float* feat_kps, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const int num_point = 17, int cls_num = 1) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.kps_feat.clear(); - for (int k = 0; k < num_point; k++) - { - float kps_x = (feat_kps[k * 3] * 2.f + w) * stride; - float kps_y = (feat_kps[k * 3 + 1] * 2.f + h) * stride; - float kps_s = sigmoid(feat_kps[k * 3 + 2]); - obj.kps_feat.push_back(kps_x); - obj.kps_feat.push_back(kps_y); - obj.kps_feat.push_back(kps_s); - } - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * reg_max); - feat_kps += 3 * num_point; - } - } - } - - static void generate_proposals_yolo_world(int stride, const float* feat_cls, const float* feat_reg, float exp, float bias, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_cls_ptr = feat_cls; - auto feat_reg_ptr = feat_reg; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_cls_ptr[s]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - class_score = class_score * exp + bias; - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_reg_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_cls_ptr += cls_num; - feat_reg_ptr += 4 * reg_max; - } - } - } - - static void generate_proposals(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - int w_stride = (cls_num + 5); - int h_stride = feat_w * w_stride; - int a_stride = feat_h * h_stride; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - int offset = a * a_stride + h * h_stride + w * w_stride; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat[offset + s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feat[offset + 4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - int loc_idx = offset; - float dx = sigmoid(feat[loc_idx + 0]); - float dy = sigmoid(feat[loc_idx + 1]); - float dw = sigmoid(feat[loc_idx + 2]); - float dh = sigmoid(feat[loc_idx + 3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - } - } - } - } - - static void generate_proposals_palm(std::vector& region_list, float score_thresh, int input_img_w, int input_img_h, float* scores_ptr, float* bboxes_ptr, int head_count, const int* strides, const int* anchor_size, const float* anchor_offset, const int* feature_map_size, float prob_threshold_unsigmoid) - { - int idx = 0; - for (int i = 0; i < head_count; i++) - { - for (int y = 0; y < feature_map_size[i]; y++) - { - for (int x = 0; x < feature_map_size[i]; x++) - { - for (int k = 0; k < anchor_size[i]; k++) - { - if (scores_ptr[idx] < prob_threshold_unsigmoid) - { - idx++; - continue; - } - - const float x_center = (x + anchor_offset[i]) * 1.0f / feature_map_size[i]; - const float y_center = (y + anchor_offset[i]) * 1.0f / feature_map_size[i]; - float score = sigmoid(scores_ptr[idx]); - - if (score > score_thresh) - { - float* p = bboxes_ptr + (idx * 18); - - float cx = p[0] / input_img_w + x_center; - float cy = p[1] / input_img_h + y_center; - float w = p[2] / input_img_w; - float h = p[3] / input_img_h; - - float x0 = cx - w * 0.5f; - float y0 = cy - h * 0.5f; - float x1 = cx + w * 0.5f; - float y1 = cy + h * 0.5f; - - PalmObject region; - region.prob = score; - region.rect.x = x0; - region.rect.y = y0; - region.rect.width = x1 - x0; - region.rect.height = y1 - y0; - - for (int j = 0; j < 7; j++) - { - float lx = p[4 + (2 * j) + 0]; - float ly = p[4 + (2 * j) + 1]; - lx += x_center * input_img_w; - ly += y_center * input_img_h; - lx /= (float)input_img_w; - ly /= (float)input_img_h; - - region.landmarks[j].x = lx; - region.landmarks[j].y = ly; - } - region_list.push_back(region); - } - idx++; - } - } - } - } - } - - static void draw_objects(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name,double fontScale = 0.5, int thickness = 1) - { - static const std::vector COCO_COLORS = { - {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - - cv::rectangle(image, obj.rect, COCO_COLORS[obj.label], thickness); - - char text[256]; - sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(0, 0, 0), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, fontScale, - cv::Scalar(255, 255, 255), thickness); - } - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_keypoints(const cv::Mat& bgr, const std::vector& objects, - const std::vector >& kps_colors, - const std::vector >& limb_colors, - const std::vector >& skeleton, - const char* output_name) - { - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], person\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); - - cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); - - char text[256]; - sprintf(text, "person %.1f%%", obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - - const int num_point = obj.kps_feat.size() / 3; - for (int j = 0; j < num_point + 2; j++) - { - // draw circle - if (j < num_point) - { - int kps_x = std::round(obj.kps_feat[j * 3]); - int kps_y = std::round(obj.kps_feat[j * 3 + 1]); - float kps_s = obj.kps_feat[j * 3 + 2]; - if (kps_s > 0.5f) - { - auto kps_color = cv::Scalar(kps_colors[j][0], kps_colors[j][1], kps_colors[j][2]); - cv::circle(image, {kps_x, kps_y}, 5, kps_color, -1); - } - } - // draw line - auto& ske = skeleton[j]; - int pos1_x = obj.kps_feat[(ske[0] - 1) * 3]; - int pos1_y = obj.kps_feat[(ske[0] - 1) * 3 + 1]; - - int pos2_x = obj.kps_feat[(ske[1] - 1) * 3]; - int pos2_y = obj.kps_feat[(ske[1] - 1) * 3 + 1]; - - float pos1_s = obj.kps_feat[(ske[0] - 1) * 3 + 2]; - float pos2_s = obj.kps_feat[(ske[1] - 1) * 3 + 2]; - - if (pos1_s > 0.5f && pos2_s > 0.5f) - { - auto limb_color = cv::Scalar(limb_colors[j][0], limb_colors[j][1], limb_colors[j][2]); - cv::line(image, {pos1_x, pos1_y}, {pos2_x, pos2_y}, limb_color, 2); - } - } - } - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_mask(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const std::vector >& colors, const char* output_name) - { - cv::Mat image = bgr.clone(); - cv::Mat mask = bgr.clone(); - int color_index = 0; - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - const auto& color = colors[color_index % 80]; - color_index++; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - - mask(cv::Rect((int)obj.rect.x, (int)obj.rect.y, (int)objects[i].rect.width, (int)objects[i].rect.height)).setTo(color, objects[i].mask); - - cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); - - char text[256]; - sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - } - float blended_alpha = 0.5; - image = (1 - blended_alpha) * mask + blended_alpha * image; - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_palm(const cv::Mat& bgr, const std::vector& objects, const char* output_name) - { - cv::Mat image = bgr.clone(); - for (size_t i = 0; i < objects.size(); i++) - { - const PalmObject& obj = objects[i]; - //fprintf(stdout, "prob:%.2f, x0:%.2f, y0:%.2f, x1:%.2f, y1:%.2f, x2:%.2f, y2:%.2f, x3:%.2f, y3:%.2f\n", obj.prob, - // obj.vertices[0].x, obj.vertices[0].y, obj.vertices[1].x, obj.vertices[1].y, obj.vertices[2].x, - // obj.vertices[2].y, obj.vertices[3].x, obj.vertices[3].y); - cv::line(image, obj.vertices[0], obj.vertices[1], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[1], obj.vertices[2], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[2], obj.vertices[3], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[3], obj.vertices[0], cv::Scalar(0, 0, 255), 2, 8, 0); - for (auto ld : obj.landmarks) - { - cv::circle(image, ld, 2, cv::Scalar(0, 255, 0), -1, 8); - } - } - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_yolopv2(const cv::Mat& bgr, const std::vector& objects, const cv::Mat& da_seg_mask, const cv::Mat& ll_seg_mask, const char* output_name) - { - cv::Mat image = bgr.clone(); - cv::Mat mask = bgr.clone(); - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f]\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); - - cv::rectangle(image, obj.rect, cv::Scalar(0, 255, 255), 2, 8, 0); - } - - mask.setTo(cv::Scalar(0, 255, 0), da_seg_mask); - mask.setTo(cv::Scalar(0, 0, 255), ll_seg_mask); - float blended_alpha = 0.5; - image = (1 - blended_alpha) * mask + blended_alpha * image; - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - void reverse_letterbox(std::vector& proposal, std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = proposal.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposal[i]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox_no_letterbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int model_h, int model_w, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float ratio_x = (float)src_cols / (float)model_w; - float ratio_y = (float)src_rows / (float)model_h; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0)*ratio_x; - y0 = (y0)*ratio_y; - x1 = (x1)*ratio_x; - y1 = (y1)*ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox(std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = objects.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - for (int l = 0; l < 5; l++) - { - auto lx = objects[i].landmark[l].x; - auto ly = objects[i].landmark[l].y; - objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); - } - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - for (int l = 0; l < 5; l++) - { - auto lx = objects[i].landmark[l].x; - auto ly = objects[i].landmark[l].y; - objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); - } - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox_mask(std::vector& proposals, std::vector& objects, const float* mask_proto, int mask_proto_dim, int mask_stride, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int mask_proto_h = int(letterbox_rows / mask_stride); - int mask_proto_w = int(letterbox_cols / mask_stride); - - int count = picked.size(); - objects.resize(count); - - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - /* naive RoiAlign by opencv */ - int hstart = std::floor(objects[i].rect.y / mask_stride); - int hend = std::ceil(objects[i].rect.y / mask_stride + objects[i].rect.height / mask_stride); - int wstart = std::floor(objects[i].rect.x / mask_stride); - int wend = std::ceil(objects[i].rect.x / mask_stride + objects[i].rect.width / mask_stride); - - hstart = std::min(std::max(hstart, 0), mask_proto_h); - wstart = std::min(std::max(wstart, 0), mask_proto_w); - hend = std::min(std::max(hend, 0), mask_proto_h); - wend = std::min(std::max(wend, 0), mask_proto_w); - - int mask_w = wend - wstart; - int mask_h = hend - hstart; - - cv::Mat mask = cv::Mat(mask_h, mask_w, CV_32FC1); - if (mask_w > 0 && mask_h > 0) - { - std::vector roi_ranges; - roi_ranges.push_back(cv::Range(0, 1)); - roi_ranges.push_back(cv::Range::all()); - roi_ranges.push_back(cv::Range(hstart, hend)); - roi_ranges.push_back(cv::Range(wstart, wend)); - - cv::Mat mask_protos = cv::Mat(mask_proto_dim, mask_proto_h * mask_proto_w, CV_32FC1, (float*)mask_proto); - int sz[] = {1, mask_proto_dim, mask_proto_h, mask_proto_w}; - cv::Mat mask_protos_reshape = mask_protos.reshape(1, 4, sz); - cv::Mat protos = mask_protos_reshape(roi_ranges).clone().reshape(0, {mask_proto_dim, mask_w * mask_h}); - cv::Mat mask_proposals = cv::Mat(1, mask_proto_dim, CV_32FC1, (float*)objects[i].mask_feat.data()); - cv::Mat masks_feature = (mask_proposals * protos); - /* sigmoid */ - cv::exp(-masks_feature.reshape(1, {mask_h, mask_w}), mask); - mask = 1.0 / (1.0 + mask); - } - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - cv::resize(mask, mask, cv::Size((int)objects[i].rect.width, (int)objects[i].rect.height)); - objects[i].mask = mask > 0.5; - } - } - - void get_out_bbox_kps(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov8 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - - for (int j = 0; j < objects[i].kps_feat.size() / 3; j++) - { - objects[i].kps_feat[j * 3] = std::max( - std::min((objects[i].kps_feat[j * 3] - tmp_w) * ratio_x, (float)(src_cols - 1)), 0.f); - objects[i].kps_feat[j * 3 + 1] = std::max( - std::min((objects[i].kps_feat[j * 3 + 1] - tmp_h) * ratio_y, (float)(src_rows - 1)), 0.f); - } - } - } - - static void transform_rects_palm(PalmObject& object) - { - float x0 = object.landmarks[0].x; - float y0 = object.landmarks[0].y; - float x1 = object.landmarks[2].x; - float y1 = object.landmarks[2].y; - float rotation = M_PI * 0.5f - std::atan2(-(y1 - y0), x1 - x0); - - float hand_cx; - float hand_cy; - float shift_x = 0.0f; - float shift_y = -0.5f; - if (rotation == 0) - { - hand_cx = object.rect.x + object.rect.width * 0.5f + (object.rect.width * shift_x); - hand_cy = object.rect.y + object.rect.height * 0.5f + (object.rect.height * shift_y); - } - else - { - float dx = (object.rect.width * shift_x) * std::cos(rotation) - (object.rect.height * shift_y) * std::sin(rotation); - float dy = (object.rect.width * shift_x) * std::sin(rotation) + (object.rect.height * shift_y) * std::cos(rotation); - hand_cx = object.rect.x + object.rect.width * 0.5f + dx; - hand_cy = object.rect.y + object.rect.height * 0.5f + dy; - } - - float long_side = (std::max)(object.rect.width, object.rect.height); - float dx = long_side * 1.3f; - float dy = long_side * 1.3f; - - object.vertices[0].x = -dx; - object.vertices[0].y = -dy; - object.vertices[1].x = +dx; - object.vertices[1].y = -dy; - object.vertices[2].x = +dx; - object.vertices[2].y = +dy; - object.vertices[3].x = -dx; - object.vertices[3].y = +dy; - - for (int i = 0; i < 4; i++) - { - float sx = object.vertices[i].x; - float sy = object.vertices[i].y; - object.vertices[i].x = sx * std::cos(rotation) - sy * std::sin(rotation); - object.vertices[i].y = sx * std::sin(rotation) + sy * std::cos(rotation); - object.vertices[i].x += hand_cx; - object.vertices[i].y += hand_cy; - } - } - - static void get_out_bbox_palm(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - int count = picked.size(); - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - transform_rects_palm(objects[i]); - } - - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_cols / resize_cols; - float ratio_y = (float)src_rows / resize_rows; - - for (auto& object : objects) - { - for (auto& vertice : object.vertices) - { - vertice.x = (vertice.x * letterbox_cols - tmp_w) * ratio_x; - vertice.y = (vertice.y * letterbox_rows - tmp_h) * ratio_y; - } - - for (auto& ld : object.landmarks) - { - ld.x = (ld.x * letterbox_cols - tmp_w) * ratio_x; - ld.y = (ld.y * letterbox_rows - tmp_h) * ratio_y; - } - // get warpaffine transform mat to landmark detect - cv::Point2f src_pts[4]; - src_pts[0] = object.vertices[0]; - src_pts[1] = object.vertices[1]; - src_pts[2] = object.vertices[2]; - src_pts[3] = object.vertices[3]; - - cv::Point2f dst_pts[4]; - dst_pts[0] = cv::Point2f(0, 0); - dst_pts[1] = cv::Point2f(224, 0); - dst_pts[2] = cv::Point2f(224, 224); - dst_pts[3] = cv::Point2f(0, 224); - - object.affine_trans_mat = cv::getAffineTransform(src_pts, dst_pts); - cv::invertAffineTransform(object.affine_trans_mat, object.affine_trans_mat_inv); - } - } - - void get_out_bbox_yolopv2(std::vector& proposals, std::vector& objects, const float* da_ptr, const float* ll_ptr, cv::Mat& ll_seg_mask, cv::Mat& da_seg_mask, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - - cv::Mat ll = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)ll_ptr); - ll = ll > 0.5; - cv::resize(ll(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), ll_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_LINEAR); - - cv::Mat da = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)da_ptr); - da = da > 0; - cv::resize(da(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), da_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_NEAREST); - } - - namespace mmyolo - { - inline static float clamp( - float val, - float min = 0.f, - float max = 1536.f) - { - return val > min ? (val < max ? val : max) : min; - } - - inline float fast_exp(const float& x) - { - union - { - uint32_t i; - float f; - } v{}; - v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); - return v.f; - } - - inline float fast_sigmoid(const float& x) - { - return 1.0f / (1.0f + fast_exp(-x)); - } - - inline static float fast_softmax( - const float* src, - float* dst, - int length) - { - const float alpha = *std::max_element(src, src + length); - float denominator = 0; - float dis_sum = 0; - for (int i = 0; i < length; ++i) - { - dst[i] = fast_exp(src[i] - alpha); - denominator += dst[i]; - } - for (int i = 0; i < length; ++i) - { - dst[i] /= denominator; - dis_sum += i * dst[i]; - } - return dis_sum; - } - - static void generate_proposals_ppyoloeplus( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - int reg_max = 17; - float dis_after_sm[reg_max]; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); - float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); - float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); - float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4 * reg_max; - } - } - } - - static void generate_proposals_yolox( - int stride, - const float* cls_feat, - const float* box_feat, - const float* conf_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - auto conf_ptr = conf_feat; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - //process cls score - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max) * fast_sigmoid(*conf_ptr); - - if (box_prob > prob_threshold) - { - float x = (w + boxes_ptr[0]) * stride; - float y = (h + boxes_ptr[1]) * stride; - - float width = fast_exp(boxes_ptr[2]) * stride; - float height = fast_exp(boxes_ptr[3]) * stride; - - float x0 = x - width * 0.5f; - float y0 = y - height * 0.5f; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = width; - obj.rect.height = height; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4; - conf_ptr++; - } - } - } - - static void generate_proposals_yolov6( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - //process cls score - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = (w + 0.5f - boxes_ptr[0]) * stride; - float y0 = (h + 0.5f - boxes_ptr[1]) * stride; - float x1 = (w + 0.5f + boxes_ptr[2]) * stride; - float y1 = (h + 0.5f + boxes_ptr[3]) * stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4; - } - } - } - - static void generate_proposals_yolov8( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - int reg_max = 16; - float dis_after_sm[reg_max]; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); - float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); - float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); - float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4 * reg_max; - } - } - } - } // namespace mmyolo - - namespace obb - { - static inline float fast_exp(const float& x) - { - union - { - uint32_t i; - float f; - } v; - v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); - return v.f; - } - - static inline float clamp_(float v, float min = 0.f, float max = std::numeric_limits::infinity()) - { - return std::max(std::min(v, max), min); - } - static inline void get_covariance_matrix( - const std::vector& objects, - std::vector& covar_maxtirx) - { - int n = objects.size(); - for (size_t i = 0; i < n; ++i) { - - float a = objects[i].rect.width * objects[i].rect.width * 0.0833333333f; - float b = objects[i].rect.height * objects[i].rect.height * 0.0833333333f; - float c = objects[i].angle; - - float c_sin = std::sin(c); - float c_cos = std::cos(c); - - float x = a * c_cos * c_cos + b * c_sin * c_sin; - float y = a * c_sin * c_sin + b * c_cos * c_cos; - float z = a * c_cos * c_sin - b * c_sin * c_cos; - covar_maxtirx.push_back(cv::Point3f(x, y, z)); - } - } - - static inline float fast_log2 (float x) - { - union { - float f; - int i; - } v; - v.f = x; - int* const exp_ptr = &v.i; - int m = *exp_ptr; - const int log_2 = ((m >> 23) & 255) - 128; - m &= ~(255 << 23); - m += 127 << 23; - *exp_ptr = m; - return (((-1.f / 3) * v.f + 2) * v.f - 2.f / 3 + log_2); - } - - static inline float fast_log (const float x) - { - return 0.69314718f * fast_log2 (x); - } - - static inline float probiou( - const Object& obj1, - const Object& obj2, - const cv::Point3f& covar1, - const cv::Point3f& covar2) - { - - float v_x1_x2 = covar1.x + covar2.x; - float v_y1_y2 = covar1.y + covar2.y; - float x1_x2 = obj1.rect.x - obj2.rect.x; - float y1_y2 = obj1.rect.y - obj2.rect.y; - float dem1 = clamp_(covar1.x * covar1.y - covar1.z * covar1.z); - float dem2 = clamp_(covar2.x * covar2.y - covar2.z * covar2.z); - float dem_num = v_x1_x2 * v_y1_y2 - (covar1.z + covar2.z) * (covar1.z + covar2.z); - - float t1 = (v_x1_x2 * y1_y2 * y1_y2 + v_y1_y2 * x1_x2 * x1_x2) / (dem_num + 1e-7) * 0.25f; - float t2 = ((covar1.z + covar2.z) * (-x1_x2) * y1_y2) / (dem_num + 1e-7) * 0.5f; - float t3 = fast_log(0.25f * dem_num / (std::sqrt(dem1 * dem2) + 1e-7) + 1e-7) * 0.5f; - - float bd = t1 + t2 + t3; - float iou = fast_exp(-clamp_(bd, 1e-7, 100.f)); - return iou; - } - - static inline void nms_rotated_sorted_bboxes( - const std::vector& objects, - std::vector& picked, - float nms_threshold) - { - std::vector covar_maxtrix; - get_covariance_matrix(objects, covar_maxtrix); - - float nms_threshold_ = (1.f - nms_threshold) * (1.f - nms_threshold); - int n = objects.size(); - - for (size_t i = 0; i < n; ++i) { - float max_iou = 0.f; - const Object& obj1 = objects[i]; - - for (size_t j = 0; j < i; ++j) { - const Object& obj2 = objects[j]; - - float iou = probiou(obj1, obj2, covar_maxtrix[i], covar_maxtrix[j]); - if (iou > max_iou){ - max_iou = iou; - } - - } - if (max_iou < nms_threshold_) - picked.push_back(i); - } - } - static void get_out_obb_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - obb::nms_rotated_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - double pi = M_PI; - double pi_2 = M_PI_2; - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; - - float w_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.width : objects[i].rect.height; - float h_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.height : objects[i].rect.width; - float a_ = (float)std::fmod((objects[i].rect.width > objects[i].rect.height ? objects[i].angle : objects[i].angle + pi_2), pi); - - float xc = (objects[i].rect.x - tmp_w) * ratio_x; - float yc = (objects[i].rect.y - tmp_h) * ratio_y; - float w = w_ * ratio_x; - float h = h_ * ratio_y; - - // clip - xc = std::max(std::min(xc, (float)(src_cols - 1)), 0.f); - yc = std::max(std::min(yc, (float)(src_rows - 1)), 0.f); - w = std::max(std::min(w, (float)(src_cols - 1)), 0.f); - h = std::max(std::min(h, (float)(src_rows - 1)), 0.f); - - - objects[i].rect.x = xc; - objects[i].rect.y = yc; - objects[i].rect.width = w; - objects[i].rect.height = h; - objects[i].angle = a_; - } - } - static void generate_proposals_yolov8_obb_native(const std::vector& grid_strides, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 15) - { - const int num_points = grid_strides.size(); - int reg_max = 16; - auto feat_ptr = feat; - std::vector dis_after_sm(reg_max, 0.f); - for (int i = 0; i < num_points; i++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * grid_strides[i].stride; - } - - float angle = feat_ptr[4 * reg_max + cls_num]; - - float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; - float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; - - float cos = std::cos(angle); - float sin = std::sin(angle); - - float x = (pred_ltrb[2] - pred_ltrb[0]) * 0.5f; - float y = (pred_ltrb[3] - pred_ltrb[1]) * 0.5f; - float xc = x * cos - y * sin + pb_cx; - float yc = x * sin + y * cos + pb_cy; - float w = pred_ltrb[2] + pred_ltrb[0]; - float h = pred_ltrb[3] + pred_ltrb[1]; - - Object obj; - obj.rect.x = xc; //center x - obj.rect.y = yc; //center y - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - obj.angle = angle; - - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * reg_max + 1); - } - } - static void draw_objects_obb(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name, int thickness = 1) - { - static const std::vector COCO_COLORS = { - {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++){ - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - { - float xc = obj.rect.x; - float yc = obj.rect.y; - float w = obj.rect.width; - float h = obj.rect.height; - float ag = obj.angle; - float wx = w / 2 * std::cos(ag); - float wy = w / 2 * std::sin(ag); - float hx = -h / 2 * std::sin(ag); - float hy = h / 2 * std::cos(ag); - cv::Point2f p1{ xc - wx - hx, yc - wy - hy }; - cv::Point2f p2{ xc + wx - hx, yc + wy - hy }; - cv::Point2f p3{ xc + wx + hx, yc + wy + hy }; - cv::Point2f p4{ xc - wx + hx, yc - wy + hy }; - std::vector points = { p1, p2, p3, p4, p1 }; - std::vector> contours= { points }; - cv::polylines(image, contours, true, COCO_COLORS[obj.label], thickness, cv::LINE_AA); - } - } - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - } // namespace obb - - static void generate_proposals_yolov10(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto feat_ptr = feat; - float dis[16]; - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - int c_index = 0; - float c_score = 0; - - for (int c = 0; c < cls_num; c++) - { - float score = feat_ptr[c]; - if (score > c_score) - { - c_index = c; - c_score = score; - } - } - - if(c_score >= prob_threshold) - { - float x0 = w + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 0 * 16,dis, 16); - float y0 = h + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 1 * 16,dis, 16); - float x1 = w + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 2 * 16,dis, 16); - float y1 = h + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 3 * 16,dis, 16); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0;; - obj.rect.height = y1 - y0; - obj.label = c_index; - obj.prob = c_score; - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * 16); - } - } - } - -} // namespace detection diff --git a/projects/llm_framework/main_yolo/src/runner/base/pose.hpp b/projects/llm_framework/main_yolo/src/runner/base/pose.hpp deleted file mode 100644 index b06badfc..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/pose.hpp +++ /dev/null @@ -1,432 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace pose -{ - typedef struct - { - float x; - float y; - float score; - } ai_point_t; - - struct skeleton - { - int connection[2]; - int left_right_neutral; - }; - - std::vector pairs = {{15, 13, 0}, - {13, 11, 0}, - {16, 14, 0}, - {14, 12, 0}, - {11, 12, 0}, - {5, 11, 0}, - {6, 12, 0}, - {5, 6, 0}, - {5, 7, 0}, - {6, 8, 0}, - {7, 9, 0}, - {8, 10, 0}, - {1, 2, 0}, - {0, 1, 0}, - {0, 2, 0}, - {1, 3, 0}, - {2, 4, 0}, - {0, 5, 0}, - {0, 6, 0}}; - std::vector hand_pairs = {{0, 1, 0}, - {1, 2, 0}, - {2, 3, 0}, - {3, 4, 0}, - {0, 5, 1}, - {5, 6, 1}, - {6, 7, 1}, - {7, 8, 1}, - {0, 9, 2}, - {9, 10, 2}, - {10, 11, 2}, - {11, 12, 2}, - {0, 13, 3}, - {13, 14, 3}, - {14, 15, 3}, - {15, 16, 3}, - {0, 17, 4}, - {17, 18, 4}, - {18, 19, 4}, - {19, 20, 4}}; - std::vector animal_pairs = {{19, 15, 0}, - {18, 14, 0}, - {17, 13, 0}, - {16, 12, 0}, - {15, 11, 0}, - {14, 10, 0}, - {13, 9, 0}, - {12, 8, 0}, - {11, 6, 0}, - {10, 6, 0}, - {9, 7, 0}, - {8, 7, 0}, - {6, 7, 0}, - {7, 5, 0}, - {5, 4, 0}, - {0, 2, 0}, - {1, 3, 0}, - {0, 1, 0}, - {0, 4, 0}, - {1, 4, 0}}; - - typedef struct ai_body_parts_s - { - std::vector keypoints; - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_body_parts_s; - - typedef struct ai_hand_parts_s - { - std::vector keypoints; - int32_t hand_side = 0; //0-left hand,1-right hand - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_hand_parts_s; - - typedef struct ai_animal_parts_s - { - std::vector keypoints; - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_animal_parts_s; - - static inline void find_max_2d(float* buf, int width, int height, int* max_idx_width, int* max_idx_height, float* max_value, int c) - { - float* ptr = buf; - *max_value = -10.f; - *max_idx_width = 0; - *max_idx_height = 0; - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - float score = ptr[c * height * width + h * width + w]; - if (score > *max_value) - { - *max_value = score; - *max_idx_height = h; - *max_idx_width = w; - } - } - } - } - - static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - } - - static inline void draw_animal_result(cv::Mat img, ai_animal_parts_s& pose, int joints_num, int model_w, int model_h) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : animal_pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - } - - static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h, const detection::Object& obj) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x); - int y = (int)(pose.keypoints[i].y); - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x); - int y1 = (int)(pose.keypoints[element.connection[0]].y); - int x2 = (int)(pose.keypoints[element.connection[1]].x); - int y2 = (int)(pose.keypoints[element.connection[1]].y); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - // 画框 - cv::rectangle(img, obj.rect, cv::Scalar(255, 0, 0)); - char text[256]; - sprintf(text, "%s %.1f%%", "person", obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > img.cols) - x = img.cols - label_size.width; - - cv::rectangle(img, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(img, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - cv::imwrite("./pose_ppl_out.png", img); - } - - static inline void draw_result_hand(cv::Mat img, ai_hand_parts_s& pose, int joints_num) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 0, 255), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : hand_pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(10, 215, 255); - break; - case 1: - color = cv::Scalar(255, 115, 55); - break; - case 2: - color = cv::Scalar(5, 255, 55); - break; - case 3: - color = cv::Scalar(25, 15, 255); - break; - default: - color = cv::Scalar(225, 15, 55); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - - cv::imwrite("./hand_pose_out.png", img); - } - - static inline void post_process(float* data, ai_body_parts_s& pose, int joint_num, int img_h, int img_w) - { - int heatmap_width = img_w / 4; - int heatmap_height = img_h / 4; - int max_idx_width, max_idx_height; - float max_score; - - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); - kp.x = (float)max_idx_width / (float)heatmap_width; - kp.y = (float)max_idx_height / (float)heatmap_height; - kp.score = max_score; - pose.keypoints.push_back(kp); - - // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " - // << pose.keypoints[c].score << std::endl; - } - } - - static inline void animal_post_process(float* data, ai_animal_parts_s& pose, int joint_num, int img_h, int img_w) - { - int heatmap_width = img_w / 4; - int heatmap_height = img_h / 4; - int max_idx_width, max_idx_height; - float max_score; - - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); - kp.x = (float)max_idx_width / (float)heatmap_width; - kp.y = (float)max_idx_height / (float)heatmap_height; - kp.score = max_score; - pose.keypoints.push_back(kp); - - // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " - // << pose.keypoints[c].score << std::endl; - } - } - - static inline void ppl_pose_post_process(float* data1, float* data2, ai_body_parts_s& pose, int joint_num, int img_h, int img_w, int offset_top, int offset_left, int offset_x, int offset_y, float ratio) - { - ai_point_t kp; - - for (int c = 0; c < joint_num; ++c) - { - kp.x = (data1[c] / 2 - offset_left) / ratio + offset_x; - kp.y = (data2[c] / 2 - offset_top) / ratio + offset_y; - std::cout << "x1: " << kp.x << ", y1: " << kp.y << std::endl; - pose.keypoints.push_back(kp); - } - } - - static inline void post_process_hand(float* point_data, float* score_data, ai_hand_parts_s& pose, int joint_num, int img_h, int img_w) - { - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - kp.x = (float)point_data[c * 3] / img_w; - kp.y = (float)point_data[c * 3 + 1] / img_h; - pose.keypoints.push_back(kp); - } - if (score_data[0] > 0.5) - pose.hand_side = 1; - } - -} // namespace pose \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/score.hpp b/projects/llm_framework/main_yolo/src/runner/base/score.hpp deleted file mode 100644 index 080815d6..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/score.hpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include - - -namespace classification -{ - typedef struct - { - uint32_t id; - float score; - } score; -} diff --git a/projects/llm_framework/main_yolo/src/runner/base/topk.hpp b/projects/llm_framework/main_yolo/src/runner/base/topk.hpp deleted file mode 100644 index 50d43144..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/topk.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include - -#include "base/score.hpp" - - -namespace classification -{ - void sort_score(std::vector& array, bool reverse = false) - { - auto compare_func = [](const score& a, const score& b) -> bool - { - return a.score > b.score; - }; - - std::sort(array.begin(), array.end(), compare_func); - - if (reverse) std::reverse(array.begin(), array.end()); - } - - - void print_score(const std::vector& array, const size_t& n) - { - for (size_t i = 0; i < n; i++) - { - fprintf(stdout, "%.4f, %d\n", array[i].score, array[i].id); - } - } -} diff --git a/projects/llm_framework/main_yolo/src/runner/base/transform.hpp b/projects/llm_framework/main_yolo/src/runner/base/transform.hpp deleted file mode 100644 index edee13a7..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/transform.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -namespace transform -{ - static void nhwc2nchw(const float* input, float* output, int h, int w, int c) - { - int output_index = 0; - for (int i = 0; i < c; ++i) - { - for (int j = 0; j < h; ++j) - { - for (int k = 0; k < w; ++k) - { - int input_index = j * w * c + k * c + i; - output[output_index++] = input[input_index]; - } - } - } - } - -} // namespace transform diff --git a/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp b/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp deleted file mode 100644 index 023cd545..00000000 --- a/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp +++ /dev/null @@ -1,578 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace yolo -{ - enum - { - YOLOV3 = 0, - YOLOV3_TINY = 1, - YOLOV4 = 2, - YOLOV4_TINY = 3, - YOLO_FASTEST = 4, - YOLO_FASTEST_XL = 5, - YOLO_FASTEST_BODY = 6, - YOLOV4_TINY_3L = 7 - }; - - struct BBoxRect - { - float score; - float xmin; - float ymin; - float xmax; - float ymax; - float area; - int label; - }; - - static inline float sigmoid(float x) - { - return (float)(1.f / (1.f + std::exp(-x))); - } - static inline float intersection_area(const BBoxRect& a, const BBoxRect& b) - { - if (a.xmin > b.xmax || a.xmax < b.xmin || a.ymin > b.ymax || a.ymax < b.ymin) - { - // no intersection - return 0.f; - } - - float inter_width = std::min(a.xmax, b.xmax) - std::max(a.xmin, b.xmin); - float inter_height = std::min(a.ymax, b.ymax) - std::max(a.ymin, b.ymin); - - return inter_width * inter_height; - } - static void qsort_descent_inplace(std::vector& datas, int left, int right) - { - int i = left; - int j = right; - float p = datas[(left + right) / 2].score; - - while (i <= j) - { - while (datas[i].score > p) - i++; - - while (datas[j].score < p) - j--; - - if (i <= j) - { - // swap - std::swap(datas[i], datas[j]); - - i++; - j--; - } - } - - if (left < j) - qsort_descent_inplace(datas, left, j); - - if (i < right) - qsort_descent_inplace(datas, i, right); - } - - static void qsort_descent_inplace(std::vector& datas) - { - if (datas.empty()) - return; - - qsort_descent_inplace(datas, 0, (int)(datas.size() - 1)); - } - - static void nms_sorted_bboxes(std::vector& bboxes, std::vector& picked, float nms_threshold) - { - picked.clear(); - - const size_t n = bboxes.size(); - - for (size_t i = 0; i < n; i++) - { - const BBoxRect& a = bboxes[i]; - - int keep = 1; - for (unsigned int j : picked) - { - const BBoxRect& b = bboxes[j]; - - // intersection over union - float inter_area = intersection_area(a, b); - float union_area = a.area + b.area - inter_area; - // float IoU = inter_area / union_area - if (inter_area > nms_threshold * union_area) - { - keep = 0; - break; - } - } - - if (keep) - picked.push_back(i); - } - } - - struct TMat - { - operator const float*() const - { - return (const float*)data; - } - - float* row(int row) const - { - return (float*)data + w * row; - } - - TMat channel_range(int start, int chn_num) const - { - TMat mat = {0}; - - mat.batch = 1; - mat.c = chn_num; - mat.h = h; - mat.w = w; - mat.data = (float*)data + start * h * w; - - return mat; - } - - TMat channel(int channel) const - { - return channel_range(channel, 1); - } - - int batch, c, h, w; - void* data; - }; - - class YoloDetectionOutput - { - public: - int init(int version, float nms_threshold = 0.45f, float confidence_threshold = 0.48f, int class_num = 80); - int forward(const std::vector& bottom_blobs, std::vector& top_blobs); - int forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs); - - private: - int m_num_box; - int m_num_class; - int m_anchors_scale[32]; - float m_biases[32]; - int m_mask[32]; - float m_confidence_threshold; - float m_confidence_threshold_unsigmoid; - float m_nms_threshold; - }; - - int YoloDetectionOutput::init(int version, float nms_threshold, float confidence_threshold, int class_num) - { - memset(this, 0, sizeof(*this)); - m_num_box = 3; - m_num_class = class_num; - fprintf(stderr, "YoloDetectionOutput init param[%d]\n", version); - - if (version == YOLOV3) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - float bias[] = {10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - else if (version == YOLOV3_TINY || version == YOLOV4_TINY) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLOV4) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - float bias[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - else if (version == YOLO_FASTEST || version == YOLO_FASTEST_XL) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {12, 18, 37, 49, 52, 132, 115, 73, 119, 199, 242, 238}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLO_FASTEST_BODY) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {7, 17, 20, 50, 45, 99, 64, 187, 123, 211, 227, 264}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLOV4_TINY_3L) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - //float bias_official[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; - float bias[] = {10, 14, 23, 27, 37, 58, 36, 75, 76, 55, 72, 146, 81, 82, 135, 169, 344, 319}; - - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - - m_confidence_threshold = confidence_threshold; - m_nms_threshold = nms_threshold; - m_confidence_threshold_unsigmoid = -1.0f * (float)std::log((1.0f / m_confidence_threshold) - 1.0f); - - return 0; - } - - int YoloDetectionOutput::forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs) - { - // gather all box - std::vector all_bbox_rects; - for (size_t b = 0; b < bottom_blobs.size(); b++) - { - const TMat& bottom_top_blobs = bottom_blobs[b]; - - int w = bottom_top_blobs.w; - int h = bottom_top_blobs.h; - size_t mask_offset = b * m_num_box; - int net_w = (int)(m_anchors_scale[b] * w); - int net_h = (int)(m_anchors_scale[b] * h); - - auto feature_ptr = (float*)bottom_top_blobs.data; - for (int i = 0; i < h; i++) - { - for (int j = 0; j < w; j++) - { - for (int box = 0; box < m_num_box; ++box) - { - if (feature_ptr[4] < m_confidence_threshold_unsigmoid) - { - feature_ptr += (m_num_class + 5); - continue; - } - - int class_index = 0; - float class_score = -FLT_MAX; - for (int k = 5; k < m_num_class + 5; ++k) - { - if (class_score < feature_ptr[k]) - { - class_score = feature_ptr[k]; - class_index = k - 5; - } - } - - //sigmoid(box_score) * sigmoid(class_score) - float confidence_1 = 1.0f / ((1.f + std::exp(-feature_ptr[4])) * (1.f + std::exp(-class_score))); - if (confidence_1 >= m_confidence_threshold) - { - int biases_index = (int)(m_mask[box + mask_offset]); - const float bias_w = m_biases[biases_index * 2]; - const float bias_h = m_biases[biases_index * 2 + 1]; - - // region box - // fprintf(stderr, "%f %f %d \n", class_score, feature_ptr[4], class_index); - float bbox_cx = ((float)j + sigmoid(feature_ptr[0])) / (float)w; - float bbox_cy = ((float)i + sigmoid(feature_ptr[1])) / (float)h; - auto bbox_w = (float)(std::exp(feature_ptr[2]) * bias_w / (float)net_w); - auto bbox_h = (float)(std::exp(feature_ptr[3]) * bias_h / (float)net_h); - - float bbox_xmin = bbox_cx - bbox_w * 0.5f; - float bbox_ymin = bbox_cy - bbox_h * 0.5f; - float bbox_xmax = bbox_cx + bbox_w * 0.5f; - float bbox_ymax = bbox_cy + bbox_h * 0.5f; - - float area = bbox_w * bbox_h; - - BBoxRect c = {confidence_1, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; - all_bbox_rects.push_back(c); - } - - feature_ptr += (m_num_class + 5); - } - } - } - } - - // global sort inplace - qsort_descent_inplace(all_bbox_rects); - - // apply nms - std::vector picked; - nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); - - // select - std::vector bbox_rects; - - for (unsigned int z : picked) - { - bbox_rects.push_back(all_bbox_rects[z]); - } - - // fill result - int num_detected = (int)(bbox_rects.size()); - if (num_detected == 0) - { - top_blobs[0].h = 0; - return 0; - } - - TMat& top_blob = top_blobs[0]; - - for (int i = 0; i < num_detected; i++) - { - const BBoxRect& r = bbox_rects[i]; - float score = r.score; - float* outptr = top_blob.row(i); - - outptr[0] = (float)r.label; // +1 for prepend background class - outptr[1] = score; - outptr[2] = r.xmin; - outptr[3] = r.ymin; - outptr[4] = r.xmax; - outptr[5] = r.ymax; - } - top_blob.h = num_detected; - return 0; - } - - int YoloDetectionOutput::forward(const std::vector& bottom_blobs, std::vector& top_blobs) - { - // gather all box - std::vector all_bbox_rects; - - for (size_t b = 0; b < bottom_blobs.size(); b++) - { - std::vector > all_box_bbox_rects; - all_box_bbox_rects.resize(m_num_box); - const TMat& bottom_top_blobs = bottom_blobs[b]; - - int w = bottom_top_blobs.w; - int h = bottom_top_blobs.h; - int channels = bottom_top_blobs.c; - //printf("%d %d %d\n", w, h, channels); - const int channels_per_box = channels / m_num_box; - - // anchor coord + box score + num_class - if (channels_per_box != 4 + 1 + m_num_class) - return -1; - size_t mask_offset = b * m_num_box; - int net_w = (int)(m_anchors_scale[b] * w); - int net_h = (int)(m_anchors_scale[b] * h); - //printf("%d %d\n", net_w, net_h); - - //printf("%d %d %d\n", w, h, channels); - for (int pp = 0; pp < m_num_box; pp++) - { - int p = pp * channels_per_box; - int biases_index = (int)(m_mask[pp + mask_offset]); - //printf("%d\n", biases_index); - const float bias_w = m_biases[biases_index * 2]; - const float bias_h = m_biases[biases_index * 2 + 1]; - //printf("%f %f\n", bias_w, bias_h); - const float* xptr = bottom_top_blobs.channel(p); - const float* yptr = bottom_top_blobs.channel(p + 1); - const float* wptr = bottom_top_blobs.channel(p + 2); - const float* hptr = bottom_top_blobs.channel(p + 3); - - const float* box_score_ptr = bottom_top_blobs.channel(p + 4); - - // softmax class scores - TMat scores = bottom_top_blobs.channel_range(p + 5, m_num_class); - //softmax->forward_inplace(scores, opt); - - for (int i = 0; i < h; i++) - { - for (int j = 0; j < w; j++) - { - // find class index with max class score - int class_index = 0; - float class_score = -FLT_MAX; - for (int q = 0; q < m_num_class; q++) - { - float score = scores.channel(q).row(i)[j]; - if (score > class_score) - { - class_index = q; - class_score = score; - } - } - - //sigmoid(box_score) * sigmoid(class_score) - float confidence = (float)1.f / ((1.f + std::exp(-box_score_ptr[0]) * (1.f + std::exp(-class_score)))); - if (confidence >= m_confidence_threshold) - { - // fprintf(stderr, "%f %d \n", class_score, class_index); - // region box - float bbox_cx = ((float)j + sigmoid(xptr[0])) / (float)w; - float bbox_cy = ((float)i + sigmoid(yptr[0])) / (float)h; - auto bbox_w = (float)(std::exp(wptr[0]) * bias_w / (float)net_w); - auto bbox_h = (float)(std::exp(hptr[0]) * bias_h / (float)net_h); - - float bbox_xmin = bbox_cx - bbox_w * 0.5f; - float bbox_ymin = bbox_cy - bbox_h * 0.5f; - float bbox_xmax = bbox_cx + bbox_w * 0.5f; - float bbox_ymax = bbox_cy + bbox_h * 0.5f; - - float area = bbox_w * bbox_h; - - BBoxRect c = {confidence, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; - all_box_bbox_rects[pp].push_back(c); - } - - xptr++; - yptr++; - wptr++; - hptr++; - - box_score_ptr++; - } - } - } - - for (int i = 0; i < m_num_box; i++) - { - const std::vector& box_bbox_rects = all_box_bbox_rects[i]; - - all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end()); - } - } - - // global sort inplace - qsort_descent_inplace(all_bbox_rects); - - // apply nms - std::vector picked; - nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); - - // select - std::vector bbox_rects; - - for (unsigned int z : picked) - { - bbox_rects.push_back(all_bbox_rects[z]); - } - - // fill result - int num_detected = (int)(bbox_rects.size()); - if (num_detected == 0) - { - top_blobs[0].h = 0; - return 0; - } - - TMat& top_blob = top_blobs[0]; - - for (int i = 0; i < num_detected; i++) - { - const BBoxRect& r = bbox_rects[i]; - float score = r.score; - float* outptr = top_blob.row(i); - - outptr[0] = (float)r.label; // +1 for prepend background class - outptr[1] = score; - outptr[2] = r.xmin; - outptr[3] = r.ymin; - outptr[4] = r.xmax; - outptr[5] = r.ymax; - } - top_blob.h = num_detected; - - return 0; - } - -} // namespace yolo diff --git a/projects/llm_framework/main_yolo/src/runner/utils/checker.h b/projects/llm_framework/main_yolo/src/runner/utils/checker.h deleted file mode 100644 index 9b7e288e..00000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/checker.h +++ /dev/null @@ -1,32 +0,0 @@ -/************************************************************************************************** - * - * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. - * - * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and - * may not be copied or distributed in any isomorphic form without the prior - * written consent of Axera Semiconductor (Ningbo) Co., Ltd. - * - **************************************************************************************************/ - -#ifndef AX_CHECKER_H -#define AX_CHECKER_H - -#include "utils/logger.h" - -#define CHECK_PTR(p) \ - do { \ - if (!p) { \ - ALOGE("%s nil pointer\n", #p); \ - return -1; \ - } \ - } while (0) - -#define CHECK_INITED(p) \ - do { \ - if (!p->HasInit()) { \ - ALOGE("%s has not init\n", #p); \ - return -1; \ - } \ - } while (0) - -#endif //AX_CHECKER_H diff --git a/projects/llm_framework/main_yolo/src/runner/utils/io.hpp b/projects/llm_framework/main_yolo/src/runner/utils/io.hpp deleted file mode 100644 index 64db991a..00000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/io.hpp +++ /dev/null @@ -1,601 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/checker.h" -#include "ax_sys_api.h" -#include "ax_engine_type.h" - - -#define IO_CMM_ALIGN_SIZE 128 - -namespace utils { - typedef enum { - IO_BUFFER_STRATEGY_DEFAULT, - IO_BUFFER_STRATEGY_CACHED - } IO_BUFFER_STRATEGY_T; - - static inline AX_S32 query_model_input_size(const AX_ENGINE_IO_INFO_T* io_info, std::array &input_size, - AX_IMG_FORMAT_E &eDtype) { - int height = 0; - int width = 0; - int size = 0; - int channel = 0; - int data_type_size = 0; - auto& input = io_info->pInputs[0]; - - switch (input.eLayout) { - case AX_ENGINE_TENSOR_LAYOUT_NHWC: - height = input.pShape[1]; - width = input.pShape[2]; - channel = input.pShape[3]; - size = input.nSize; - break; - case AX_ENGINE_TENSOR_LAYOUT_NCHW: - channel = input.pShape[1]; - height = input.pShape[2]; - width = input.pShape[3]; - size = input.nSize; - break; - default: // NHWC - height = input.pShape[1]; - width = input.pShape[2]; - channel = input.pShape[3]; - size = input.nSize; - break; - } - - switch (input.eDataType) { - case AX_ENGINE_DT_UINT8: - case AX_ENGINE_DT_SINT8: - data_type_size = 1; - break; - case AX_ENGINE_DT_UINT16: - case AX_ENGINE_DT_SINT16: - data_type_size = 2; - break; - case AX_ENGINE_DT_FLOAT32: - data_type_size = 4; - break; - case AX_ENGINE_DT_SINT32: - case AX_ENGINE_DT_UINT32: - data_type_size = 4; - break; - case AX_ENGINE_DT_FLOAT64: - data_type_size = 8; - break; - default: - data_type_size = 1; - break; - } - - if (channel == 0 || height == 0 || width == 0 || size == 0 || data_type_size == 0) { - return -1; - } - - if (input.pExtraMeta) { - switch (input.pExtraMeta->eColorSpace) { - case AX_ENGINE_CS_BGR: - input_size[0] = height; - input_size[1] = width; - eDtype = AX_FORMAT_BGR888; - break; - case AX_ENGINE_CS_RGB: - input_size[0] = height; - input_size[1] = width; - eDtype = AX_FORMAT_RGB888; - break; - case AX_ENGINE_CS_NV12: - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - break; - case AX_ENGINE_CS_NV21: - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR_VU; - break; - default: // AX_ENGINE_CS_NV12 - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - break; - } - } - else { - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - } - - ALOGD("eLayout:%d, eDataType:%d, channel:%d, height:%d, width:%d, size:%d, data_type_size:%d", - input.eLayout, input.eDataType, channel, input_size[0], input_size[1], size, data_type_size); - - return 0; - } - - static inline void brief_io_info(std::string strModel, const AX_ENGINE_IO_INFO_T* io_info) { - auto describe_shape_type = [](AX_ENGINE_TENSOR_LAYOUT_T type) -> const char* { - switch (type) { - case AX_ENGINE_TENSOR_LAYOUT_NHWC: - return "NHWC"; - case AX_ENGINE_TENSOR_LAYOUT_NCHW: - return "NCHW"; - default: - return "unknown"; - } - }; - auto describe_data_type = [](AX_ENGINE_DATA_TYPE_T type) -> const char* { - switch (type) { - case AX_ENGINE_DT_UINT8: - return "uint8"; - case AX_ENGINE_DT_UINT16: - return "uint16"; - case AX_ENGINE_DT_FLOAT32: - return "float32"; - case AX_ENGINE_DT_SINT16: - return "sint16"; - case AX_ENGINE_DT_SINT8: - return "sint8"; - case AX_ENGINE_DT_SINT32: - return "sint32"; - case AX_ENGINE_DT_UINT32: - return "uint32"; - case AX_ENGINE_DT_FLOAT64: - return "float64"; - case AX_ENGINE_DT_UINT10_PACKED: - return "uint10_packed"; - case AX_ENGINE_DT_UINT12_PACKED: - return "uint12_packed"; - case AX_ENGINE_DT_UINT14_PACKED: - return "uint14_packed"; - case AX_ENGINE_DT_UINT16_PACKED: - return "uint16_packed"; - default: - return "unknown"; - } - }; - auto describe_memory_type = [](AX_ENGINE_MEMORY_TYPE_T type) -> const char* { - switch (type) { - case AX_ENGINE_MT_PHYSICAL: - return "Physical"; - case AX_ENGINE_MT_VIRTUAL: - return "Virtual"; - default: - return "unknown"; - } - }; - auto describe_color_space = [](AX_ENGINE_COLOR_SPACE_T cs) -> const char* { - switch (cs) { - case AX_ENGINE_CS_FEATUREMAP: - return "FeatureMap"; - case AX_ENGINE_CS_BGR: - return "BGR"; - case AX_ENGINE_CS_RGB: - return "RGB"; - case AX_ENGINE_CS_RGBA: - return "RGBA"; - case AX_ENGINE_CS_GRAY: - return "GRAY"; - case AX_ENGINE_CS_NV12: - return "NV12"; - case AX_ENGINE_CS_NV21: - return "NV21"; - case AX_ENGINE_CS_YUV444: - return "YUV444"; - case AX_ENGINE_CS_RAW8: - return "RAW8"; - case AX_ENGINE_CS_RAW10: - return "RAW10"; - case AX_ENGINE_CS_RAW12: - return "RAW12"; - case AX_ENGINE_CS_RAW14: - return "RAW14"; - case AX_ENGINE_CS_RAW16: - return "RAW16"; - default: - return "unknown"; - } - }; - printf("Model Name: %s\n", strModel.c_str()); - printf("Max Batch Size %d\n", io_info->nMaxBatchSize); - printf("Support Dynamic Batch? %s\n", io_info->bDynamicBatchSize == AX_TRUE ? "Yes" : "No"); - - for (uint32_t i = 0; i < io_info->nInputSize; ++i) { - auto& input = io_info->pInputs[i]; - printf("Input[%d]: %s\n", i, input.pName); - printf(" Shape ["); - for (uint32_t j = 0; j < input.nShapeSize; ++j) { - printf("%d", (int)input.pShape[j]); - if (j + 1 < input.nShapeSize) printf(", "); - } - printf("] %s %s %s %s\n", describe_shape_type(input.eLayout), describe_data_type(input.eDataType), - input.pExtraMeta ? describe_color_space(input.pExtraMeta->eColorSpace) : "", - input.nQuantizationValue > 0 ? ("Q=" + std::to_string(input.nQuantizationValue)).c_str() : ""); - printf(" Memory %s\n", describe_memory_type(input.eMemoryType)); - printf(" Size %u\n", input.nSize); - } - for (uint32_t i = 0; i < io_info->nOutputSize; ++i) { - auto& output = io_info->pOutputs[i]; - printf("Output[%d]: %s\n", i, output.pName); - printf(" Shape ["); - for (uint32_t j = 0; j < output.nShapeSize; ++j) { - printf("%d", (int)output.pShape[j]); - if (j + 1 < output.nShapeSize) printf(", "); - } - printf("] %s %s %s\n", describe_shape_type(output.eLayout), describe_data_type(output.eDataType), - output.nQuantizationValue > 0 ? ("Q=" + std::to_string(output.nQuantizationValue)).c_str() : ""); - printf(" Memory %s\n", describe_memory_type(output.eMemoryType)); - printf(" Size %u\n", output.nSize); - } - } - - static inline AX_S32 alloc_engine_buffer(const std::string& token, const std::string& appendix, size_t index, const AX_ENGINE_IOMETA_T* pMeta, AX_ENGINE_IO_BUFFER_T* pBuf, IO_BUFFER_STRATEGY_T eStrategy = IO_BUFFER_STRATEGY_DEFAULT) { - AX_S32 ret = -1; - if (eStrategy != IO_BUFFER_STRATEGY_DEFAULT && eStrategy != IO_BUFFER_STRATEGY_CACHED) { - fprintf(stderr, "strategy %d not supported\n", (int)eStrategy); - return -1; - } - memset(pBuf, 0, sizeof(AX_ENGINE_IO_BUFFER_T)); - pBuf->nSize = pMeta->nSize; - - const std::string token_name = "skel_" + token + appendix + std::to_string(index); - - if (eStrategy == IO_BUFFER_STRATEGY_CACHED) { - ret = AX_SYS_MemAllocCached((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); - } - else { - ret = AX_SYS_MemAlloc((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); - } - - return ret; - } - - static inline AX_S32 free_engine_buffer(AX_ENGINE_IO_BUFFER_T* pBuf) { - if (pBuf->phyAddr == 0) { - delete[] reinterpret_cast(pBuf->pVirAddr); - } - else { - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - pBuf->phyAddr = 0; - pBuf->pVirAddr = nullptr; - - return 0; - } - - static inline void free_io_index(AX_ENGINE_IO_BUFFER_T* io_buf, size_t index) { - AX_ENGINE_IO_BUFFER_T* pBuf = io_buf + index; - free_engine_buffer(pBuf); - } - - static inline void free_io(AX_ENGINE_IO_T &io) { - for (size_t j = 0; j < io.nInputSize; ++j) - { - AX_ENGINE_IO_BUFFER_T *pBuf = io.pInputs + j; - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - for (size_t j = 0; j < io.nOutputSize; ++j) - { - AX_ENGINE_IO_BUFFER_T *pBuf = io.pOutputs + j; - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - delete[] io.pInputs; - delete[] io.pOutputs; - } - - static inline void free_io(AX_ENGINE_IO_T &io, std::vector> &vecOutputBuffer) { - if (io.pInputs) { - delete[] io.pInputs; - io.pInputs = nullptr; - } - - if (io.pOutputs) { - for (size_t index = 0; index < vecOutputBuffer.size(); ++index) { - AX_ENGINE_IO_BUFFER_T *pOutputs = &vecOutputBuffer[index][0]; - for (size_t j = 0; j < io.nOutputSize; ++j) { - free_io_index(pOutputs, j); - } - } - - delete[] io.pOutputs; - io.pOutputs = nullptr; - } - } - - static inline int prepare_io(const std::string& token, const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, IO_BUFFER_STRATEGY_T strategy) { - auto ret = 0; - - memset(&io, 0, sizeof(io)); - - io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; - - if (!io.pInputs) { - goto EXIT; - } - - memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); - io.nInputSize = info->nInputSize; - for (AX_U32 i = 0; i < info->nInputSize; ++i) - { - auto meta = info->pInputs[i]; - auto buffer = &io.pInputs[i]; - ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); - if (ret != 0) - { - free_io_index(io.pInputs, i); - return ret; - } - } - - io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; - - if (!io.pOutputs) { - goto EXIT; - } - - memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); - io.nOutputSize = info->nOutputSize; - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto meta = info->pOutputs[i]; - auto buffer = &io.pOutputs[i]; - ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); - if (ret != 0) { - goto EXIT; - } - } - - EXIT: - if (ret != 0) { - free_io(io); - return -1; - } - - return 0; - } - - static inline int prepare_io(const std::string& token, - const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, - std::vector &vecOutputBuffer, - const IO_BUFFER_STRATEGY_T &strategy) { - AX_S32 ret = 0; - memset(&io, 0, sizeof(io)); - - std::vector outputBuffer; - - if (1 != info->nInputSize) { - fprintf(stderr, "[ERR]: Only single input was accepted(got %u).\n", info->nInputSize); - return -1; - } - - io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; - - if (!io.pInputs) { - goto EXIT; - } - - memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); - io.nInputSize = info->nInputSize; - - for (AX_U32 i = 0; i < info->nInputSize; ++i) - { - auto meta = info->pInputs[i]; - auto buffer = &io.pInputs[i]; - ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); - if (ret != 0) - { - free_io_index(io.pInputs, i); - return ret; - } - } - - io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; - - if (!io.pOutputs) { - goto EXIT; - } - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto meta = info->pOutputs[i]; - auto buffer = &io.pOutputs[i]; - ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); - - if (ret != 0) { - goto EXIT; - } - - vecOutputBuffer.push_back(*buffer); - } - - memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); - io.nOutputSize = info->nOutputSize; - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto buffer = &io.pOutputs[i]; - *buffer = vecOutputBuffer[i]; - } - - EXIT: - if (ret != 0) { - free_io(io); - return -1; - } - - return 0; - } - - static inline AX_S32 push_io_output(const AX_ENGINE_IO_INFO_T* info, - AX_ENGINE_IO_T& io, - std::vector &outputBuffer) { - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto buffer = &io.pOutputs[i]; - *buffer = outputBuffer[i]; - } - - return 0; - } - - static inline AX_S32 push_io_input(void* input, int index, AX_ENGINE_IO_T& io) { - // img ranks_depth ranks_feat ranks_bev, n_points - AX_ENGINE_IO_BUFFER_T* pImg = &io.pInputs[index]; - memcpy(pImg->pVirAddr, input, pImg->nSize); - return 0; - } - - static inline AX_S32 cache_io_flush(const AX_ENGINE_IO_BUFFER_T *io_buf) { - if (io_buf->phyAddr != 0) { - AX_SYS_MflushCache(io_buf->phyAddr, io_buf->pVirAddr, io_buf->nSize); - } - - return 0; - } - - static inline AX_S32 push_io_output(void* output, - int index, - AX_ENGINE_IO_T& io) { - AX_ENGINE_IO_BUFFER_T* pImg = &io.pOutputs[index]; - cache_io_flush(pImg); - memcpy(output, pImg->pVirAddr, pImg->nSize); - return 0; - } - - static inline AX_S32 cpu_copy(AX_U64 nPhyAddrSrc, AX_U64 nPhyAddrDst, AX_U32 nLen) { - if (nPhyAddrSrc != 0 && nPhyAddrDst != 0 && nLen > 0) { - AX_VOID* pSrcVirAddr = AX_SYS_MmapCache(nPhyAddrSrc, nLen); - AX_VOID* pDstVirAddr = AX_SYS_MmapCache(nPhyAddrDst, nLen); - - memcpy((AX_VOID*)pDstVirAddr, (AX_VOID*)pSrcVirAddr, nLen); - - AX_SYS_Munmap(pSrcVirAddr, nLen); - AX_SYS_Munmap(pDstVirAddr, nLen); - - return 0; - } - - return -1; - } - - static inline AX_S32 inc_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { - if (stFrame.u32BlkId[0] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[0]); - } - if (stFrame.u32BlkId[1] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[1]); - } - if (stFrame.u32BlkId[2] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[2]); - } - - return 0; - } - - static inline AX_S32 dec_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { - if (stFrame.u32BlkId[0] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[0]); - } - if (stFrame.u32BlkId[1] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[1]); - } - if (stFrame.u32BlkId[2] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[2]); - } - - return 0; - } - - static inline bool read_file(const char* path, std::vector& data) { - std::fstream fs(path, std::ios::in | std::ios::binary); - - if (!fs.is_open()) { - return false; - } - - fs.seekg(std::ios::end); - auto fs_end = fs.tellg(); - fs.seekg(std::ios::beg); - auto fs_beg = fs.tellg(); - - auto file_size = static_cast(fs_end - fs_beg); - auto vector_size = data.size(); - - data.reserve(vector_size + file_size); - data.insert(data.end(), std::istreambuf_iterator(fs), std::istreambuf_iterator()); - - fs.close(); - - return true; - } - - static inline bool read_file(const char* path, AX_VOID **pModelBufferVirAddr, - AX_U64 &u64ModelBufferPhyAddr, AX_U32 &nModelBufferSize) { - std::fstream fs(path, std::ios::in | std::ios::binary); - - if (!fs.is_open()) { - return false; - } - - fs.seekg(0, std::ios::end); - int file_size = fs.tellg(); - fs.seekg(0, std::ios::beg); - - nModelBufferSize = (AX_U32)file_size; - - AX_SYS_MemAlloc(&u64ModelBufferPhyAddr, pModelBufferVirAddr, nModelBufferSize, 0x100, (AX_S8 *)"SKEL-CV"); - - if (!pModelBufferVirAddr || (u64ModelBufferPhyAddr == 0)) { - return false; - } - - fs.read((AX_CHAR *)*pModelBufferVirAddr, nModelBufferSize); - - fs.close(); - - return true; - } - - static inline void dequant(float** pptrOutput, const AX_ENGINE_IOMETA_T& ptrIoInfo, const AX_ENGINE_IO_BUFFER_T& ioBuf, float zp, float scale) - { - if (ptrIoInfo.eDataType == AX_ENGINE_DT_FLOAT32) - { - *pptrOutput = (float*)ioBuf.pVirAddr; - return; - } - - *pptrOutput = (float*)malloc(ptrIoInfo.nSize * sizeof(float)); - uint8_t *pBuf = (uint8_t*)ioBuf.pVirAddr; - float* pOutput = *pptrOutput; - // float inv_scale = 1.0f / scale; - for (int i = 0; i < ptrIoInfo.nSize; i++) - { - pOutput[i] = ((float)pBuf[i] - zp) * scale; - } - } -} - diff --git a/projects/llm_framework/main_yolo/src/runner/utils/logger.h b/projects/llm_framework/main_yolo/src/runner/utils/logger.h deleted file mode 100644 index 83a2e691..00000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/logger.h +++ /dev/null @@ -1,82 +0,0 @@ -/************************************************************************************************** - * - * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. - * - * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and - * may not be copied or distributed in any isomorphic form without the prior - * written consent of Axera Semiconductor (Ningbo) Co., Ltd. - * - **************************************************************************************************/ - -#ifndef SKEL_LOGGER_H -#define SKEL_LOGGER_H - -#include "ax_global_type.h" -#include "ax_sys_log.h" - -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - -//#define SKEL_LOG_TAG "SKEL" -// -//#define ALOGE(fmt, ...) AX_LOG_ERR_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGW(fmt, ...) AX_LOG_WARN_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGI(fmt, ...) AX_LOG_INFO_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGD(fmt, ...) AX_LOG_DBG_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGN(fmt, ...) AX_LOG_NOTICE_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) - - -typedef enum { - SKEL_LOG_MIN = -1, - SKEL_LOG_EMERGENCY = 0, - SKEL_LOG_ALERT = 1, - SKEL_LOG_CRITICAL = 2, - SKEL_LOG_ERROR = 3, - SKEL_LOG_WARN = 4, - SKEL_LOG_NOTICE = 5, - SKEL_LOG_INFO = 6, - SKEL_LOG_DEBUG = 7, - SKEL_LOG_MAX -} SKEL_LOG_LEVEL_E; - -static SKEL_LOG_LEVEL_E log_level = SKEL_LOG_DEBUG; - -#if 1 -#define MACRO_BLACK "\033[1;30;30m" -#define MACRO_RED "\033[1;30;31m" -#define MACRO_GREEN "\033[1;30;32m" -#define MACRO_YELLOW "\033[1;30;33m" -#define MACRO_BLUE "\033[1;30;34m" -#define MACRO_PURPLE "\033[1;30;35m" -#define MACRO_WHITE "\033[1;30;37m" -#define MACRO_END "\033[0m" -#else -#define MACRO_BLACK -#define MACRO_RED -#define MACRO_GREEN -#define MACRO_YELLOW -#define MACRO_BLUE -#define MACRO_PURPLE -#define MACRO_WHITE -#define MACRO_END -#endif - -#define ALOGE(fmt, ...) printf(MACRO_RED "[E][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGW(fmt, ...) if (log_level >= SKEL_LOG_WARN) \ - printf(MACRO_YELLOW "[W][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGI(fmt, ...) if (log_level >= SKEL_LOG_INFO) \ - printf(MACRO_GREEN "[I][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGD(fmt, ...) if (log_level >= SKEL_LOG_DEBUG) \ - printf(MACRO_WHITE "[D][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGN(fmt, ...) if (log_level >= SKEL_LOG_NOTICE) \ - printf(MACRO_PURPLE "[N][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) - -#ifdef __cplusplus -} -#endif - -#endif //SKEL_LOGGER_H diff --git a/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp b/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp deleted file mode 100644 index e753192c..00000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include - -class timer -{ -private: - std::chrono::system_clock::time_point start_time, end_time; - -public: - timer() - { - start(); - } - - void start() - { - stop(); - this->start_time = this->end_time; - } - - void stop() - { -#ifdef _MSC_VER - this->end_time = std::chrono::system_clock::now(); -#else - this->end_time = std::chrono::high_resolution_clock::now(); -#endif - } - - float cost() - { - if (this->end_time <= this->start_time) - { - this->stop(); - } - - auto ms = std::chrono::duration_cast(this->end_time - this->start_time).count(); - return static_cast(ms) / 1000.f; - } -}; From 011a9614390d1644fc338d9e32e0d7716f95955f Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Fri, 22 Nov 2024 17:02:22 +0800 Subject: [PATCH 13/44] [update] yolo output class_name. --- projects/llm_framework/main_yolo/SConstruct | 2 + projects/llm_framework/main_yolo/src/main.cpp | 4 +- projects/llm_framework/main_yolo/yolo11s.json | 88 ++++++++++++++++++- 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 7685db17..3a42ea03 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -27,6 +27,8 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 +STATIC_FILES += [AFile('yolo11s.json')] + env['COMPONENTS'].append({'target':'llm_yolo', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index ef6fd1c8..71f98ebe 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -26,6 +26,7 @@ static std::string base_model_config_path_; typedef struct { std::string yolo_model; + std::vector cls_name; int img_h = 640; int img_w = 640; int cls_num = 80; @@ -110,6 +111,7 @@ class llm_task { CONFIG_AUTO_SET(file_body["mode_param"], img_w); CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], cls_name); mode_config_.yolo_model = base_model + mode_config_.yolo_model; yolo_ = std::make_unique(); if (0 != yolo_->Init(mode_config_.yolo_model.c_str())) { @@ -157,7 +159,7 @@ class llm_task { for (size_t i = 0; i < objects.size(); i++) { const detection::Object &obj = objects[i]; nlohmann::json output; - output["class"] = obj.label; + output["class"] = mode_config_.cls_name[obj.label]; output["confidence"] = format_float(obj.prob * 100, 2); output["bbox"] = nlohmann::json::array(); output["bbox"].push_back(format_float(obj.rect.x, 0)); diff --git a/projects/llm_framework/main_yolo/yolo11s.json b/projects/llm_framework/main_yolo/yolo11s.json index 742976b2..f29b0775 100644 --- a/projects/llm_framework/main_yolo/yolo11s.json +++ b/projects/llm_framework/main_yolo/yolo11s.json @@ -5,13 +5,95 @@ "object detect" ], "input_type":[ - "cv.jpeg.base64" + "yolo.jpeg.base64" ], "output_type":[ - "cv.yolobox" + "yolo.yolobox" ], "mode_param":{ - "cv_model":"yolo11s.axmodel" + "yolo_model":"yolo11s.axmodel", + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] }, "mode_param_bak":{ "model_config.img_h":640, From ee21a2355d32d9534e1135252134b88c8282db91 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 22 Nov 2024 17:04:32 +0800 Subject: [PATCH 14/44] [update] add error msg --- .../StackFlow/stackflow/StackFlowUtil.cpp | 24 +++---- projects/llm_framework/main/src/main.cpp | 25 ++++++- projects/llm_framework/main_asr/src/main.cpp | 19 +++++- projects/llm_framework/main_kws/src/main.cpp | 19 +++++- projects/llm_framework/main_llm/src/main.cpp | 19 +++++- projects/llm_framework/main_vlm/src/main.cpp | 19 +++++- projects/llm_framework/main_yolo/src/main.cpp | 68 +++++++++++++------ 7 files changed, 153 insertions(+), 40 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index f50b9229..eff09360 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -182,22 +182,18 @@ std::string StackFlows::sample_unescapeString(const std::string &input) bool StackFlows::decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff) { - try { - int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); - std::string finish = StackFlows::sample_json_str_get(in, "finish"); - std::string delta = StackFlows::sample_json_str_get(in, "delta"); - stream_buff[index] = delta; - if (finish == "true") { - for (size_t i = 0; i < stream_buff.size(); i++) { - out += stream_buff.at(i); - } - stream_buff.clear(); - return false; - } else if (finish != "false") { - throw true; + int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); + std::string finish = StackFlows::sample_json_str_get(in, "finish"); + std::string delta = StackFlows::sample_json_str_get(in, "delta"); + stream_buff[index] = delta; + if (finish == "true") { + for (size_t i = 0; i < stream_buff.size(); i++) { + out += stream_buff.at(i); } - } catch (...) { stream_buff.clear(); + return false; + } else if (finish != "false") { + throw true; } return true; } diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 67c6fa3e..0fc5700d 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -118,9 +118,19 @@ class llm_llm : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); return; } const std::string *next_data = &data; @@ -128,13 +138,26 @@ class llm_llm : public StackFlow { std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 27152b7a..64eb5564 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -349,9 +349,13 @@ class llm_asr : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); return; } std::string tmp_msg1; @@ -359,13 +363,26 @@ class llm_asr : public StackFlow { int ret; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index e2ac7165..90b6dcca 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -379,9 +379,13 @@ class llm_kws : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); return; } std::string tmp_msg1; @@ -389,13 +393,26 @@ class llm_kws : public StackFlow { int ret; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index f040d67a..3b57ff88 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -287,9 +287,13 @@ class llm_llm : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); return; } const std::string *next_data = &data; @@ -297,13 +301,26 @@ class llm_llm : public StackFlow { std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index cd487f17..ea8ad222 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -297,9 +297,13 @@ class llm_llm : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); return; } const std::string *next_data = &data; @@ -307,13 +311,26 @@ class llm_llm : public StackFlow { std::string tmp_msg1; if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index ef6fd1c8..7710d751 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -135,12 +135,11 @@ class llm_task { out_callback_ = out_callback; } - void inference(const std::string &msg) + bool inference(const std::string &msg) { try { - if ((image_data_.empty())) return; - cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR); - if (src.empty()) return; + cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); + if (src.empty()) return true; std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); @@ -170,7 +169,9 @@ class llm_task { if (out_callback_) out_callback_(yolo_output, true); } catch (...) { SLOGW("yolo_->Run have error!"); + return true; } + return false; } void _ax_init() @@ -274,34 +275,54 @@ class llm_yolo : public StackFlow { const std::weak_ptr llm_channel_weak, const std::string &object, const std::string &data) { + nlohmann::json error_body; auto llm_task_obj = llm_task_obj_weak.lock(); auto llm_channel = llm_channel_weak.lock(); if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); return; } - if (data.empty() || (data == "None")) return; - nlohmann::json error_body; const std::string *next_data = &data; - bool enbase64 = (object.find("base64") == std::string::npos) ? false : true; bool enstream = (object.find("stream") == std::string::npos) ? false : true; int ret; std::string tmp_msg1; if (enstream) { static std::unordered_map stream_buff; - if (decode_stream(data, tmp_msg1, stream_buff)) return; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } next_data = &tmp_msg1; } + // must encode base64 std::string tmp_msg2; - if (enbase64) { - ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { - return; - } - next_data = &tmp_msg2; + ret = decode_base64((*next_data), tmp_msg2); + if (!ret) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); + return; } - if (object.find("jpeg") != std::string::npos) { - llm_task_obj->image_data_.assign(next_data->begin(), next_data->end()); - llm_task_obj->inference((*next_data)); + next_data = &tmp_msg2; + if (llm_task_obj->inference(*next_data)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); } } @@ -343,11 +364,14 @@ class llm_yolo : public StackFlow { "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + } else if (input.find("sys") != std::string::npos) { + // TODO:... } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; } - llm_task_[work_id_num] = llm_task_obj; - SLOGI("load_mode success"); - send("None", "None", LLM_NO_ERROR, work_id); return 0; } else { SLOGE("load_mode Failed"); @@ -374,10 +398,12 @@ class llm_yolo : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("yolo") != std::string::npos) { ret = llm_channel->subscriber_work_id( - data, + "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); + } else if (data.find("sys") != std::string::npos) { + // TODO:... } if (ret) { error_body["code"] = -20; From 06194f053193804c5f383422029316d49d6f4f8e Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Mon, 25 Nov 2024 11:29:25 +0800 Subject: [PATCH 15/44] [update] SDK --- SDK | 2 +- ext_components/ax_msp/SConstruct | 40 +---------------------- projects/llm_framework/SConstruct | 53 +++++-------------------------- 3 files changed, 10 insertions(+), 85 deletions(-) diff --git a/SDK b/SDK index c767769d..7b3db295 160000 --- a/SDK +++ b/SDK @@ -1 +1 @@ -Subproject commit c767769d28ec511b613a24fc8680ec3acd4c769a +Subproject commit 7b3db295a4e0ca96bbfbea8ba2ab873c23fbd3a8 diff --git a/ext_components/ax_msp/SConstruct b/ext_components/ax_msp/SConstruct index a49c7d75..82b6590f 100644 --- a/ext_components/ax_msp/SConstruct +++ b/ext_components/ax_msp/SConstruct @@ -5,45 +5,7 @@ with open(env['PROJECT_TOOL_S']) as f: exec(f.read()) if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: - MSP_PATH = os.path.join(os.environ['GIT_REPO_PATH'], 'm5stack_msp') - if 'MSP_PATH' in os.environ: - MSP_PATH = os.environ['MSP_PATH'] - else: - if not os.path.exists(MSP_PATH): - zip_file = MSP_PATH + '.tar.gz' - zip_file_extrpath = MSP_PATH + '_tmp' - down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_msp.tar.gz" - - if 'CONFIG_REPO_AUTOMATION' in os.environ: - down = 'y' - else: - down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('m5stack_msp.tar.gz')) - down = down.lower() - if down == 'y': - # from git import Repo - import requests - import parse - import tarfile - import shutil - try: - # Downloading via HTTP (more common) - if not os.path.exists(zip_file): - response = requests.get(down_url) - if response.status_code == 200: - with open(zip_file, 'wb') as file: - file.write(response.content) - else: - env.Fatal("{} down failed".format(down_url)) - with tarfile.open(zip_file, 'r:gz') as tar: - tar.extractall(path=zip_file_extrpath) - shutil.move(zip_file_extrpath, MSP_PATH) - # shutil.rmtree(zip_file_extrpath) - print("The {} download successful.".format(down_url)) - except Exception as e: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - env.Fatal("Cloning failed.: {}".format(e)) - else: - env.Fatal('Please manually download {} to {} .'.format(down_url, zip_file)) + MSP_PATH = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_msp.tar.gz", 'm5stack_msp.tar.gz') SRCS=[] INCLUDE=[] PRIVATE_INCLUDE=[] diff --git a/projects/llm_framework/SConstruct b/projects/llm_framework/SConstruct index af1dbcdc..4c0f64e2 100644 --- a/projects/llm_framework/SConstruct +++ b/projects/llm_framework/SConstruct @@ -8,6 +8,10 @@ os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/ version = 'v0.0.5' static_lib = 'static_lib' update = False + +with open(str(Path(os.getcwd())/'..'/'..'/'SDK'/'tools'/'scons'/'project.py')) as f: + exec(f.read()) + if not os.path.exists(static_lib): update = True else: @@ -17,50 +21,9 @@ else: update = True except: update = True - if update: - zip_file = static_lib + '_{}.tar.gz'.format(version) - zip_file_extrpath = static_lib + '_tmp' + with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/static_lib_{}.tar.gz".format(version) - - if 'CONFIG_REPO_AUTOMATION' in os.environ: - down = 'y' - else: - down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('static_lib_{}.tar.gz').format(version)) - down = down.lower() - if down == 'y': - # from git import Repo - import requests - import parse - import tarfile - import shutil - try: - # Downloading via HTTP (more common) - if not os.path.exists(zip_file): - response = requests.get(down_url) - if response.status_code == 200: - with open(zip_file, 'wb') as file: - file.write(response.content) - else: - print("{} down failed".format(down_url)) - try: - shutil.rmtree(static_lib) - except: - pass - with tarfile.open(zip_file, 'r:gz') as tar: - tar.extractall(path=zip_file_extrpath) - shutil.move(zip_file_extrpath, static_lib) - try: - os.remove(zip_file) - except: - pass - print("The {} download successful.".format(down_url)) - except Exception as e: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - print("Cloning failed.: {}".format(e)) - else: - print('Please manually download {} to {} .'.format(down_url, zip_file)) - - -with open(str(Path(os.getcwd())/'..'/'..'/'SDK'/'tools'/'scons'/'project.py')) as f: - exec(f.read()) + down_path = check_wget_down(down_url, "static_lib_{}.tar.gz".format(version)) + shutil.move(down_path, static_lib) From 93b69a8661d7011935a324a936d8d3ba2391f970 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 26 Nov 2024 09:03:26 +0800 Subject: [PATCH 16/44] [update] supports seg & pose model. --- projects/llm_framework/main_yolo/SConstruct | 2 +- .../main_yolo/src/EngineWrapper.cpp | 109 ++++++++++++++++-- .../main_yolo/src/EngineWrapper.hpp | 2 +- projects/llm_framework/main_yolo/src/main.cpp | 9 +- .../llm_framework/main_yolo/yolo11s-pose.json | 106 +++++++++++++++++ .../llm_framework/main_yolo/yolo11s-seg.json | 106 +++++++++++++++++ projects/llm_framework/main_yolo/yolo11s.json | 11 +- 7 files changed, 325 insertions(+), 20 deletions(-) create mode 100644 projects/llm_framework/main_yolo/yolo11s-pose.json create mode 100644 projects/llm_framework/main_yolo/yolo11s-seg.json diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 3a42ea03..1870067e 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -27,7 +27,7 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 -STATIC_FILES += [AFile('yolo11s.json')] +STATIC_FILES += [AFile('yolo11s.json'), AFile('yolo11s-seg.json'), AFile('yolo11s-pose.json')] env['COMPONENTS'].append({'target':'llm_yolo', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp index 12bbb949..286c6aeb 100644 --- a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -266,27 +266,112 @@ const char* CLASS_NAMES[] = { "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; +const char* OBB_CLASS_NAMES[] = {"plane", + "ship", + "storage tank", + "baseball diamond", + "tennis court", + "basketball court", + "ground track field", + "harbor", + "bridge", + "large vehicle", + "small vehicle", + "helicopter", + "roundabout", + "soccer ball field", + "swimming pool"}; + +static const std::vector> COCO_COLORS = { + {56, 0, 255}, {226, 255, 0}, {0, 94, 255}, {0, 37, 255}, {0, 255, 94}, {255, 226, 0}, {0, 18, 255}, + {255, 151, 0}, {170, 0, 255}, {0, 255, 56}, {255, 0, 75}, {0, 75, 255}, {0, 255, 169}, {255, 0, 207}, + {75, 255, 0}, {207, 0, 255}, {37, 0, 255}, {0, 207, 255}, {94, 0, 255}, {0, 255, 113}, {255, 18, 0}, + {255, 0, 56}, {18, 0, 255}, {0, 255, 226}, {170, 255, 0}, {255, 0, 245}, {151, 255, 0}, {132, 255, 0}, + {75, 0, 255}, {151, 0, 255}, {0, 151, 255}, {132, 0, 255}, {0, 255, 245}, {255, 132, 0}, {226, 0, 255}, + {255, 37, 0}, {207, 255, 0}, {0, 255, 207}, {94, 255, 0}, {0, 226, 255}, {56, 255, 0}, {255, 94, 0}, + {255, 113, 0}, {0, 132, 255}, {255, 0, 132}, {255, 170, 0}, {255, 0, 188}, {113, 255, 0}, {245, 0, 255}, + {113, 0, 255}, {255, 188, 0}, {0, 113, 255}, {255, 0, 0}, {0, 56, 255}, {255, 0, 113}, {0, 255, 188}, + {255, 0, 94}, {255, 0, 18}, {18, 255, 0}, {0, 255, 132}, {0, 188, 255}, {0, 245, 255}, {0, 169, 255}, + {37, 255, 0}, {255, 0, 151}, {188, 0, 255}, {0, 255, 37}, {0, 255, 0}, {255, 0, 170}, {255, 0, 37}, + {255, 75, 0}, {0, 0, 255}, {255, 207, 0}, {255, 0, 226}, {255, 245, 0}, {188, 255, 0}, {0, 255, 18}, + {0, 255, 75}, {0, 255, 151}, {255, 56, 0}, {245, 255, 0}}; + +static const std::vector> KPS_COLORS = { + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {255, 128, 0}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {51, 153, 255}, + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}}; + +static const std::vector> LIMB_COLORS = { + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {255, 51, 255}, {255, 51, 255}, {255, 51, 255}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {0, 255, 0}, {0, 255, 0}, + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}}; + +static const std::vector> SKELETON = { + {16, 14}, {14, 12}, {17, 15}, {15, 13}, {12, 13}, {6, 12}, {7, 13}, {6, 7}, {6, 8}, {7, 9}, + {8, 10}, {9, 11}, {2, 3}, {1, 2}, {1, 3}, {2, 4}, {3, 5}, {4, 6}, {5, 7}}; + void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, - int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) + int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects, + std::string& model_type) { // std::vector objects; std::vector proposals; - for (int i = 0; i < 3; ++i) { - auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; - int32_t stride = (1 << i) * 8; - detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, - cls_num); + if (model_type == "detect") { + for (int i = 0; i < 3; ++i) { + auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, + cls_num); + } + detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); + } else if (model_type == "segment") { + float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, + (float*)io_data->pOutputs[2].pVirAddr}; + float* output_seg_ptr[3] = {(float*)io_data->pOutputs[3].pVirAddr, (float*)io_data->pOutputs[4].pVirAddr, + (float*)io_data->pOutputs[5].pVirAddr}; + for (int i = 0; i < 3; ++i) { + auto feat_ptr = output_ptr[i]; + auto feat_seg_ptr = output_seg_ptr[i]; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_seg_native(stride, feat_ptr, feat_seg_ptr, prob_threshold, proposals, + input_w, input_h, cls_num); + } + auto mask_proto_ptr = (float*)io_data->pOutputs[6].pVirAddr; + detection::get_out_bbox_mask(proposals, objects, mask_proto_ptr, 32, 4, nms_threshold, input_h, input_w, + mat.rows, mat.cols); + detection::draw_objects_mask(mat, objects, CLASS_NAMES, COCO_COLORS, "yolo11_seg_out"); + } else if (model_type == "pose") { + float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, + (float*)io_data->pOutputs[2].pVirAddr}; + float* output_kps_ptr[3] = {(float*)io_data->pOutputs[3].pVirAddr, (float*)io_data->pOutputs[4].pVirAddr, + (float*)io_data->pOutputs[5].pVirAddr}; + + for (int i = 0; i < 3; ++i) { + auto feat_ptr = output_ptr[i]; + auto feat_kps_ptr = output_kps_ptr[i]; + int32_t stride = (1 << i) * 8; + detection::generate_proposals_yolov8_pose_native(stride, feat_ptr, feat_kps_ptr, prob_threshold, proposals, + input_h, input_w, 17, cls_num); + } + detection::get_out_bbox_kps(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + detection::draw_keypoints(mat, objects, KPS_COLORS, LIMB_COLORS, SKELETON, "yolo11_pose_out"); + } else if (model_type == "obb") { + std::vector strides = {8, 16, 32}; + std::vector grid_strides; + detection::generate_grids_and_stride(input_w, input_h, strides, grid_strides); + auto feat_ptr = (float*)io_data->pOutputs[0].pVirAddr; + detection::obb::generate_proposals_yolov8_obb_native(grid_strides, feat_ptr, prob_threshold, proposals, input_w, + input_h, cls_num); + detection::obb::get_out_obb_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); + detection::obb::draw_objects_obb(mat, objects, OBB_CLASS_NAMES, "yolo11_obb_out", 1); } - detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); - fprintf(stdout, "detection num: %zu\n", objects.size()); - - detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); } int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, - float& nms_threshold, std::vector& objects) + float& nms_threshold, std::vector& objects, std::string& model_type) { - post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects); + post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects, model_type); return 0; } diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.hpp b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp index 4c018f4d..3a6dfa07 100644 --- a/projects/llm_framework/main_yolo/src/EngineWrapper.hpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp @@ -50,7 +50,7 @@ class EngineWrapper { int RunSync(); int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, - std::vector& objects); + std::vector& objects, std::string& model_type); int GetOutput(void* pOutput, int index); diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 71f98ebe..7c06c4d8 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -26,6 +26,7 @@ static std::string base_model_config_path_; typedef struct { std::string yolo_model; + std::string model_type = "detect"; std::vector cls_name; int img_h = 640; int img_w = 640; @@ -112,6 +113,8 @@ class llm_task { CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); CONFIG_AUTO_SET(file_body["mode_param"], cls_name); + CONFIG_AUTO_SET(file_body["mode_param"], cls_num); + CONFIG_AUTO_SET(file_body["mode_param"], model_type); mode_config_.yolo_model = base_model + mode_config_.yolo_model; yolo_ = std::make_unique(); if (0 != yolo_->Init(mode_config_.yolo_model.c_str())) { @@ -154,7 +157,8 @@ class llm_task { } std::vector objects; yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, - mode_config_.pron_threshold, mode_config_.nms_threshold, objects); + mode_config_.pron_threshold, mode_config_.nms_threshold, objects, + mode_config_.model_type); std::vector yolo_output; for (size_t i = 0; i < objects.size(); i++) { const detection::Object &obj = objects[i]; @@ -166,6 +170,9 @@ class llm_task { output["bbox"].push_back(format_float(obj.rect.y, 0)); output["bbox"].push_back(format_float(obj.rect.x + obj.rect.width, 0)); output["bbox"].push_back(format_float(obj.rect.y + obj.rect.height, 0)); + if (mode_config_.model_type == "segment") output["mask"] = obj.mask_feat; + if (mode_config_.model_type == "pose") output["kps"] = obj.kps_feat; + if (mode_config_.model_type == "obb") output["angle"] = obj.angle; yolo_output.push_back(output); if (out_callback_) out_callback_(yolo_output, false); } diff --git a/projects/llm_framework/main_yolo/yolo11s-pose.json b/projects/llm_framework/main_yolo/yolo11s-pose.json new file mode 100644 index 00000000..7b28eaab --- /dev/null +++ b/projects/llm_framework/main_yolo/yolo11s-pose.json @@ -0,0 +1,106 @@ +{ + "mode":"yolo11s", + "type":"cv", + "capabilities":[ + "object detect" + ], + "input_type":[ + "cv.jpeg.base64" + ], + "output_type":[ + "cv.yolobox" + ], + "mode_param":{ + "yolo_model":"yolo11s-pose.axmodel", + "model_type":"pose", + "img_h":640, + "img_w":640, + "cls_num":1, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/yolo11s-seg.json b/projects/llm_framework/main_yolo/yolo11s-seg.json new file mode 100644 index 00000000..62d89801 --- /dev/null +++ b/projects/llm_framework/main_yolo/yolo11s-seg.json @@ -0,0 +1,106 @@ +{ + "mode":"yolo11s", + "type":"cv", + "capabilities":[ + "segment" + ], + "input_type":[ + "cv.jpeg.base64" + ], + "output_type":[ + "cv.yolobox" + ], + "mode_param":{ + "yolo_model":"yolo11s-seg.axmodel", + "model_type":"segment", + "img_h":640, + "img_w":640, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/yolo11s.json b/projects/llm_framework/main_yolo/yolo11s.json index f29b0775..a58db697 100644 --- a/projects/llm_framework/main_yolo/yolo11s.json +++ b/projects/llm_framework/main_yolo/yolo11s.json @@ -12,6 +12,12 @@ ], "mode_param":{ "yolo_model":"yolo11s.axmodel", + "model_type":"detect", + "img_h":640, + "img_w":640, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, "cls_name":[ "person", "bicycle", @@ -96,10 +102,5 @@ ] }, "mode_param_bak":{ - "model_config.img_h":640, - "model_config.img_w":640, - "model_config.cls_num":80, - "model_config.pron_threshold":0.45, - "model_config.nms_threshold":0.45 } } \ No newline at end of file From ebdcfe9b7ec5e1e572a75515ac9a3de1f9c0f1b6 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 26 Nov 2024 17:08:13 +0800 Subject: [PATCH 17/44] [super update] - pzmq add ctx - add neon accelerate the calculation. - add simdjson - change compile flage -O2 --- SDK | 2 +- ext_components/StackFlow/SConstruct | 2 +- .../StackFlow/stackflow/StackFlow.cpp | 87 ++++++++---- .../StackFlow/stackflow/StackFlow.h | 22 ++-- .../StackFlow/stackflow/StackFlowUtil.cpp | 33 +++-- ext_components/StackFlow/stackflow/pzmq.hpp | 124 +++++++++++++++--- projects/llm_framework/config_defaults.mk | 3 +- projects/llm_framework/main/SConstruct | 2 +- projects/llm_framework/main_asr/SConstruct | 2 + projects/llm_framework/main_asr/src/main.cpp | 16 +-- projects/llm_framework/main_audio/SConstruct | 2 +- .../llm_framework/main_audio/src/main.cpp | 56 ++++---- projects/llm_framework/main_kws/SConstruct | 2 +- projects/llm_framework/main_kws/src/main.cpp | 12 +- projects/llm_framework/main_llm/SConstruct | 2 +- .../llm_framework/main_melotts/SConstruct | 2 +- projects/llm_framework/main_sys/SConstruct | 4 +- .../llm_framework/main_sys/include/zmq_bus.h | 4 +- .../llm_framework/main_sys/src/event_loop.cpp | 73 +++++++---- .../main_sys/src/remote_action.cpp | 2 +- .../main_sys/src/remote_server.cpp | 25 ++-- .../llm_framework/main_sys/src/serial_com.cpp | 33 ++--- .../llm_framework/main_sys/src/tcp_com.cpp | 33 ++--- .../llm_framework/main_sys/src/zmq_bus.cpp | 52 +++++++- projects/llm_framework/main_tts/SConstruct | 2 +- projects/llm_framework/main_vlm/SConstruct | 2 +- projects/llm_framework/main_yolo/SConstruct | 2 +- 27 files changed, 393 insertions(+), 208 deletions(-) diff --git a/SDK b/SDK index 7b3db295..23712b73 160000 --- a/SDK +++ b/SDK @@ -1 +1 @@ -Subproject commit 7b3db295a4e0ca96bbfbea8ba2ab873c23fbd3a8 +Subproject commit 23712b73c833f5109f1a3a64e960ed7a244c4315 diff --git a/ext_components/StackFlow/SConstruct b/ext_components/StackFlow/SConstruct index 8007eb97..abac5610 100644 --- a/ext_components/StackFlow/SConstruct +++ b/ext_components/StackFlow/SConstruct @@ -20,7 +20,7 @@ if 'CONFIG_STACKFLOW_ENABLED' in os.environ: INCLUDE.append(ADir("stackflow")) PRIVATE_INCLUDE.append(ADir("stackflow/libzmq")) - REQUIREMENTS += ['eventpp', 'utilities', 'zmq'] + REQUIREMENTS += ['eventpp', 'utilities', 'zmq', 'simdjson_component'] env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), 'SRCS':SRCS, diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index 7fbc6da3..25aca68d 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -5,6 +5,7 @@ */ #include "StackFlow.h" #include "sample_log.h" +#include using namespace StackFlows; @@ -21,7 +22,7 @@ llm_channel_obj::llm_channel_obj(const std::string &_publisher_url, const std::s : unit_name_(unit_name), inference_url_(inference_url) { zmq_url_index_ = -1000; - zmq_[-1] = std::make_unique(_publisher_url, ZMQ_PUB); + zmq_[-1] = std::make_shared(_publisher_url, ZMQ_PUB); zmq_[-2].reset(); } @@ -30,15 +31,36 @@ llm_channel_obj::~llm_channel_obj() } void llm_channel_obj::subscriber_event_call(const std::function &call, - const std::string &raw) + pzmq *_pzmq, const std::string &raw) { - if (sample_json_str_get(raw, "action") == "inference") { - std::string zmq_com = sample_json_str_get(raw, "zmq_com"); - if (!zmq_com.empty()) set_push_url(zmq_com); - request_id_ = sample_json_str_get(raw, "request_id"); - work_id_ = sample_json_str_get(raw, "work_id"); + try { + simdjson::padded_string json_string(raw); + simdjson::ondemand::document doc; + auto parser = _pzmq->getContextPtr(); + auto error = parser->iterate(json_string).get(doc); + if (error) { + return; + } + std::string action; + error = doc["action"].get_string(action); + if (action == "inference") { + std::string zmq_com; + error = doc["zmq_com"].get_string(zmq_com); + if (!zmq_com.empty()) set_push_url(zmq_com); + error = doc["request_id"].get_string(request_id_); + error = doc["work_id"].get_string(work_id_); + } + std::string object; + error = doc["object"].get_string(object); + auto result = doc["data"].raw_json(); + if (result.error() == simdjson::SUCCESS) { + call(object, result.value().data()); + } else { + std::cerr << "result: " << result.value() << "error mesg:" << result.error() << std::endl; + } + } catch (simdjson::simdjson_error &e) { + std::cerr << "Error: " << simdjson::error_message(e.error()) << std::endl; } - call(sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } int llm_channel_obj::subscriber_work_id(const std::string &work_id, @@ -63,8 +85,10 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, id_num = 0; subscriber_url = inference_url_; } - zmq_[id_num] = std::make_unique( - subscriber_url, ZMQ_SUB, std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1)); + zmq_[id_num] = std::make_shared( + subscriber_url, ZMQ_SUB, + std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1, std::placeholders::_2)); + zmq_[id_num]->newContextPtr(); return 0; } @@ -84,10 +108,10 @@ void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) if (zmq_.find(id_num) != zmq_.end()) zmq_.erase(id_num); } -void llm_channel_obj::subscriber(const std::string &zmq_url, const std::function &call) +void llm_channel_obj::subscriber(const std::string &zmq_url, const pzmq::msg_callback_fun &call) { zmq_url_map_[zmq_url] = zmq_url_index_--; - zmq_[zmq_url_map_[zmq_url]] = std::make_unique(zmq_url, ZMQ_SUB, call); + zmq_[zmq_url_map_[zmq_url]] = std::make_shared(zmq_url, ZMQ_SUB, call); } void llm_channel_obj::stop_subscriber(const std::string &zmq_url) @@ -162,13 +186,20 @@ StackFlow::StackFlow::StackFlow(const std::string &unit_name) std::bind(&StackFlow::_sys_init, this, std::placeholders::_1, std::placeholders::_2)); event_queue_.appendListener( EVENT_REPEAT_EVENT, std::bind(&StackFlow::_repeat_loop, this, std::placeholders::_1, std::placeholders::_2)); - rpc_ctx_->register_rpc_action("setup", std::bind(&StackFlow::_rpc_setup, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("pause", std::bind(&StackFlow::_rpc_pause, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("work", std::bind(&StackFlow::_rpc_work, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("exit", std::bind(&StackFlow::_rpc_exit, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("link", std::bind(&StackFlow::_rpc_link, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("unlink", std::bind(&StackFlow::_rpc_unlink, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("taskinfo", std::bind(&StackFlow::_rpc_taskinfo, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action( + "setup", std::bind(&StackFlow::_rpc_setup, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "pause", std::bind(&StackFlow::_rpc_pause, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("work", + std::bind(&StackFlow::_rpc_work, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("exit", + std::bind(&StackFlow::_rpc_exit, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("link", + std::bind(&StackFlow::_rpc_link, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "unlink", std::bind(&StackFlow::_rpc_unlink, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "taskinfo", std::bind(&StackFlow::_rpc_taskinfo, this, std::placeholders::_1, std::placeholders::_2)); status_.store(0); exit_flage_.store(false); @@ -219,7 +250,7 @@ void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) } } -std::string StackFlow::_rpc_setup(const std::string &data) +std::string StackFlow::_rpc_setup(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_SETUP, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -253,7 +284,7 @@ int StackFlow::setup(const std::string &work_id, const std::string &object, cons return -1; } -std::string StackFlow::_rpc_link(const std::string &data) +std::string StackFlow::_rpc_link(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_LINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -284,7 +315,7 @@ void StackFlow::link(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_unlink(const std::string &data) +std::string StackFlow::_rpc_unlink(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_UNLINK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -315,7 +346,7 @@ void StackFlow::unlink(const std::string &work_id, const std::string &object, co send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_work(const std::string &data) +std::string StackFlow::_rpc_work(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_WORK, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -346,7 +377,7 @@ void StackFlow::work(const std::string &work_id, const std::string &object, cons send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_exit(const std::string &data) +std::string StackFlow::_rpc_exit(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_EXIT, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -380,7 +411,7 @@ int StackFlow::exit(const std::string &work_id, const std::string &object, const return 0; } -std::string StackFlow::_rpc_pause(const std::string &data) +std::string StackFlow::_rpc_pause(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_PAUSE, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -411,7 +442,7 @@ void StackFlow::pause(const std::string &work_id, const std::string &object, con send("None", "None", error_body, work_id); } -std::string StackFlow::_rpc_taskinfo(const std::string &data) +std::string StackFlow::_rpc_taskinfo(pzmq *_pzmq, const std::string &data) { event_queue_.enqueue(EVENT_TASKINFO, RPC_PARSE_TO_PARAM(data)); return std::string("None"); @@ -445,8 +476,8 @@ void StackFlow::taskinfo(const std::string &work_id, const std::string &object, int StackFlow::sys_register_unit(const std::string &unit_name) { int work_id_number; - std::string component_msg = unit_call("sys", "register_unit", unit_name); - std::string str_port = RPC_PARSE_TO_FIRST(component_msg); + std::string component_msg = unit_call("sys", "register_unit", unit_name); + std::string str_port = RPC_PARSE_TO_FIRST(component_msg); work_id_number = std::stoi(str_port); std::string tmp_buf = RPC_PARSE_TO_SECOND(component_msg); std::string out_port = RPC_PARSE_TO_FIRST(tmp_buf); diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index 6f7d397e..30ce62f5 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -64,7 +64,7 @@ class ThreadSafeWrapper { class llm_channel_obj { private: - std::unordered_map> zmq_; + std::unordered_map> zmq_; std::atomic zmq_url_index_; std::unordered_map zmq_url_map_; @@ -97,12 +97,12 @@ class llm_channel_obj { { return enstream_; } - void subscriber_event_call(const std::function &call, + void subscriber_event_call(const std::function &call, pzmq *_pzmq, const std::string &raw); int subscriber_work_id(const std::string &work_id, const std::function &call); void stop_subscriber_work_id(const std::string &work_id); - void subscriber(const std::string &zmq_url, const std::function &call); + void subscriber(const std::string &zmq_url, const pzmq::msg_callback_fun &call); void stop_subscriber(const std::string &zmq_url); int check_zmq_errno(void *ctx, void *com, int code); int send_raw_to_pub(const std::string &raw); @@ -249,7 +249,7 @@ class StackFlow { return llm_task_channel_.at(_work_id_num); } - std::string _rpc_setup(const std::string &data); + std::string _rpc_setup(pzmq *_pzmq, const std::string &data); void _setup(const std::string &zmq_url, const std::string &data) { // printf("void _setup run \n"); @@ -260,7 +260,7 @@ class StackFlow { virtual int setup(const std::string &zmq_url, const std::string &raw); virtual int setup(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_link(const std::string &data); + std::string _rpc_link(pzmq *_pzmq, const std::string &data); void _link(const std::string &zmq_url, const std::string &data) { // printf("void _link run \n"); @@ -271,7 +271,7 @@ class StackFlow { virtual void link(const std::string &zmq_url, const std::string &raw); virtual void link(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_unlink(const std::string &data); + std::string _rpc_unlink(pzmq *_pzmq, const std::string &data); void _unlink(const std::string &zmq_url, const std::string &data) { // printf("void _unlink run \n"); @@ -282,7 +282,7 @@ class StackFlow { virtual void unlink(const std::string &zmq_url, const std::string &raw); virtual void unlink(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_exit(const std::string &data); + std::string _rpc_exit(pzmq *_pzmq, const std::string &data); void _exit(const std::string &zmq_url, const std::string &data) { request_id_ = sample_json_str_get(data, "request_id"); @@ -292,7 +292,7 @@ class StackFlow { virtual int exit(const std::string &zmq_url, const std::string &raw); virtual int exit(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_work(const std::string &data); + std::string _rpc_work(pzmq *_pzmq, const std::string &data); void _work(const std::string &zmq_url, const std::string &data) { request_id_ = sample_json_str_get(data, "request_id"); @@ -302,7 +302,7 @@ class StackFlow { virtual void work(const std::string &zmq_url, const std::string &raw); virtual void work(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_pause(const std::string &data); + std::string _rpc_pause(pzmq *_pzmq, const std::string &data); void _pause(const std::string &zmq_url, const std::string &data) { request_id_ = sample_json_str_get(data, "request_id"); @@ -312,7 +312,7 @@ class StackFlow { virtual void pause(const std::string &zmq_url, const std::string &raw); virtual void pause(const std::string &work_id, const std::string &object, const std::string &data); - std::string _rpc_taskinfo(const std::string &data); + std::string _rpc_taskinfo(pzmq *_pzmq, const std::string &data); void _taskinfo(const std::string &zmq_url, const std::string &data) { request_id_ = sample_json_str_get(data, "request_id"); @@ -372,7 +372,7 @@ class StackFlow { return false; } pzmq _call("sys"); - _call.call_rpc_action("release_unit", _work_id, [](const std::string &data) {}); + _call.call_rpc_action("release_unit", _work_id, [](pzmq *_pzmq, const std::string &data) {}); llm_task_channel_[_work_id_num].reset(); llm_task_channel_.erase(_work_id_num); // SLOGI("release work_id %s success", _work_id.c_str()); diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index eff09360..880fe82d 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -6,6 +6,7 @@ #include "StackFlowUtil.h" #include #include "pzmq.hpp" +#include std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { @@ -182,17 +183,27 @@ std::string StackFlows::sample_unescapeString(const std::string &input) bool StackFlows::decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff) { - int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); - std::string finish = StackFlows::sample_json_str_get(in, "finish"); - std::string delta = StackFlows::sample_json_str_get(in, "delta"); - stream_buff[index] = delta; - if (finish == "true") { - for (size_t i = 0; i < stream_buff.size(); i++) { - out += stream_buff.at(i); + simdjson::ondemand::parser parser; + simdjson::padded_string json_string(in); + simdjson::ondemand::document doc; + auto error = parser.iterate(json_string).get(doc); + if (error) { + throw true; + } + int index = doc["index"].get_int64(); + bool finish = doc["finish"].get_bool(); + auto result = doc["delta"].raw_json(); + if (result.error() == simdjson::SUCCESS) { + stream_buff[index] = std::string(result.value().data()); + if (finish) { + for (size_t i = 0; i < stream_buff.size(); i++) { + out += stream_buff.at(i); + } + stream_buff.clear(); + return false; } - stream_buff.clear(); - return false; - } else if (finish != "false") { + } else { + std::cerr << "result: " << result.value() << "error mesg:" << result.error() << std::endl; throw true; } return true; @@ -402,6 +413,6 @@ std::string StackFlows::unit_call(const std::string &unit_name, const std::strin { std::string value; pzmq _call(unit_name); - _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); + _call.call_rpc_action(unit_action, data, [&value](pzmq *_pzmq, const std::string &raw) { value = raw; }); return value; } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index e4a99539..5023da04 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -16,14 +16,19 @@ #include #define ZMQ_RPC_FUN (ZMQ_REP | 0x80) #define ZMQ_RPC_CALL (ZMQ_REQ | 0x80) + namespace StackFlows { class pzmq { +public: + typedef std::function rpc_callback_fun; + typedef std::function msg_callback_fun; + private: const int rpc_url_head_length = 6; std::string rpc_url_head_ = "ipc:///tmp/rpc."; void *zmq_ctx_; void *zmq_socket_; - std::unordered_map> zmq_fun_; + std::unordered_map zmq_fun_; std::mutex zmq_fun_mtx_; std::atomic flage_; std::unique_ptr zmq_thread_; @@ -51,7 +56,7 @@ class pzmq { rpc_url_head_.clear(); } } - pzmq(const std::string &url, int mode, const std::function &raw_call = nullptr) + pzmq(const std::string &url, int mode, const msg_callback_fun &raw_call = nullptr) : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) { if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call); @@ -77,7 +82,7 @@ class pzmq { } return zmq_url_; } - std::string _rpc_list_action(const std::string &_None) + std::string _rpc_list_action(pzmq *self, const std::string &_None) { std::string action_list; action_list.reserve(128); @@ -95,7 +100,7 @@ class pzmq { } return action_list; } - int register_rpc_action(const std::string &action, const std::function &raw_call) + int register_rpc_action(const std::string &action, const rpc_callback_fun &raw_call) { int ret = 0; std::unique_lock lock(zmq_fun_mtx_); @@ -104,10 +109,11 @@ class pzmq { return ret; } if (zmq_fun_.empty()) { - std::string url = rpc_url_head_ + rpc_server_; - mode_ = ZMQ_RPC_FUN; - zmq_fun_["list_action"] = std::bind(&pzmq::_rpc_list_action, this, std::placeholders::_1); - ret = creat(url); + std::string url = rpc_url_head_ + rpc_server_; + mode_ = ZMQ_RPC_FUN; + zmq_fun_["list_action"] = + std::bind(&pzmq::_rpc_list_action, this, std::placeholders::_1, std::placeholders::_2); + ret = creat(url); } zmq_fun_[action] = raw_call; return ret; @@ -119,8 +125,7 @@ class pzmq { zmq_fun_.erase(action); } } - int call_rpc_action(const std::string &action, const std::string &data, - const std::function &raw_call) + int call_rpc_action(const std::string &action, const std::string &data, const msg_callback_fun &raw_call) { int ret; zmq_msg_t msg; @@ -144,7 +149,7 @@ class pzmq { { zmq_msg_recv(&msg, zmq_socket_, 0); } - raw_call(std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); + raw_call(this, std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); } catch (int e) { ret = e; } @@ -152,7 +157,7 @@ class pzmq { close_zmq(); return ret; } - int creat(const std::string &url, const std::function &raw_call = nullptr) + int creat(const std::string &url, const msg_callback_fun &raw_call = nullptr) { zmq_url_ = url; do { @@ -243,14 +248,14 @@ class pzmq { zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout, sizeof(timeout)); return zmq_connect(zmq_socket_, url.c_str()); } - inline int creat_pull(const std::string &url, const std::function &raw_call) + inline int creat_pull(const std::string &url, const msg_callback_fun &raw_call) { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int subscriber_url(const std::string &url, const std::function &raw_call) + inline int subscriber_url(const std::string &url, const msg_callback_fun &raw_call) { int ret = zmq_connect(zmq_socket_, url.c_str()); zmq_setsockopt(zmq_socket_, ZMQ_SUBSCRIBE, "", 0); @@ -258,7 +263,7 @@ class pzmq { zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); return ret; } - inline int creat_rep(const std::string &url, const std::function &raw_call) + inline int creat_rep(const std::string &url, const msg_callback_fun &raw_call) { int ret = zmq_bind(zmq_socket_, url.c_str()); flage_ = false; @@ -277,7 +282,7 @@ class pzmq { zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout_, sizeof(timeout_)); return zmq_connect(zmq_socket_, url.c_str()); } - void zmq_event_loop(const std::function &raw_call) + void zmq_event_loop(const msg_callback_fun &raw_call) { int ret; zmq_pollitem_t items[1]; @@ -314,14 +319,14 @@ class pzmq { std::string retval; try { std::unique_lock lock(zmq_fun_mtx_); - retval = zmq_fun_.at(raw_data)(_raw_data); + retval = zmq_fun_.at(raw_data)(this, _raw_data); } catch (...) { retval = "NotAction"; } zmq_send(zmq_socket_, retval.c_str(), retval.length(), 0); zmq_msg_close(&msg1); } else { - raw_call(raw_data); + raw_call(this, raw_data); } zmq_msg_close(&msg); } @@ -353,5 +358,88 @@ class pzmq { } close_zmq(); } + +private: + std::shared_ptr context_ptr_; + std::weak_ptr wcontext_ptr_; + void *ctx_; + +public: + // context + void *context() + { + return ctx_; + } + void setContext(void *ctx) + { + ctx_ = ctx; + } + template + T *newContext() + { + ctx_ = new T; + return (T *)ctx_; + } + template + T *getContext() + { + return (T *)ctx_; + } + template + void deleteContext() + { + if (ctx_) { + delete (T *)ctx_; + ctx_ = NULL; + } + } + + // contextPtr + std::shared_ptr contextPtr() + { + return context_ptr_; + } + void setContextPtr(const std::shared_ptr &ctx) + { + context_ptr_ = ctx; + } + void setContextPtr(std::shared_ptr &&ctx) + { + context_ptr_ = std::move(ctx); + } + template + std::shared_ptr newContextPtr() + { + context_ptr_ = std::make_shared(); + return std::static_pointer_cast(context_ptr_); + } + template + std::shared_ptr getContextPtr() + { + return std::static_pointer_cast(context_ptr_); + } + void deleteContextPtr() + { + context_ptr_.reset(); + } + + // wcontextPtr + std::shared_ptr wcontextPtr() + { + return wcontext_ptr_.lock(); + } + void wsetContextPtr(const std::shared_ptr &ctx) + { + wcontext_ptr_ = ctx; + } + template + std::shared_ptr wgetContextPtr() + { + return std::static_pointer_cast(wcontext_ptr_.lock()); + } + void wdeleteContextPtr() + { + wcontext_ptr_.reset(); + } }; }; // namespace StackFlows \ No newline at end of file diff --git a/projects/llm_framework/config_defaults.mk b/projects/llm_framework/config_defaults.mk index 5ea858fb..7076e548 100644 --- a/projects/llm_framework/config_defaults.mk +++ b/projects/llm_framework/config_defaults.mk @@ -13,7 +13,7 @@ CONFIG_DEVICE_UART_ENABLED=y # CONFIG_LHV_WITH_EVPP=y CONFIG_UTILITIES_ENABLED=y CONFIG_BACKWARD_CPP_ENABLED=y -CONFIG_TOOLCHAIN_FLAGS="-g" +CONFIG_TOOLCHAIN_FLAGS="-O2" CONFIG_EVENTPP_ENABLED=y CONFIG_LHV_ENABLED=y CONFIG_LHV_WITH_EVPP=y @@ -22,3 +22,4 @@ CONFIG_STACKFLOW_ENABLED=y CONFIG_UTILITIES_BASE64_ENABLED=y CONFIG_SINGLE_HEADER_LIBS_ENABLED=y CONFIG_AX_SAMPLES_ENABLED=y +CONFIG_SIMDJSON_COMPENENT_ENABLED=y diff --git a/projects/llm_framework/main/SConstruct b/projects/llm_framework/main/SConstruct index 32841632..6e7374c4 100644 --- a/projects/llm_framework/main/SConstruct +++ b/projects/llm_framework/main/SConstruct @@ -26,7 +26,7 @@ STATIC_FILES += [AFile('../static_lib/sherpa/ncnn/libsherpa-ncnn-core.so'), AFile('../static_lib/sherpa/ncnn/libkaldi-native-fbank-core.so'), AFile('../static_lib/libonnxruntime.so.1.14.0') ] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'static_file', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index af59ecb3..dbf03fc3 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -26,6 +26,8 @@ REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] STATIC_FILES += [AFile('sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json'), AFile('sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json')] +REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_asr', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 64eb5564..183f349f 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -179,7 +179,7 @@ class llm_task { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) + void sys_pcm_on_data(pzmq *_pzmq, const std::string &raw) { static int count = 0; if (count < delay_audio_frame_) { @@ -403,7 +403,7 @@ class llm_asr : public StackFlow { } next_data = &tmp_msg4; } - llm_task_obj->sys_pcm_on_data((*next_data)); + llm_task_obj->sys_pcm_on_data(nullptr, (*next_data)); } void _task_pause(const std::string &work_id, const std::string &data) @@ -435,8 +435,8 @@ class llm_asr : public StackFlow { } llm_task_obj->kws_awake(); if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), + std::placeholders::_1, std::placeholders::_2)); llm_task_obj->audio_flage_ = true; } } @@ -522,8 +522,8 @@ class llm_asr : public StackFlow { for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { audio_url_ = unit_call("audio", "cap", input); - llm_channel->subscriber( - audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), + std::placeholders::_1, std::placeholders::_2)); llm_task_obj->audio_flage_ = true; } else if (input.find("asr") != std::string::npos) { llm_channel->subscriber_work_id( @@ -568,8 +568,8 @@ class llm_asr : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("sys") != std::string::npos) { if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data); - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), + std::placeholders::_1, std::placeholders::_2)); llm_task_obj->audio_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 8e6721e6..c8371f81 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -27,7 +27,7 @@ REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_audio', 'ax_audio_3a', 'ax_fdk' REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] STATIC_FILES += [AFile('audio.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_audio', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 141c5313..5dda246e 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -115,16 +115,24 @@ class llm_audio : public StackFlow { setup("", "audio.cap", "{\"None\":\"None\"}"); self = this; cap_status_ = 0; - rpc_ctx_->register_rpc_action("play", std::bind(&llm_audio::play, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("play_raw", std::bind(&llm_audio::play_raw, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("queue_play", std::bind(&llm_audio::enqueue_play, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("play_stop", std::bind(&llm_audio::play_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("queue_play_stop", - std::bind(&llm_audio::queue_play_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("audio_status", std::bind(&llm_audio::audio_status, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap", std::bind(&llm_audio::cap, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap_stop", std::bind(&llm_audio::cap_stop, this, std::placeholders::_1)); - rpc_ctx_->register_rpc_action("cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("play", + std::bind(&llm_audio::play, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "play_raw", std::bind(&llm_audio::play_raw, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "queue_play", std::bind(&llm_audio::enqueue_play, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "play_stop", std::bind(&llm_audio::play_stop, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("queue_play_stop", std::bind(&llm_audio::queue_play_stop, this, + std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "audio_status", std::bind(&llm_audio::audio_status, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("cap", + std::bind(&llm_audio::cap, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "cap_stop", std::bind(&llm_audio::cap_stop, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action( + "cap_stop_all", std::bind(&llm_audio::cap_stop_all, this, std::placeholders::_1, std::placeholders::_2)); } int setup(const std::string &work_id, const std::string &object, const std::string &data) override { @@ -368,32 +376,32 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string play(const std::string &rawdata) + std::string play(pzmq *_pzmq, const std::string &rawdata) { - std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); + std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); std::string audio_json = RPC_PARSE_TO_SECOND(rawdata); - std::string ret_val = parse_data(sample_json_str_get(audio_json, "object"), sample_json_str_get(audio_json, "data")); - request_id_ = sample_json_str_get(audio_json, "request_id"); + std::string ret_val = + parse_data(sample_json_str_get(audio_json, "object"), sample_json_str_get(audio_json, "data")); + request_id_ = sample_json_str_get(audio_json, "request_id"); send(LLM_NONE, LLM_NONE, LLM_NO_ERROR, sample_json_str_get(audio_json, "work_id"), zmq_url); return ret_val; } - std::string play_raw(const std::string &rawdata) + std::string play_raw(pzmq *_pzmq, const std::string &rawdata) { - if(rawdata.empty()) - return std::string("rawdata empty"); + if (rawdata.empty()) return std::string("rawdata empty"); _play(rawdata); return LLM_NONE; } - std::string enqueue_play(const std::string &rawdata) + std::string enqueue_play(pzmq *_pzmq, const std::string &rawdata) { audio_clear_flage_ = false; event_queue_.enqueue(EVENT_QUEUE_PLAY, rawdata, ""); return LLM_NONE; } - std::string audio_status(const std::string &rawdata) + std::string audio_status(pzmq *_pzmq, const std::string &rawdata) { if (rawdata == "play") { if (ax_play_status()) { @@ -426,19 +434,19 @@ class llm_audio : public StackFlow { } } - std::string play_stop(const std::string &rawdata) + std::string play_stop(pzmq *_pzmq, const std::string &rawdata) { _play_stop(); return LLM_NONE; } - std::string queue_play_stop(const std::string &rawdata) + std::string queue_play_stop(pzmq *_pzmq, const std::string &rawdata) { audio_clear_flage_ = true; return LLM_NONE; } - std::string cap(const std::string &rawdata) + std::string cap(pzmq *_pzmq, const std::string &rawdata) { if (cap_status_ == 0) { _cap(); @@ -447,7 +455,7 @@ class llm_audio : public StackFlow { return sys_pcm_cap_channel; } - std::string cap_stop(const std::string &rawdata) + std::string cap_stop(pzmq *_pzmq, const std::string &rawdata) { if (cap_status_ > 0) { cap_status_--; @@ -458,7 +466,7 @@ class llm_audio : public StackFlow { return LLM_NONE; } - std::string cap_stop_all(const std::string &rawdata) + std::string cap_stop_all(pzmq *_pzmq, const std::string &rawdata) { cap_status_ = 0; _cap_stop(); diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 95d3dafa..690cad3d 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -33,7 +33,7 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_kws', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 90b6dcca..8fc3ea28 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -203,7 +203,7 @@ class llm_task { out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) + void sys_pcm_on_data(pzmq *_pzmq, const std::string &raw) { static int count = 0; if (count < delay_audio_frame_) { @@ -337,8 +337,8 @@ class llm_kws : public StackFlow { return; } if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, - std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), + std::placeholders::_1, std::placeholders::_2)); llm_task_obj->audio_flage_ = true; } } @@ -417,7 +417,7 @@ class llm_kws : public StackFlow { } next_data = &tmp_msg2; } - llm_task_obj->sys_pcm_on_data((*next_data)); + llm_task_obj->sys_pcm_on_data(nullptr, (*next_data)); } int setup(const std::string &work_id, const std::string &object, const std::string &data) override @@ -455,8 +455,8 @@ class llm_kws : public StackFlow { for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { audio_url_ = unit_call("audio", "cap", "None"); - llm_channel->subscriber( - audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), std::placeholders::_1)); + llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), + std::placeholders::_1, std::placeholders::_2)); llm_task_obj->audio_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index c95c1458..9b8fc773 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -42,7 +42,7 @@ STATIC_FILES += [AFile('qwen2.5-0.5B-prefill-20e.json'), AFile('openbuddy-llama3.2-1b-ax630c_tokenizer.py'), AFile('qwen2.5-coder-0.5B-ax630c_tokenizer.py') ] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index 6124c3e7..ad89084c 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -27,7 +27,7 @@ INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] STATIC_FILES += [AFile('melotts_zh-cn.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_melotts', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_sys/SConstruct b/projects/llm_framework/main_sys/SConstruct index 22513bcb..0808de47 100644 --- a/projects/llm_framework/main_sys/SConstruct +++ b/projects/llm_framework/main_sys/SConstruct @@ -8,7 +8,7 @@ with open(env['PROJECT_TOOL_S']) as f: SRCS = Glob('src/*.c*') INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] -REQUIREMENTS = ['hv', 'pthread', 'utilities', 'DeviceDriver', 'util', 'StackFlow'] +REQUIREMENTS = ['hv', 'pthread', 'utilities', 'DeviceDriver', 'util', 'StackFlow', 'simdjson_component'] STATIC_LIB = [] DYNAMIC_LIB = [] DEFINITIONS = [] @@ -32,7 +32,7 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] STATIC_FILES += [AFile('sys_config.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_sys', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_sys/include/zmq_bus.h b/projects/llm_framework/main_sys/include/zmq_bus.h index 1542b7f2..ec5360a7 100644 --- a/projects/llm_framework/main_sys/include/zmq_bus.h +++ b/projects/llm_framework/main_sys/include/zmq_bus.h @@ -46,6 +46,8 @@ class zmq_bus_com { int exit_flage; int err_count; int _port; + std::string json_str_; + int json_str_flage_; std::unique_ptr reace_data_event_thread; public: @@ -53,11 +55,11 @@ class zmq_bus_com { zmq_bus_com(); void work(const std::string &zmq_url_format, int port); void stop(); + void select_json_str(const std::string &json_src, std::function out_fun); virtual void on_data(const std::string &data); virtual void send_data(const std::string &data); virtual void reace_data_event(); virtual void send_data_event(); ~zmq_bus_com(); }; - #endif \ No newline at end of file diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 7b7bae8c..6e80bb4d 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -34,6 +34,7 @@ #include "subprocess.h" #include "zmq_bus.h" #include "remote_action.h" +#include void usr_print_error(const std::string &request_id, const std::string &work_id, const std::string &error_msg, int zmq_out) @@ -501,27 +502,42 @@ void server_work() void server_stop_work() { } - +std::mutex unit_action_match_mtx; +simdjson::ondemand::parser parser; typedef int (*sys_fun_call)(int, const nlohmann::json &); void unit_action_match(int com_id, const std::string &json_str) { - int ret; - std::string out_str; - nlohmann::json req_body; - - std::string action; + std::lock_guard guard(unit_action_match_mtx); + simdjson::padded_string json_string(json_str); + simdjson::ondemand::document doc; + auto error = parser.iterate(json_string).get(doc); + if (error) { + SLOGE("json format error:%s", json_str.c_str()); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } std::string request_id; + error = doc["request_id"].get_string(request_id); + if (error) { + SLOGE("miss request_id"); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } std::string work_id; - try { - req_body = nlohmann::json::parse(json_str); - action = req_body["action"]; - request_id = req_body["request_id"]; - work_id = req_body["work_id"]; - } catch (...) { - SLOGE("json format error:%s", json_str.c_str()); + error = doc["work_id"].get_string(work_id); + if (error) { + SLOGE("miss work_id"); usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } + std::string action; + error = doc["action"].get_string(action); + if (error) { + SLOGE("miss action"); + usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); + return; + } + // SLOGI("request_id:%s work_id:%s action:%s", request_id.c_str(), work_id.c_str(), action.c_str()); std::vector work_id_fragment; std::string fragment; for (auto c : work_id) { @@ -533,22 +549,29 @@ void unit_action_match(int com_id, const std::string &json_str) } } if (fragment.length()) work_id_fragment.push_back(fragment); - - if ((work_id_fragment.size() > 0) && (work_id_fragment[0] == "sys")) { + if (action == "inference") { + char zmq_push_url[128]; + int post = sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); + std::string inference_raw_data; + inference_raw_data.resize(post + json_str.length() + 13); + post = sprintf(inference_raw_data.data(), "{\"zmq_com\":\""); + post += sprintf(inference_raw_data.data() + post, "%s", zmq_push_url); + post += sprintf(inference_raw_data.data() + post, "\","); + memcpy(inference_raw_data.data() + post, json_str.data() + 1, json_str.length() - 1); + // post += json_str.length() - 1; + // SLOGI("inference_raw_data:%s size:%d %d", inference_raw_data.c_str(), inference_raw_data.length(), post); + int ret = zmq_bus_publisher_push(work_id, inference_raw_data); + if (ret) { + usr_print_error(request_id, work_id, "{\"code\":-4, \"message\":\"inference data push false\"}", com_id); + } + } else if ((work_id_fragment.size() > 0) && (work_id_fragment[0] == "sys")) { std::string unit_action = "sys." + action; sys_fun_call call_fun = NULL; SAFE_READING(call_fun, sys_fun_call, unit_action); - if (call_fun) - call_fun(com_id, req_body); - else + if (call_fun) { + call_fun(com_id, nlohmann::json::parse(json_str)); + } else { usr_print_error(request_id, work_id, "{\"code\":-3, \"message\":\"action match false\"}", com_id); - } else if (action == "inference") { - char zmq_push_url[128]; - sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); - req_body["zmq_com"] = std::string(zmq_push_url); - int ret = zmq_bus_publisher_push(work_id, req_body.dump()); - if (ret) { - usr_print_error(request_id, work_id, "{\"code\":-4, \"message\":\"inference data push false\"}", com_id); } } else { if ((work_id_fragment[0].length() != 0) && (remote_call(com_id, json_str) != 0)) { diff --git a/projects/llm_framework/main_sys/src/remote_action.cpp b/projects/llm_framework/main_sys/src/remote_action.cpp index 03ff87e6..25933692 100644 --- a/projects/llm_framework/main_sys/src/remote_action.cpp +++ b/projects/llm_framework/main_sys/src/remote_action.cpp @@ -25,7 +25,7 @@ int remote_call(int com_id, const std::string &json_str) std::string com_urls(com_url); RPC_PUSH_PARAM(send_data, com_urls, json_str); pzmq clent(work_unit); - return clent.call_rpc_action(action, send_data, [](const std::string &val) {}); + return clent.call_rpc_action(action, send_data, [](pzmq *_pzmq, const std::string &val) {}); } void remote_action_work() diff --git a/projects/llm_framework/main_sys/src/remote_server.cpp b/projects/llm_framework/main_sys/src/remote_server.cpp index 83335dfe..5a4567f7 100644 --- a/projects/llm_framework/main_sys/src/remote_server.cpp +++ b/projects/llm_framework/main_sys/src/remote_server.cpp @@ -171,7 +171,7 @@ int c_sys_release_unit(char const *unit) return sys_release_unit(unit); } -std::string rpc_allocate_unit(const std::string &raw) +std::string rpc_allocate_unit(pzmq *_pzmq, const std::string &raw) { unit_data *unit_info = sys_allocate_unit(raw); std::string send_data; @@ -182,18 +182,18 @@ std::string rpc_allocate_unit(const std::string &raw) return send_data; } -std::string rpc_release_unit(const std::string &raw) +std::string rpc_release_unit(pzmq *_pzmq, const std::string &raw) { sys_release_unit(raw); return "Success"; } -std::string rpc_sql_select(const std::string &raw) +std::string rpc_sql_select(pzmq *_pzmq, const std::string &raw) { return sys_sql_select(raw); } -std::string rpc_sql_set(const std::string &raw) +std::string rpc_sql_set(pzmq *_pzmq, const std::string &raw) { std::string key = sample_json_str_get(raw, "key"); std::string val = sample_json_str_get(raw, "val"); @@ -202,7 +202,7 @@ std::string rpc_sql_set(const std::string &raw) return "Success"; } -std::string rpc_sql_unset(const std::string &raw) +std::string rpc_sql_unset(pzmq *_pzmq, const std::string &raw) { sys_sql_unset(raw); return "Success"; @@ -217,11 +217,16 @@ void remote_server_work() port_list.resize(port_list_end - port_list_start, 0); sys_rpc_server_ = std::make_unique("sys"); - sys_rpc_server_->register_rpc_action("sql_select", std::bind(rpc_sql_select, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("register_unit", std::bind(rpc_allocate_unit, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("release_unit", std::bind(rpc_release_unit, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("sql_set", std::bind(rpc_sql_set, std::placeholders::_1)); - sys_rpc_server_->register_rpc_action("sql_unset", std::bind(rpc_sql_unset, std::placeholders::_1)); + sys_rpc_server_->register_rpc_action("sql_select", + std::bind(rpc_sql_select, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("register_unit", + std::bind(rpc_allocate_unit, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("release_unit", + std::bind(rpc_release_unit, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("sql_set", + std::bind(rpc_sql_set, std::placeholders::_1, std::placeholders::_2)); + sys_rpc_server_->register_rpc_action("sql_unset", + std::bind(rpc_sql_unset, std::placeholders::_1, std::placeholders::_2)); } void remote_server_stop_work() diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 871d90f5..049490a6 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -41,7 +41,6 @@ class serial_com : public zmq_bus_com { void send_data(const std::string &data) { - SLOGD("serial send:%s", data.c_str()); linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); } @@ -58,29 +57,15 @@ class serial_com : public zmq_bus_com { if ((select(uart_fd + 1, &readfds, NULL, NULL, &timeout) <= 0) || (!FD_ISSET(uart_fd, &readfds))) continue; int len = linux_uart_read(uart_fd, 128, reace_buf); { - char *data = reace_buf; - for (int i = 0; i < len; i++) { - json_str += data[i]; - if (data[i] == '{') flage++; - if (data[i] == '}') flage--; - if (flage == 0) { - if (json_str[0] == '{') { - SLOGD("uart reace:%s", json_str.c_str()); - on_data(json_str); - } - json_str.clear(); - } - if (flage < 0) { - flage = 0; - json_str.clear(); - { - std::string out_str; - out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; - out_str += std::to_string(time(NULL)); - out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; - send_data(out_str); - } - } + try { + select_json_str(std::string(reace_buf, len), + std::bind(&serial_com::on_data, this, std::placeholders::_1)); + } catch (...) { + std::string out_str; + out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; + out_str += std::to_string(time(NULL)); + out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; + send_data(out_str); } } } diff --git a/projects/llm_framework/main_sys/src/tcp_com.cpp b/projects/llm_framework/main_sys/src/tcp_com.cpp index 2ba0f368..238017cb 100644 --- a/projects/llm_framework/main_sys/src/tcp_com.cpp +++ b/projects/llm_framework/main_sys/src/tcp_com.cpp @@ -34,12 +34,9 @@ class tcp_com : public zmq_bus_com { private: public: SocketChannelPtr channel; - std::string json_str; - int flage; std::mutex tcp_server_mutex; tcp_com() : zmq_bus_com() { - flage = 0; } void send_data(const std::string& data) @@ -75,27 +72,15 @@ void onMessage(const SocketChannelPtr& channel, Buffer* buf) char* data = (char*)buf->data(); tcp_com* con_data = (tcp_com*)channel->context(); con_data->tcp_server_mutex.lock(); - for (size_t i = 0; i < len; i++) { - con_data->json_str += data[i]; - if (data[i] == '{') con_data->flage++; - if (data[i] == '}') con_data->flage--; - if (con_data->flage == 0) { - if (con_data->json_str[0] == '{') { - con_data->on_data(con_data->json_str); - } - con_data->json_str.clear(); - } - if (con_data->flage < 0) { - con_data->flage = 0; - con_data->json_str.clear(); - { - std::string out_str; - out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; - out_str += std::to_string(time(NULL)); - out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; - channel->write(out_str); - } - } + try { + con_data->select_json_str(std::string(data, len), + std::bind(&tcp_com::on_data, con_data, std::placeholders::_1)); + } catch (...) { + std::string out_str; + out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; + out_str += std::to_string(time(NULL)); + out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; + channel->write(out_str); } con_data->tcp_server_mutex.unlock(); } diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index 84113a73..ebcdbf5c 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -10,6 +10,9 @@ #include #include #include +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif using namespace StackFlows; @@ -17,8 +20,9 @@ void unit_action_match(int com_id, const std::string &json_str); zmq_bus_com::zmq_bus_com() { - exit_flage = 1; - err_count = 0; + exit_flage = 1; + err_count = 0; + json_str_flage_ = 0; } void zmq_bus_com::work(const std::string &zmq_url_format, int port) @@ -30,8 +34,8 @@ void zmq_bus_com::work(const std::string &zmq_url_format, int port) sprintf((char *)buff.data(), zmq_url_format.c_str(), port); _zmq_url = std::string((char *)buff.data()); SAFE_SETTING("serial_zmq_url", _zmq_url); - user_chennal_ = - std::make_unique(_zmq_url, ZMQ_PULL, std::bind(&zmq_bus_com::send_data, this, std::placeholders::_1)); + user_chennal_ = std::make_unique(_zmq_url, ZMQ_PULL, + [this](pzmq *_pzmq, const std::string &data) { this->send_data(data); }); reace_data_event_thread = std::make_unique(std::bind(&zmq_bus_com::reace_data_event, this)); } @@ -121,4 +125,44 @@ void zmq_bus_work() void zmq_bus_stop_work() { +} + +void zmq_bus_com::select_json_str(const std::string &json_src, std::function out_fun) +{ + json_str_.reserve(json_str_.length() + json_src.length()); + const char *data = json_src.c_str(); + for (int i = 0; i < json_src.length(); i++) { +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + if (json_src.length() - i >= 16) { + uint8x16_t target_open = vdupq_n_u8('{'); + uint8x16_t target_close = vdupq_n_u8('}'); + uint8x16_t input_vector = vld1q_u8((const uint8_t *)&data[i]); + uint8x16_t result_open = vceqq_u8(input_vector, target_open); + uint8x16_t result_close = vceqq_u8(input_vector, target_close); + uint8x16_t result_mask = vorrq_u8(result_open, result_close); + __uint128_t jflage; + vst1q_u8((uint8_t *)&jflage, result_mask); + if (jflage == 0) { + json_str_.append(data + i, 16); + i += 15; + continue; + } + } +#endif + json_str_ += json_src[i]; + int last_index = (i == 0) ? 0 : (i - 1); + if ((data[i] == '{') && (data[last_index] != '\\')) json_str_flage_++; + if ((data[i] == '}') && (data[last_index] != '\\')) json_str_flage_--; + if (json_str_flage_ == 0) { + if (json_str_[0] == '{') { + out_fun(json_str_); + } + json_str_.clear(); + } + if (json_str_flage_ < 0) { + json_str_flage_ = 0; + json_str_.clear(); + throw std::runtime_error("json package error"); + } + } } \ No newline at end of file diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index 0ce3b6bb..e7f38030 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -28,7 +28,7 @@ INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), AD STATIC_FILES += [AFile('single_speaker_english_fast.json'), AFile('single_speaker_fast.json') ] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_tts', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index b1f51c8a..7a5b8746 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -52,7 +52,7 @@ STATIC_LIB += static_file * 4 STATIC_FILES += [AFile('internvl2-1b-ax630c.json'), AFile('internvl2-1b-ax630c_tokenizer.py') ] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_vlm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 3a42ea03..c3de7d67 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -28,7 +28,7 @@ static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 STATIC_FILES += [AFile('yolo11s.json')] - +REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_yolo', 'SRCS':SRCS, 'INCLUDE':INCLUDE, From 1253d7ddb64579f32e0416ae83a60ca2c9cb363b Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 26 Nov 2024 17:32:17 +0800 Subject: [PATCH 18/44] [fix] sys.reset error --- projects/llm_framework/main_sys/src/event_loop.cpp | 2 +- projects/llm_framework/main_sys/src/serial_com.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 6e80bb4d..9910f5e3 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -460,7 +460,7 @@ int sys_reset(int com_id, const nlohmann::json &json_obj) usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"llm server restarting ...\"}", com_id); const char *cmd = - "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm/reset.lock ; sync ; sleep 1 ; rm /tmp/llm/* -f ; " + "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm_reset.lock ; sync ; sleep 1 ; rm /tmp/llm/* -f ; " "systemctl restart llm-* \" > /dev/null 2>&1 & "; system(cmd); return out; diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 049490a6..8b0f80f5 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -73,6 +73,8 @@ class serial_com : public zmq_bus_com { void stop() { + exit_flage = false; + reace_data_event_thread->join(); zmq_bus_com::stop(); linux_uart_deinit(uart_fd); } @@ -111,8 +113,8 @@ void serial_work() out_str += ",\"error\":{\"code\":0, \"message\":\"upgrade over\"}}"; serial_con_->send_data(out_str); } - if (access("/tmp/llm/reset.lock", F_OK) == 0) { - remove("/tmp/llm/reset.lock"); + if (access("/tmp/llm_reset.lock", F_OK) == 0) { + remove("/tmp/llm_reset.lock"); sync(); std::string out_str; out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; From 6881a5a766d45a857354b24eb3cacd49b58d90e6 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 26 Nov 2024 19:08:26 +0800 Subject: [PATCH 19/44] [fix] decode_stream --- .../StackFlow/stackflow/StackFlowUtil.cpp | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 880fe82d..a4c3c0d9 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -183,28 +183,16 @@ std::string StackFlows::sample_unescapeString(const std::string &input) bool StackFlows::decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff) { - simdjson::ondemand::parser parser; - simdjson::padded_string json_string(in); - simdjson::ondemand::document doc; - auto error = parser.iterate(json_string).get(doc); - if (error) { - throw true; - } - int index = doc["index"].get_int64(); - bool finish = doc["finish"].get_bool(); - auto result = doc["delta"].raw_json(); - if (result.error() == simdjson::SUCCESS) { - stream_buff[index] = std::string(result.value().data()); - if (finish) { - for (size_t i = 0; i < stream_buff.size(); i++) { - out += stream_buff.at(i); - } - stream_buff.clear(); - return false; + int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); + std::string finish = StackFlows::sample_json_str_get(in, "finish"); + std::string delta = StackFlows::sample_json_str_get(in, "delta"); + stream_buff[index] = delta; + if (finish.find("true") != std::string::npos) { + for (size_t i = 0; i < stream_buff.size(); i++) { + out += stream_buff.at(i); } - } else { - std::cerr << "result: " << result.value() << "error mesg:" << result.error() << std::endl; - throw true; + stream_buff.clear(); + return false; } return true; } From 225ab5bff4b37542c082d134b6a00804a7e6bc90 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 27 Nov 2024 00:30:58 +0800 Subject: [PATCH 20/44] [update] StackFlow subscriber_event_call decode_stream,delet simdjson. fix decodebase64 --- .../StackFlow/stackflow/StackFlow.cpp | 39 ++++++------------- .../StackFlow/stackflow/StackFlowUtil.cpp | 7 ++-- projects/llm_framework/main/SConstruct | 2 +- projects/llm_framework/main/src/main.cpp | 2 +- projects/llm_framework/main_asr/SConstruct | 1 - projects/llm_framework/main_asr/src/main.cpp | 2 +- projects/llm_framework/main_audio/SConstruct | 2 +- projects/llm_framework/main_kws/SConstruct | 2 +- projects/llm_framework/main_kws/src/main.cpp | 2 +- projects/llm_framework/main_llm/SConstruct | 2 +- projects/llm_framework/main_llm/src/main.cpp | 2 +- .../llm_framework/main_melotts/SConstruct | 2 +- .../llm_framework/main_melotts/src/main.cpp | 2 +- projects/llm_framework/main_tts/SConstruct | 2 +- projects/llm_framework/main_tts/src/main.cpp | 2 +- projects/llm_framework/main_vlm/SConstruct | 2 +- projects/llm_framework/main_vlm/src/main.cpp | 2 +- projects/llm_framework/main_yolo/SConstruct | 2 +- projects/llm_framework/main_yolo/src/main.cpp | 2 +- 19 files changed, 31 insertions(+), 48 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index 25aca68d..046ed9ff 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -5,7 +5,6 @@ */ #include "StackFlow.h" #include "sample_log.h" -#include using namespace StackFlows; @@ -33,34 +32,21 @@ llm_channel_obj::~llm_channel_obj() void llm_channel_obj::subscriber_event_call(const std::function &call, pzmq *_pzmq, const std::string &raw) { - try { - simdjson::padded_string json_string(raw); - simdjson::ondemand::document doc; - auto parser = _pzmq->getContextPtr(); - auto error = parser->iterate(json_string).get(doc); - if (error) { - return; - } - std::string action; - error = doc["action"].get_string(action); - if (action == "inference") { - std::string zmq_com; - error = doc["zmq_com"].get_string(zmq_com); + const char *user_inference_flage_str = "\"action\""; + std::size_t pos = raw.find(user_inference_flage_str); + while (true) { + if (pos == std::string::npos) { + break; + } else if ((pos > 0) && (raw[pos - 1] != '\\')) { + std::string zmq_com = sample_json_str_get(raw, "zmq_com"); if (!zmq_com.empty()) set_push_url(zmq_com); - error = doc["request_id"].get_string(request_id_); - error = doc["work_id"].get_string(work_id_); - } - std::string object; - error = doc["object"].get_string(object); - auto result = doc["data"].raw_json(); - if (result.error() == simdjson::SUCCESS) { - call(object, result.value().data()); - } else { - std::cerr << "result: " << result.value() << "error mesg:" << result.error() << std::endl; + request_id_ = sample_json_str_get(raw, "request_id"); + work_id_ = sample_json_str_get(raw, "work_id"); + break; } - } catch (simdjson::simdjson_error &e) { - std::cerr << "Error: " << simdjson::error_message(e.error()) << std::endl; + pos = raw.find(user_inference_flage_str, pos + sizeof(user_inference_flage_str)); } + call(sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); } int llm_channel_obj::subscriber_work_id(const std::string &work_id, @@ -88,7 +74,6 @@ int llm_channel_obj::subscriber_work_id(const std::string &work_id, zmq_[id_num] = std::make_shared( subscriber_url, ZMQ_SUB, std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1, std::placeholders::_2)); - zmq_[id_num]->newContextPtr(); return 0; } diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index a4c3c0d9..e83504fa 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -6,7 +6,6 @@ #include "StackFlowUtil.h" #include #include "pzmq.hpp" -#include std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { @@ -185,9 +184,9 @@ bool StackFlows::decode_stream(const std::string &in, std::string &out, { int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); std::string finish = StackFlows::sample_json_str_get(in, "finish"); - std::string delta = StackFlows::sample_json_str_get(in, "delta"); - stream_buff[index] = delta; - if (finish.find("true") != std::string::npos) { + stream_buff[index] = StackFlows::sample_json_str_get(in, "delta"); + // sample find flage: false:true + if (finish.find("f") == std::string::npos) { for (size_t i = 0; i < stream_buff.size(); i++) { out += stream_buff.at(i); } diff --git a/projects/llm_framework/main/SConstruct b/projects/llm_framework/main/SConstruct index 6e7374c4..32841632 100644 --- a/projects/llm_framework/main/SConstruct +++ b/projects/llm_framework/main/SConstruct @@ -26,7 +26,7 @@ STATIC_FILES += [AFile('../static_lib/sherpa/ncnn/libsherpa-ncnn-core.so'), AFile('../static_lib/sherpa/ncnn/libkaldi-native-fbank-core.so'), AFile('../static_lib/libonnxruntime.so.1.14.0') ] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'static_file', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 0fc5700d..9004bd60 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -154,7 +154,7 @@ class llm_llm : public StackFlow { std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index dbf03fc3..9a84dddd 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -26,7 +26,6 @@ REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] STATIC_FILES += [AFile('sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json'), AFile('sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json')] -REQUIREMENTS += ['simdjson_component'] env['COMPONENTS'].append({'target':'llm_asr', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 183f349f..27e3798e 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -379,7 +379,7 @@ class llm_asr : public StackFlow { std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index c8371f81..8e6721e6 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -27,7 +27,7 @@ REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_audio', 'ax_audio_3a', 'ax_fdk' REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] STATIC_FILES += [AFile('audio.json')] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_audio', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 690cad3d..95d3dafa 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -33,7 +33,7 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_kws', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 8fc3ea28..5b8f9433 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -409,7 +409,7 @@ class llm_kws : public StackFlow { std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index 9b8fc773..c95c1458 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -42,7 +42,7 @@ STATIC_FILES += [AFile('qwen2.5-0.5B-prefill-20e.json'), AFile('openbuddy-llama3.2-1b-ax630c_tokenizer.py'), AFile('qwen2.5-coder-0.5B-ax630c_tokenizer.py') ] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 3b57ff88..8b6c5572 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -317,7 +317,7 @@ class llm_llm : public StackFlow { std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index ad89084c..6124c3e7 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -27,7 +27,7 @@ INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] STATIC_FILES += [AFile('melotts_zh-cn.json')] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_melotts', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index ba18fffd..cd6ce1ee 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -436,7 +436,7 @@ class llm_tts : public StackFlow { std::string tmp_msg2; if (enbase64) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index e7f38030..0ce3b6bb 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -28,7 +28,7 @@ INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), AD STATIC_FILES += [AFile('single_speaker_english_fast.json'), AFile('single_speaker_fast.json') ] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_tts', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index b87647d8..cfc91b28 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -273,7 +273,7 @@ class llm_tts : public StackFlow { std::string tmp_msg2; if (enbase64) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { return; } next_data = &tmp_msg2; diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index 7a5b8746..b1f51c8a 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -52,7 +52,7 @@ STATIC_LIB += static_file * 4 STATIC_FILES += [AFile('internvl2-1b-ax630c.json'), AFile('internvl2-1b-ax630c_tokenizer.py') ] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_vlm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index ea8ad222..4a1c1eb7 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -327,7 +327,7 @@ class llm_llm : public StackFlow { std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 06c1f551..372ee62f 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -29,7 +29,7 @@ STATIC_LIB += static_file * 2 STATIC_FILES += [AFile('yolo11s.json'), AFile('yolo11s-seg.json'), AFile('yolo11s-pose.json')] -REQUIREMENTS += ['simdjson_component'] + env['COMPONENTS'].append({'target':'llm_yolo', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 35ddd4d1..56738b29 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -321,7 +321,7 @@ class llm_yolo : public StackFlow { // must encode base64 std::string tmp_msg2; ret = decode_base64((*next_data), tmp_msg2); - if (!ret) { + if (ret == -1) { error_body["code"] = -23; error_body["message"] = "Base64 decoding error."; send("None", "None", error_body, unit_name_); From 394640cdb87ad414f42b9ff9369a33a2fb3c9f4b Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 27 Nov 2024 09:37:25 +0800 Subject: [PATCH 21/44] [update] - sys select_json_str add new action - update serial_com && tcp_com --- .../llm_framework/main_sys/include/zmq_bus.h | 6 + .../llm_framework/main_sys/src/serial_com.cpp | 11 +- .../llm_framework/main_sys/src/tcp_com.cpp | 51 ++++--- .../llm_framework/main_sys/src/zmq_bus.cpp | 138 +++++++++++++----- 4 files changed, 145 insertions(+), 61 deletions(-) diff --git a/projects/llm_framework/main_sys/include/zmq_bus.h b/projects/llm_framework/main_sys/include/zmq_bus.h index ec5360a7..e32c7c63 100644 --- a/projects/llm_framework/main_sys/include/zmq_bus.h +++ b/projects/llm_framework/main_sys/include/zmq_bus.h @@ -41,6 +41,11 @@ int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_s void zmq_com_send(int com_id, const std::string &out_str); class zmq_bus_com { +private: + int reace_event_; + int raw_msg_len_; + std::string raw_msg_buff_; + protected: std::string _zmq_url; int exit_flage; @@ -57,6 +62,7 @@ class zmq_bus_com { void stop(); void select_json_str(const std::string &json_src, std::function out_fun); virtual void on_data(const std::string &data); + virtual void on_raw_data(const std::string &data); virtual void send_data(const std::string &data); virtual void reace_data_event(); virtual void send_data_event(); diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 8b0f80f5..1d8aba58 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -46,19 +46,18 @@ class serial_com : public zmq_bus_com { void reace_data_event() { - std::string json_str; - int flage = 0; fd_set readfds; + std::vector buff(1024); while (exit_flage) { - char reace_buf[128] = {0}; FD_ZERO(&readfds); FD_SET(uart_fd, &readfds); struct timeval timeout = {0, 500000}; if ((select(uart_fd + 1, &readfds, NULL, NULL, &timeout) <= 0) || (!FD_ISSET(uart_fd, &readfds))) continue; - int len = linux_uart_read(uart_fd, 128, reace_buf); + ssize_t len = linux_uart_read(uart_fd, buff.size(), buff.data()); + if(len <= 0) continue; { try { - select_json_str(std::string(reace_buf, len), + select_json_str(std::string(buff.data(), buff.size()), std::bind(&serial_com::on_data, this, std::placeholders::_1)); } catch (...) { std::string out_str; @@ -73,8 +72,6 @@ class serial_com : public zmq_bus_com { void stop() { - exit_flage = false; - reace_data_event_thread->join(); zmq_bus_com::stop(); linux_uart_deinit(uart_fd); } diff --git a/projects/llm_framework/main_sys/src/tcp_com.cpp b/projects/llm_framework/main_sys/src/tcp_com.cpp index 238017cb..b908f44b 100644 --- a/projects/llm_framework/main_sys/src/tcp_com.cpp +++ b/projects/llm_framework/main_sys/src/tcp_com.cpp @@ -33,7 +33,7 @@ TcpServer srv; class tcp_com : public zmq_bus_com { private: public: - SocketChannelPtr channel; + std::weak_ptr channel; std::mutex tcp_server_mutex; tcp_com() : zmq_bus_com() { @@ -41,28 +41,37 @@ class tcp_com : public zmq_bus_com { void send_data(const std::string& data) { - tcp_server_mutex.lock(); - if (exit_flage) channel->write(data); - tcp_server_mutex.unlock(); + // tcp_server_mutex.lock(); + // if (exit_flage) + // { + // auto tcp_channel = channel.lock(); + // if(tcp_channel) + // { + // tcp_channel->write(data); + // } + // } + // tcp_server_mutex.unlock(); + auto tcp_channel = channel.lock(); + if(tcp_channel) + { + tcp_channel->write(data); + } } }; void onConnection(const SocketChannelPtr& channel) { - std::string peeraddr = channel->peeraddr(); - tcp_com* con_data; if (channel->isConnected()) { - con_data = new tcp_com(); - con_data->channel = channel; - con_data->work(zmq_s_format, counter_port.fetch_add(1)); + auto p_com = channel->newContextPtr(); + p_com->channel = channel; + p_com->work(zmq_s_format, counter_port.fetch_add(1)); if (counter_port.load() > 65535) counter_port.store(8000); - channel->setContext(con_data); } else { - con_data = (tcp_com*)channel->context(); - con_data->tcp_server_mutex.lock(); - con_data->stop(); - con_data->tcp_server_mutex.unlock(); - delete con_data; + auto p_com = channel->getContextPtr(); + // p_com->tcp_server_mutex.lock(); + p_com->stop(); + // p_com->tcp_server_mutex.unlock(); + channel->deleteContextPtr(); } } @@ -70,11 +79,11 @@ void onMessage(const SocketChannelPtr& channel, Buffer* buf) { int len = (int)buf->size(); char* data = (char*)buf->data(); - tcp_com* con_data = (tcp_com*)channel->context(); - con_data->tcp_server_mutex.lock(); + auto p_com = channel->getContextPtr(); + p_com->tcp_server_mutex.lock(); try { - con_data->select_json_str(std::string(data, len), - std::bind(&tcp_com::on_data, con_data, std::placeholders::_1)); + p_com->select_json_str(std::string(data, len), + std::bind(&tcp_com::on_data, p_com, std::placeholders::_1)); } catch (...) { std::string out_str; out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; @@ -82,7 +91,7 @@ void onMessage(const SocketChannelPtr& channel, Buffer* buf) out_str += ",\"error\":{\"code\":-1, \"message\":\"reace reset\"}}"; channel->write(out_str); } - con_data->tcp_server_mutex.unlock(); + p_com->tcp_server_mutex.unlock(); } void tcp_work() @@ -97,7 +106,7 @@ void tcp_work() printf("server listen on port %d, listenfd=%d ...\n", listenport, listenfd); srv.onConnection = onConnection; srv.onMessage = onMessage; - srv.setThreadNum(1); + srv.setThreadNum(2); srv.start(); } void tcp_stop_work() diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index ebcdbf5c..b0da1e7f 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #if defined(__ARM_NEON) || defined(__ARM_NEON__) #include #endif @@ -23,6 +24,8 @@ zmq_bus_com::zmq_bus_com() exit_flage = 1; err_count = 0; json_str_flage_ = 0; + reace_event_ = 0; + raw_msg_len_ = 0; } void zmq_bus_com::work(const std::string &zmq_url_format, int port) @@ -51,6 +54,33 @@ void zmq_bus_com::on_data(const std::string &data) unit_action_match(_port, data); } +void zmq_bus_com::on_raw_data(const std::string &data) +{ + std::string base64_data; + int ret = StackFlows::encode_base64(data, base64_data); + ssize_t pos = json_str_.find("\"data\""); + if (pos != std::string::npos) { + ssize_t end_pos = pos + 6; + int status = 0; + for (ssize_t end_pos = pos + 6; end_pos < json_str_.length(); end_pos++) { + if ((json_str_[end_pos] == '\"') || (json_str_[end_pos] == ',') || (json_str_[end_pos] == '}')) { + status++; + } + if (status == 3) { + break; + } + } + json_str_.erase(pos, end_pos - pos); + } + std::string new_data; + new_data.resize(json_str_.length() + 10 + base64_data.length()); + memcpy((void *)new_data.data(), json_str_.c_str(), json_str_.length()); + sprintf((char *)(new_data.data() + json_str_.length() - 1), ",\"data\":\""); + memcpy((void *)(new_data.data() + json_str_.length() + 8), base64_data.c_str(), base64_data.length()); + new_data[json_str_.length() + 9 + base64_data.length()] = '}'; + on_data(new_data); +} + void zmq_bus_com::send_data(const std::string &data) { // printf("zmq_bus_com::send_data : send:%s\n", data.c_str()); @@ -74,6 +104,9 @@ void zmq_bus_com::send_data_event() zmq_bus_com::~zmq_bus_com() { + if (exit_flage) { + stop(); + } } unit_data::unit_data() @@ -129,40 +162,79 @@ void zmq_bus_stop_work() void zmq_bus_com::select_json_str(const std::string &json_src, std::function out_fun) { - json_str_.reserve(json_str_.length() + json_src.length()); - const char *data = json_src.c_str(); - for (int i = 0; i < json_src.length(); i++) { + bool enloop = false; + std::shared_ptr src_str((std::string *)&json_src, [](std::string *pt) {}); + do { + enloop = false; + switch (reace_event_) { + case 0: { + json_str_.reserve(json_str_.length() + src_str->length()); + const char *data = src_str->c_str(); + for (int i = 0; i < src_str->length(); i++) { #if defined(__ARM_NEON) || defined(__ARM_NEON__) - if (json_src.length() - i >= 16) { - uint8x16_t target_open = vdupq_n_u8('{'); - uint8x16_t target_close = vdupq_n_u8('}'); - uint8x16_t input_vector = vld1q_u8((const uint8_t *)&data[i]); - uint8x16_t result_open = vceqq_u8(input_vector, target_open); - uint8x16_t result_close = vceqq_u8(input_vector, target_close); - uint8x16_t result_mask = vorrq_u8(result_open, result_close); - __uint128_t jflage; - vst1q_u8((uint8_t *)&jflage, result_mask); - if (jflage == 0) { - json_str_.append(data + i, 16); - i += 15; - continue; - } - } + if (src_str->length() - i >= 16) { + uint8x16_t target_open = vdupq_n_u8('{'); + uint8x16_t target_close = vdupq_n_u8('}'); + uint8x16_t input_vector = vld1q_u8((const uint8_t *)&data[i]); + uint8x16_t result_open = vceqq_u8(input_vector, target_open); + uint8x16_t result_close = vceqq_u8(input_vector, target_close); + uint8x16_t result_mask = vorrq_u8(result_open, result_close); + __uint128_t jflage; + vst1q_u8((uint8_t *)&jflage, result_mask); + if (jflage == 0) { + json_str_.append(data + i, 16); + i += 15; + continue; + } + } #endif - json_str_ += json_src[i]; - int last_index = (i == 0) ? 0 : (i - 1); - if ((data[i] == '{') && (data[last_index] != '\\')) json_str_flage_++; - if ((data[i] == '}') && (data[last_index] != '\\')) json_str_flage_--; - if (json_str_flage_ == 0) { - if (json_str_[0] == '{') { - out_fun(json_str_); - } - json_str_.clear(); - } - if (json_str_flage_ < 0) { - json_str_flage_ = 0; - json_str_.clear(); - throw std::runtime_error("json package error"); + json_str_ += data[i]; + int last_index = (i == 0) ? 0 : (i - 1); + if ((data[i] == '{') && (data[last_index] != '\\')) json_str_flage_++; + if ((data[i] == '}') && (data[last_index] != '\\')) json_str_flage_--; + if (json_str_flage_ == 0) { + if (json_str_[0] == '{') { + if ((json_str_.length() > 7) && (json_str_[1] == '\"') && (json_str_[2] == 'R') && + (json_str_[3] == 'A') && (json_str_[4] == 'W') && (json_str_[5] == '\"') && + (json_str_[6] == ':')) { + reace_event_ = 10; + raw_msg_len_ = std::stoi(StackFlows::sample_json_str_get(json_str_, "RAW")); + raw_msg_buff_.reserve(raw_msg_len_); + if (json_src.length() > i) { + src_str = std::make_shared(src_str->substr(i + 1)); + enloop = true; + } + break; + } + out_fun(json_str_); + } + json_str_.clear(); + } + if (json_str_flage_ < 0) { + json_str_flage_ = 0; + json_str_.clear(); + throw std::runtime_error("json package error"); + } + } + } break; + case 10: { + if (raw_msg_len_ > src_str->length()) { + raw_msg_buff_.append(src_str->c_str(), src_str->length()); + raw_msg_len_ -= src_str->length(); + } else { + reace_event_ = 0; + raw_msg_buff_.append(src_str->c_str(), raw_msg_len_); + on_raw_data(raw_msg_buff_); + raw_msg_buff_.clear(); + json_str_.clear(); + if (src_str->length() > raw_msg_len_) { + src_str = std::make_shared(src_str->substr(raw_msg_len_ + 1)); + enloop = true; + } + } + } break; + default: + break; } - } + } while (enloop); } \ No newline at end of file From 689d5071c923e360f7dd500a2c22648cb06ed803 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 27 Nov 2024 10:37:05 +0800 Subject: [PATCH 22/44] [fix] serial_com error --- projects/llm_framework/main_sys/src/serial_com.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index 1d8aba58..d046a98c 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -57,7 +57,7 @@ class serial_com : public zmq_bus_com { if(len <= 0) continue; { try { - select_json_str(std::string(buff.data(), buff.size()), + select_json_str(std::string(buff.data(), len), std::bind(&serial_com::on_data, this, std::placeholders::_1)); } catch (...) { std::string out_str; From 1a9971d23cd2e9b7f74e34a2468f1e59bb8dd26d Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 27 Nov 2024 14:09:23 +0800 Subject: [PATCH 23/44] [fix] Run the error individually. --- ext_components/StackFlow/stackflow/pzmq.hpp | 5 +---- projects/llm_framework/main_sys/src/event_loop.cpp | 2 +- projects/llm_framework/main_sys/src/serial_com.cpp | 3 ++- projects/llm_framework/main_sys/src/zmq_bus.cpp | 6 +++++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index 5023da04..ca028528 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -183,7 +183,6 @@ class pzmq { case ZMQ_PUSH: { int reconnect_interval = 100; zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL, &reconnect_interval, sizeof(reconnect_interval)); - int max_reconnect_interval = 1000; // 5 seconds zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL_MAX, &max_reconnect_interval, sizeof(max_reconnect_interval)); @@ -243,9 +242,7 @@ class pzmq { } inline int creat_push(const std::string &url) { - int timeout = 3000; - zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout, sizeof(timeout)); - zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout, sizeof(timeout)); + zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout_, sizeof(timeout_)); return zmq_connect(zmq_socket_, url.c_str()); } inline int creat_pull(const std::string &url, const msg_callback_fun &raw_call) diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 9910f5e3..af1d22d9 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -460,7 +460,7 @@ int sys_reset(int com_id, const nlohmann::json &json_obj) usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"llm server restarting ...\"}", com_id); const char *cmd = - "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm_reset.lock ; sync ; sleep 1 ; rm /tmp/llm/* -f ; " + "[ -f '/tmp/llm/reset.lock' ] || bash -c \"touch /tmp/llm_reset.lock ; sync ; " "systemctl restart llm-* \" > /dev/null 2>&1 & "; system(cmd); return out; diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index d046a98c..d9952357 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -41,7 +41,8 @@ class serial_com : public zmq_bus_com { void send_data(const std::string &data) { - linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); + if(exit_flage) + linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); } void reace_data_event() diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index b0da1e7f..e863623f 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -131,6 +131,10 @@ unit_data::~unit_data() int zmq_bus_publisher_push(const std::string &work_id, const std::string &json_str) { + if (work_id.empty()) { + SLOGW("work_id is empty"); + return -1; + } unit_data *unit_p = NULL; SAFE_READING(unit_p, unit_data *, work_id); if (unit_p) @@ -193,7 +197,7 @@ void zmq_bus_com::select_json_str(const std::string &json_src, std::function 7) && (json_str_[1] == '\"') && (json_str_[2] == 'R') && (json_str_[3] == 'A') && (json_str_[4] == 'W') && (json_str_[5] == '\"') && (json_str_[6] == ':')) { From f5e81ca161d418d2912abb1d6a5f4756227db0fc Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 27 Nov 2024 17:06:14 +0800 Subject: [PATCH 24/44] [update] pzmq close && base64 size --- .../StackFlow/stackflow/StackFlowUtil.cpp | 14 ++++++++++---- ext_components/StackFlow/stackflow/pzmq.hpp | 4 ++-- projects/llm_framework/main_sys/src/event_loop.cpp | 6 +++--- projects/llm_framework/main_sys/src/zmq_bus.cpp | 1 + .../llm_framework/main_yolo/src/EngineWrapper.cpp | 2 +- projects/llm_framework/main_yolo/src/main.cpp | 2 +- 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index e83504fa..b0bc14c0 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -386,14 +386,20 @@ static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) int StackFlows::decode_base64(const std::string &in, std::string &out) { - out.resize(BASE64_DECODE_OUT_SIZE(in.length())); - return base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); + int out_size = BASE64_DECODE_OUT_SIZE(in.length()); + out.resize(out_size); + int ret = base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); + if ((ret > 0) && (ret != out_size)) out.erase(ret); + return ret; } int StackFlows::encode_base64(const std::string &in, std::string &out) { - out.resize(BASE64_ENCODE_OUT_SIZE(in.length())); - return base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); + int out_size = BASE64_ENCODE_OUT_SIZE(in.length()); + out.resize(out_size); + int ret = base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); + if ((ret > 0) && (ret != out_size)) out.erase(ret); + return ret; } std::string StackFlows::unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data) diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index ca028528..19421093 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -348,9 +348,9 @@ class pzmq { if (!zmq_socket_) { return; } + flage_ = true; + zmq_ctx_shutdown(zmq_ctx_); if (zmq_thread_) { - flage_ = true; - zmq_ctx_shutdown(zmq_ctx_); zmq_thread_->join(); } close_zmq(); diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index af1d22d9..5fba679a 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -519,21 +519,21 @@ void unit_action_match(int com_id, const std::string &json_str) std::string request_id; error = doc["request_id"].get_string(request_id); if (error) { - SLOGE("miss request_id"); + SLOGE("miss request_id, error:%s", simdjson::error_message(error)); usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } std::string work_id; error = doc["work_id"].get_string(work_id); if (error) { - SLOGE("miss work_id"); + SLOGE("miss work_id, error:%s", simdjson::error_message(error)); usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } std::string action; error = doc["action"].get_string(action); if (error) { - SLOGE("miss action"); + SLOGE("miss action, error:%s", simdjson::error_message(error)); usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index e863623f..e60e5a13 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -77,6 +77,7 @@ void zmq_bus_com::on_raw_data(const std::string &data) memcpy((void *)new_data.data(), json_str_.c_str(), json_str_.length()); sprintf((char *)(new_data.data() + json_str_.length() - 1), ",\"data\":\""); memcpy((void *)(new_data.data() + json_str_.length() + 8), base64_data.c_str(), base64_data.length()); + new_data[json_str_.length() + 8 + base64_data.length()] = '"'; new_data[json_str_.length() + 9 + base64_data.length()] = '}'; on_data(new_data); } diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp index 286c6aeb..592d74e4 100644 --- a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -324,7 +324,7 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c cls_num); } detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); - detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); + // detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); } else if (model_type == "segment") { float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, (float*)io_data->pOutputs[2].pVirAddr}; diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 56738b29..de9e01b9 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -275,7 +275,7 @@ class llm_yolo : public StackFlow { if (finish) count = 0; llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); } else if (finish) { - SLOGI("send utf-8"); + // SLOGI("send utf-8"); llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); } } From d2747b8c0a1c84440784283b119e905f1f6df48b Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 28 Nov 2024 09:15:49 +0800 Subject: [PATCH 25/44] [update] yolo11s to yolo11n --- projects/llm_framework/main_yolo/SConstruct | 2 +- .../{yolo11s-pose.json => yolo11n-pose.json} | 14 +++++++------- .../{yolo11s-seg.json => yolo11n-seg.json} | 14 +++++++------- .../main_yolo/{yolo11s.json => yolo11n.json} | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) rename projects/llm_framework/main_yolo/{yolo11s-pose.json => yolo11n-pose.json} (92%) rename projects/llm_framework/main_yolo/{yolo11s-seg.json => yolo11n-seg.json} (91%) rename projects/llm_framework/main_yolo/{yolo11s.json => yolo11n.json} (94%) diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 372ee62f..02f8a7d2 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -27,7 +27,7 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 -STATIC_FILES += [AFile('yolo11s.json'), AFile('yolo11s-seg.json'), AFile('yolo11s-pose.json')] +STATIC_FILES += [AFile('yolo11n.json'), AFile('yolo11n-seg.json'), AFile('yolo11n-pose.json')] env['COMPONENTS'].append({'target':'llm_yolo', diff --git a/projects/llm_framework/main_yolo/yolo11s-pose.json b/projects/llm_framework/main_yolo/yolo11n-pose.json similarity index 92% rename from projects/llm_framework/main_yolo/yolo11s-pose.json rename to projects/llm_framework/main_yolo/yolo11n-pose.json index 7b28eaab..7bfc51f7 100644 --- a/projects/llm_framework/main_yolo/yolo11s-pose.json +++ b/projects/llm_framework/main_yolo/yolo11n-pose.json @@ -1,20 +1,20 @@ { - "mode":"yolo11s", + "mode":"yolo11n-pose", "type":"cv", "capabilities":[ - "object detect" + "Pose" ], "input_type":[ - "cv.jpeg.base64" + "yolo.jpeg.base64" ], "output_type":[ - "cv.yolobox" + "yolo.yolobox" ], "mode_param":{ - "yolo_model":"yolo11s-pose.axmodel", + "yolo_model":"yolo11n-pose.axmodel", "model_type":"pose", - "img_h":640, - "img_w":640, + "img_h":320, + "img_w":320, "cls_num":1, "pron_threshold":0.45, "nms_threshold":0.45, diff --git a/projects/llm_framework/main_yolo/yolo11s-seg.json b/projects/llm_framework/main_yolo/yolo11n-seg.json similarity index 91% rename from projects/llm_framework/main_yolo/yolo11s-seg.json rename to projects/llm_framework/main_yolo/yolo11n-seg.json index 62d89801..776fae50 100644 --- a/projects/llm_framework/main_yolo/yolo11s-seg.json +++ b/projects/llm_framework/main_yolo/yolo11n-seg.json @@ -1,20 +1,20 @@ { - "mode":"yolo11s", + "mode":"yolo11s-seg", "type":"cv", "capabilities":[ - "segment" + "Segmentation" ], "input_type":[ - "cv.jpeg.base64" + "yolo.jpeg.base64" ], "output_type":[ - "cv.yolobox" + "yolo.yolobox" ], "mode_param":{ - "yolo_model":"yolo11s-seg.axmodel", + "yolo_model":"yolo11n-seg.axmodel", "model_type":"segment", - "img_h":640, - "img_w":640, + "img_h":320, + "img_w":320, "cls_num":80, "pron_threshold":0.45, "nms_threshold":0.45, diff --git a/projects/llm_framework/main_yolo/yolo11s.json b/projects/llm_framework/main_yolo/yolo11n.json similarity index 94% rename from projects/llm_framework/main_yolo/yolo11s.json rename to projects/llm_framework/main_yolo/yolo11n.json index a58db697..abd92827 100644 --- a/projects/llm_framework/main_yolo/yolo11s.json +++ b/projects/llm_framework/main_yolo/yolo11n.json @@ -1,8 +1,8 @@ { - "mode":"yolo11s", + "mode":"yolo11n", "type":"cv", "capabilities":[ - "object detect" + "Detection" ], "input_type":[ "yolo.jpeg.base64" @@ -11,10 +11,10 @@ "yolo.yolobox" ], "mode_param":{ - "yolo_model":"yolo11s.axmodel", + "yolo_model":"yolo11n.axmodel", "model_type":"detect", - "img_h":640, - "img_w":640, + "img_h":320, + "img_w":320, "cls_num":80, "pron_threshold":0.45, "nms_threshold":0.45, From 234cc58e1f4238f91d2b559291449c21d044e93c Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 28 Nov 2024 10:31:46 +0800 Subject: [PATCH 26/44] [update] fix confidence --- projects/llm_framework/main_yolo/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index de9e01b9..929f8f58 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -163,7 +163,7 @@ class llm_task { const detection::Object &obj = objects[i]; nlohmann::json output; output["class"] = mode_config_.cls_name[obj.label]; - output["confidence"] = format_float(obj.prob * 100, 2); + output["confidence"] = format_float(obj.prob, 2); output["bbox"] = nlohmann::json::array(); output["bbox"].push_back(format_float(obj.rect.x, 0)); output["bbox"].push_back(format_float(obj.rect.y, 0)); From 8ec0be849d36de6bc6e412be2fa5e690dc10b4db Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 28 Nov 2024 15:37:23 +0800 Subject: [PATCH 27/44] [update] add llm_skel(Temporary version) --- projects/llm_framework/main_skel/Kconfig | 0 projects/llm_framework/main_skel/SConstruct | 47 + projects/llm_framework/main_skel/src/main.c | 2285 +++++++++++++++++ .../llm_framework/main_skel/src/main.cpp.bak | 525 ++++ .../main_skel/src/runner/YuvHandler.c | 269 ++ .../main_skel/src/runner/YuvHandler.h | 46 + .../main_skel/src/runner/attrParser.cpp | 224 ++ .../main_skel/src/runner/attrParser.h | 60 + .../main_skel/src/runner/common_vdec_utils.c | 2233 ++++++++++++++++ .../main_skel/src/runner/frameMgr.cpp | 72 + .../main_skel/src/runner/frameMgr.h | 33 + .../main_skel/src/runner/picojson.h | 1151 +++++++++ .../main_skel/src/runner/sample_skel.c | 2285 +++++++++++++++++ .../main_skel/src/runner/skel_log.h | 61 + .../main_skel/src/runner/statMgr.cpp | 65 + .../main_skel/src/runner/statMgr.h | 36 + 16 files changed, 9392 insertions(+) create mode 100644 projects/llm_framework/main_skel/Kconfig create mode 100644 projects/llm_framework/main_skel/SConstruct create mode 100644 projects/llm_framework/main_skel/src/main.c create mode 100644 projects/llm_framework/main_skel/src/main.cpp.bak create mode 100644 projects/llm_framework/main_skel/src/runner/YuvHandler.c create mode 100644 projects/llm_framework/main_skel/src/runner/YuvHandler.h create mode 100644 projects/llm_framework/main_skel/src/runner/attrParser.cpp create mode 100644 projects/llm_framework/main_skel/src/runner/attrParser.h create mode 100755 projects/llm_framework/main_skel/src/runner/common_vdec_utils.c create mode 100644 projects/llm_framework/main_skel/src/runner/frameMgr.cpp create mode 100644 projects/llm_framework/main_skel/src/runner/frameMgr.h create mode 100644 projects/llm_framework/main_skel/src/runner/picojson.h create mode 100644 projects/llm_framework/main_skel/src/runner/sample_skel.c create mode 100644 projects/llm_framework/main_skel/src/runner/skel_log.h create mode 100644 projects/llm_framework/main_skel/src/runner/statMgr.cpp create mode 100644 projects/llm_framework/main_skel/src/runner/statMgr.h diff --git a/projects/llm_framework/main_skel/Kconfig b/projects/llm_framework/main_skel/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_skel/SConstruct b/projects/llm_framework/main_skel/SConstruct new file mode 100644 index 00000000..b6aeb01a --- /dev/null +++ b/projects/llm_framework/main_skel/SConstruct @@ -0,0 +1,47 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = append_srcs_dir(ADir('src')) +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'ax-samples'] + +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] +REQUIREMENTS += ['ax_skel', 'ax_engine', 'ax_venc', 'ax_vdec', 'ax_ivps', 'ax_proton'] + +INCLUDE += [ADir('../include'), ADir('src/runner'), ADir('../include/opencv4')] + +static_file = Glob('../static_lib/module-llm/libabsl_*') +static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') +STATIC_LIB += static_file * 2 + + + +env['COMPONENTS'].append({'target':'llm_skel', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_skel/src/main.c b/projects/llm_framework/main_skel/src/main.c new file mode 100644 index 00000000..b1e9364a --- /dev/null +++ b/projects/llm_framework/main_skel/src/main.c @@ -0,0 +1,2285 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "YuvHandler.h" +#include "ax_skel_api.h" +#include "ax_sys_api.h" +#include "ax_ivps_api.h" +#include "ax_venc_api.h" +#include "ax_buffer_tool.h" +#include "skel_log.h" +#include "ax_engine_api.h" +#include "frameMgr.h" +#include "attrParser.h" +#include "statMgr.h" +#include "common_vdec_api.h" + +#ifndef ALIGN_UP +#define ALIGN_UP(x, align) ((((x) + ((align) - 1)) / (align)) * (align)) +#endif + +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, align) (((x) / (align)) * (align)) +#endif + +#ifndef AX_MAX +#define AX_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef AX_MIN +#define AX_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" +#define SKEL_SAMPLE_OUTPUT_VEHICLE_PATH "vehicle" +#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" +#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" +#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" +#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" + +#define SAMPLE_SKEL_SHIFT_LEFT_ALIGN(a) (1 << (a)) +#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) +#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) + +#define RUN_COMMAND(format, ...) do { \ + char cmd[512]; \ + snprintf(cmd, 512, format, ##__VA_ARGS__); \ + system(cmd); \ + } while(0); + +AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) { + if (file) { + va_list args; + char szLog[1024] = {0}; + + va_start(args, fmt); + + AX_U32 len = vsnprintf(szLog, sizeof(szLog), (char *)fmt, args); + if (len < (AX_U32)sizeof(szLog)) { + szLog[len] = '\0'; + } else { + szLog[sizeof(szLog) - 1] = '\0'; + } + + fwrite((AX_U8 *)szLog, 1, strlen(szLog), file); + + va_end(args); + } +} +#define OUTPUT_LOG_SAVE(fmt, ...) LogSaveToFile(fpResultFile, fmt "\n", ##__VA_ARGS__) +#define BLACK_VIDEO_FRAME_COUNT 60 +#define SKEL_SAMPLE_OBJECT_SIZE 512 + +struct skeleton { + int connection[2]; + int left_right_neutral; +}; + +struct skeleton pairs[] = {{{15, 13}, 0}, {{13, 11}, 0}, {{16, 14}, 0}, {{14, 12}, 0}, {{11, 12}, 0}, + {{5, 11}, 0}, {{6, 12}, 0}, {{5, 6}, 0}, {{5, 7}, 0}, {{6, 8}, 0}, + {{7, 9}, 0}, {{8, 10}, 0}, {{1, 2}, 0}, {{0, 1}, 0}, {{0, 2}, 0}, + {{1, 3}, 0}, {{2, 4}, 0}, {{0, 5}, 0}, {{0, 6}, 0}}; + +#define DETECT_SKEL_POINT_COUNT 256 + +typedef struct _AI_Detection_Box_t { + AX_F32 fX, fY, fW, fH; +} AI_Detection_Box_t; + +typedef struct _AI_Detection_Point_t { + AX_F32 fX, fY; +} AI_Detection_Point_t; + +typedef struct _AI_Detection_SkelResult_t { + const AX_CHAR *pstrObjectCategory; + AX_U8 nPointNum; + AI_Detection_Point_t tPoint[DETECT_SKEL_POINT_COUNT]; + AI_Detection_Box_t tBox; +} AI_Detection_SkelResult_t; + +static AX_U64 get_tick_count(AX_VOID) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +// static AX_U64 get_tick_count_us(AX_VOID) { +// struct timespec ts; +// clock_gettime(CLOCK_MONOTONIC, &ts); +// return (ts.tv_sec * 1000000 + ts.tv_nsec / 1000); +// } + +static AX_VOID ShowUsage(AX_VOID) { + // printf("usage: ./%s ...\n", SAMPLE_SKEL_NAME); + printf("options:\n"); + printf("-i, \tInput File(yuv)\n"); + printf("-r, \tInput File Resolution(wxh)(yuv: should be input)\n"); + printf("-w, \tWrite result image to new jpg file((unsigned int), default=0)\n"); + printf("-o, \tSave result to file(file name)\n"); + printf("-m, \tModels deployment path(path name)\n"); + printf("-t, \tRepeat times((unsigned int), default=1)\n"); + printf("-I, \tInterval repeat time((unsigned int)ms, default=0)\n"); + printf("-c, \tConfidence((float: 0-1), default=0)\n"); + printf("-H, \tHuman track size limit((unsigned int), default=0)\n"); + printf("-V, \tVehicle track size limit((unsigned int), default=0)\n"); + printf("-C, \tCylcle track size limit((unsigned int), default=0)\n"); + printf("-d, \tSkel detect type((unsigned int), default=2)\n" + "\t\t0: detect only\n" + "\t\t1: detect + track\n" + "\t\t2: detect + track + push\n"); + printf("-u, \tSkel push strategy((unsigned int), default=3)\n" + "\t\t1: fast push strategy\n" + "\t\t2: push strategy\n" + "\t\t3: best push strategy\n"); + printf("-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" + "\t\t0: VNPU Disable\n" + "\t\t1: STD-VNPU Default\n" + "\t\t2: STD-VNPU1\n" + "\t\t3: STD-VNPU2\n"); + printf("-p, \tSkel PPL((unsigned int), default=1)\n" + "\t\t1: AX_SKEL_PPL_HVCP\n" + "\t\t2: AX_SKEL_PPL_FACE\n"); + printf("-v, \tLog level((unsigned int), default=5)\n" + "\t\t0: LOG_EMERGENCY_LEVEL\n" + "\t\t1: LOG_ALERT_LEVEL\n" + "\t\t2: LOG_CRITICAL_LEVEL\n" + "\t\t3: LOG_ERROR_LEVEL\n" + "\t\t4: LOG_WARN_LEVEL\n" + "\t\t5: LOG_NOTICE_LEVEL\n" + "\t\t6: LOG_INFO_LEVEL\n" + "\t\t7: LOG_DEBUG_LEVEL\n"); + printf("-h, \tprint this message\n"); +} + +AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { + if (pstConfig->nSize > 0 && pstConfig->pstItems) { + for (size_t i = 0; i < pstConfig->nSize; i++) { + if (pstConfig->pstItems[i].pstrType && pstConfig->pstItems[i].pstrValue) { + // cmd: "body_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + if (strcmp(pstConfig->pstItems[i].pstrType, "body_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "crop_encoder_qpLevel", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "crop_encoder_qpLevel") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "detect_roi_polygon") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T)) { + AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, pstConf->nPointNum); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_strategy", value_type: AX_SKEL_PUSH_STRATEGY_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_strategy") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_STRATEGY_T)) { + AX_SKEL_PUSH_STRATEGY_T *pstConf = (AX_SKEL_PUSH_STRATEGY_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, pstConf->ePushMode, pstConf->nIntervalTimes, + pstConf->nPushCounts, pstConf->bPushSameFrame); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_panorama", value_type: AX_SKEL_PUSH_PANORAMA_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_panorama") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T)) { + AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Enable: %d, Quality: %d]", pstConfig->pstItems[i].pstrType, + pstConf->bEnable, pstConf->nQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_body", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_body") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_vehicle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_vehicle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_cycle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_cycle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_face", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_face") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [W: %d, H: %d, P: %f, Y: %f, R: %f, B: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, + pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, + pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_plate", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_plate") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_bind_enable", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_bind_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "track_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "target_config", value_type: AX_SKEL_TARGET_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_target_config", value_type: AX_SKEL_TARGET_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "analyzer_attr_config", value_type: AX_SKEL_ANALYZER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "analyzer_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ANALYZER_CONFIG_T)) { + AX_SKEL_ANALYZER_CONFIG_T *pstConf = (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->peItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + ALOGI("SKEL set %s: [%d]%d", pstConfig->pstItems[i].pstrType, j, pstConf->peItems[j]); + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "venc_attr_config", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "venc_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, (AX_U32)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_attr_always", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_attr_always") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + else { + ALOGE("SKEL cmd: %s not support", pstConfig->pstItems[i].pstrType); + } + } + } + } + + return 0; +} + +FILE* LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + + if (f_in) { + if (pLen) { + fseeko(f_in, 0L, SEEK_END); + AX_U64 nFileSize = ftello(f_in); + rewind(f_in); + + *pLen = (AX_U64)nFileSize; + } + } + + return f_in; +} + +int ReLoadFile(FILE* pFile) +{ + if (pFile) { + rewind(pFile); + return 0; + } + + return -1; +} + +int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, + AX_VOID *pSrcVirAddr, + AX_U32 nSrcSize, + AX_U32 nSrcStride, + AX_U32 nSrcHeight, + AX_U64 nDstPhyAddr, + AX_VOID *pDstVirAddr, + AX_U32 nDstSize, + AX_U32 nDstStride, + AX_U32 nDstHeight + ) { + AX_U64 nStartTime = get_tick_count(); + + // src check + if (!pSrcVirAddr) { + return -1; + } + + if (nSrcSize != nSrcStride * nSrcHeight * 3 /2) { + return -1; + } + + // dst check + if (!pDstVirAddr) { + return -1; + } + + if (nDstSize != nDstStride * nDstHeight * 3 /2) { + return -1; + } + + if (nDstSize < nSrcSize) { + return -1; + } + + if (nDstStride < nSrcStride) { + return -1; + } + + if (nDstHeight != nSrcHeight) { + return -1; + } + + AX_VOID *src = pSrcVirAddr; + AX_VOID *dst = pDstVirAddr; + for(AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { + memcpy(dst, src, nSrcStride); + src += nSrcStride; + dst += nDstStride; + } + + AX_U64 nProcessElasped = get_tick_count() - nStartTime; + + ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, nProcessElasped); + + return 0; +} + +int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) +{ + AX_S32 nReadSize = 0; + + if (pFile) { + if (nPhyAddr != 0 && pVirAddr) { + memset((AX_U8 *)pVirAddr, 0x00, nSize); + nReadSize = fread((AX_U8 *)pVirAddr, 1, nSize, pFile); + } + else { + fseeko(pFile, nSize, SEEK_CUR); + } + } + + return nReadSize; +} + +int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) +{ + SAMPLE_INPUT_FILE_INFO_T stStreamInfo; + SAMPLE_STREAM_BUF_T stStreamBuf; + size_t nReadLen = 0; + AX_U64 streamPhyAddr = 0; + AX_VOID *pStreamVirAddr = NULL; + AX_VDEC_DEC_ONE_FRM_T decOneFrmParam; + size_t nFileSize = 0; + AX_U32 heightAlign = 0; + AX_U32 frmStride = 0; + frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; + heightAlign = ALIGN_UP(nHeight, 16); + AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; + + memset(&stStreamInfo, 0, sizeof(SAMPLE_INPUT_FILE_INFO_T)); + memset(&stStreamBuf, 0, sizeof(SAMPLE_STREAM_BUF_T)); + memset(&decOneFrmParam, 0, sizeof(AX_VDEC_DEC_ONE_FRM_T)); + + rewind(InputFileHandle); + fseek(InputFileHandle, 0, SEEK_END); + nFileSize = (size_t)ftello(InputFileHandle); + + stStreamInfo.fInput = InputFileHandle; + stStreamInfo.sFileSize = nFileSize; + + stStreamBuf.uBufSize = ((AX_U32)nFileSize) > STREAM_BUFFER_MAX_SIZE ? STREAM_BUFFER_MAX_SIZE : nFileSize; + int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, + stStreamBuf.uBufSize, 0x100, (AX_S8 *)"vdec_input_stream"); + if (ret != AX_SUCCESS) { + ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", + stStreamBuf.uBufSize, ret); + return -1; + } + + stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; + stStreamBuf.tBufAddr.u64PhyAddr = streamPhyAddr; + + ret = StreamParserReadFrameJpeg(&stStreamInfo, &stStreamBuf, &nReadLen); + if (0 != ret) { + ALOGE("StreamParserReadFrameJpeg failed! ret = 0x%x", ret); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return -1; + } + + if (!nReadLen) { + ALOGE("read jpeg frame FAILED!\n"); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + ret = AX_ERR_VDEC_STRM_ERROR; + return ret; + } + + decOneFrmParam.stStream.pu8Addr = (AX_U8*)stStreamBuf.tBufAddr.pVirAddr; + decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; + decOneFrmParam.stStream.u32StreamPackLen = (AX_U32)nReadLen; + + decOneFrmParam.stFrame.u64VirAddr[0] = (AX_ULONG)OneYUVDataVir; + decOneFrmParam.stFrame.u64VirAddr[1] = (AX_ULONG)OneYUVDataVir + frmStride * heightAlign; + decOneFrmParam.stFrame.u64PhyAddr[0] = OneYUVDataPhy; + decOneFrmParam.stFrame.u64PhyAddr[1] = OneYUVDataPhy + frmStride * heightAlign; + decOneFrmParam.stFrame.u32FrameSize = nFrameSize; + + ret = AX_VDEC_JpegDecodeOneFrame(&decOneFrmParam); + if (ret != AX_SUCCESS) { + ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", + ret, SampleVdecRetStr(ret)); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return ret; + } + + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + + return 0; +} + +int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr, AX_U32 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + if (f_in == NULL) { + ALOGE("Unable to open input file\n"); + return -1; + } + + /* file i/o pointer to full */ + fseek(f_in, 0L, SEEK_END); + AX_U32 nFileSize = ftell(f_in); + rewind(f_in); + + if (pPhyAddr && ppVirAddr) { + AX_U64 nPhyAddr = 0; + AX_VOID *pVirAddr = NULL; + + // pVirAddr = (AX_VOID *)malloc(nFileSize); + AX_S32 nRet = AX_SYS_MemAlloc(&nPhyAddr, (AX_VOID **)&pVirAddr, nFileSize, 128, (AX_S8 *)"SKEL_TEST"); + + if (nRet != 0) { + fclose(f_in); + ALOGE("AX_SYS_MemAlloc failed"); + return -1; + } + + fread((AX_U8 *)pVirAddr, nFileSize, 1, f_in); + + *pPhyAddr = nPhyAddr; + *ppVirAddr = pVirAddr; + } + + if (pLen) { + *pLen = nFileSize; + } + + fclose(f_in); + + return 0; +} + +int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_U32 nLen) { + AX_S32 s32Ret = AX_SUCCESS; + + AX_U32 frameSize = nLen; + AX_JPEG_ENCODE_ONCE_PARAMS_T stJpegEncodeOnceParam; + memset(&stJpegEncodeOnceParam, 0, sizeof(stJpegEncodeOnceParam)); + + AX_U64 outPhyAddr = 0; + AX_VOID *outVirAddr = NULL; + + s32Ret = AX_SYS_MemAlloc(&outPhyAddr, &outVirAddr, frameSize, 128, (AX_S8 *)"SKEL-VENC"); + if (s32Ret) { + ALOGE("alloc mem err, size(%d).", frameSize); + return -1; + } + + // output + stJpegEncodeOnceParam.u32OutBufSize = frameSize; + stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; + stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; + + // input + stJpegEncodeOnceParam.stJpegParam.u32Qfactor = 90; + stJpegEncodeOnceParam.u32Width = nWidth; + stJpegEncodeOnceParam.u32Height = nHeight; + stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stJpegEncodeOnceParam.u32PicStride[0] = nStride; + stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; + stJpegEncodeOnceParam.u32PicStride[2] = 0; + + stJpegEncodeOnceParam.u64PhyAddr[0] = nPhyAddr; + stJpegEncodeOnceParam.u64PhyAddr[1] = stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64PhyAddr[2] = 0; + + stJpegEncodeOnceParam.u64VirAddr[0] = (AX_ULONG)pVirAddr; + stJpegEncodeOnceParam.u64VirAddr[1] = stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64VirAddr[2] = 0; + + s32Ret = AX_VENC_JpegEncodeOneFrame(&stJpegEncodeOnceParam); + if (AX_SUCCESS != s32Ret) { + ALOGE("AX_VENC_JpegEncodeOneFrame fail, ret=0x%x", s32Ret); + s32Ret = -1; + goto JENC_EXIT; + } + +JENC_EXIT: + if (0 == s32Ret && dstFile) { + FILE *fp_w = fopen(dstFile, "wb"); + + if (fp_w) { + ALOGN("\tWrite new JPG result image to file: %s", dstFile); + fwrite((AX_U8 *)stJpegEncodeOnceParam.pu8Addr, stJpegEncodeOnceParam.u32Len, 1, fp_w); + fclose(fp_w); + } + } + + if (outPhyAddr && (NULL != outVirAddr)) { + AX_SYS_MemFree(outPhyAddr, outVirAddr); + } + + return s32Ret; +} + +AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) { + if (nFrameSeqNum < 1) { + nFrameSeqNum = 1; + } + + if (nSrcFrameRate == nDstFrameRate) { + return AX_FALSE; + } + + if (nDstFrameRate > nSrcFrameRate) { + return AX_FALSE; + } + + if ((nFrameSeqNum * nDstFrameRate / (nSrcFrameRate)) > ((nFrameSeqNum - 1) * nDstFrameRate / (nSrcFrameRate))) { + return AX_FALSE; + } else { + return AX_TRUE; + } +} + +AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { + AX_S32 nRet = 0; + AX_S32 c; + AX_S32 isExit = 0; + AX_BOOL bJpgFile = AX_FALSE; + FILE *InputFileHandle = NULL; + const AX_CHAR *InputFile = NULL; + const AX_CHAR *InputResolution = NULL; + AX_S32 InputFileFrameCnt = 1; + const AX_CHAR *ReWritePath = NULL; + const AX_CHAR *SaveResultPath = NULL; + const AX_CHAR *ModelsPath = NULL; + FILE *fpResultFile = NULL; + AX_S32 nRepeatTimes = 1; + AX_S32 nPPL = AX_SKEL_PPL_HVCP; + AX_S32 nDetectType = 2; + AX_S32 nPushStrategy = 3; + AX_S32 nNpuType = 0; + AX_S32 nInterval = 0; + AX_U32 nStride = 1920; + AX_U32 nWidth = 1920; + AX_U32 nHeight = 1080; + AX_U32 nJencBufSize = 0; + AX_U32 nFrameDepth = 1; + AX_U32 nOneSize = 0; + AX_U64 OneYUVDataPhy = 0; + AX_VOID *OneYUVDataVir = NULL; + AX_U32 nFrameSize = nStride * nHeight * 3 / 2; + AX_U32 nCacheListDepth = 1; + AX_S32 nSrcFrameRate = 25; + AX_S32 nDstFrameRate = 25; + AX_VOID *YUVDataVir = NULL; + AX_U64 YUVDataPhy = 0; + AX_VOID *YUVDataWrVir = NULL; + AX_U64 YUVDataWrPhy = 0; + AX_SKEL_HANDLE pHandle = NULL; + AX_SKEL_FRAME_T stFrame = {0}; + AX_SKEL_RESULT_T *pstResult = NULL; + AX_U64 nFileSize = 0; + AX_U64 nStartTime = 0; + AX_U64 nInitElasped = 0; + AX_U64 nCreateElasped = 0; + AX_U64 nProcessElasped = 0; + AX_U64 nResultElasped = 0; + AX_U64 nResultElaspedMin = (AX_U64)-1; + AX_U64 nResultElaspedMax = 0; + AX_U64 nResultElaspedTotal = 0; + AX_F32 fConfidence = 0.0; + // AX_U32 nHumantracksize = 0; + // AX_U32 nVehicletracksize = 0; + // AX_U32 nCycletracksize = 0; + AX_U64 nSkelFrameId = 1; + +#if defined(SAMPLE_SKEL_BUILD_VERSION) + printf("SKEL sample: %s build: %s %s\n", SAMPLE_SKEL_BUILD_VERSION, __DATE__, __TIME__); +#endif + + while ((c = getopt(argc, argv, "i:r:I:w:o:m:t:d:u:p:j:v:c:N:H:V:C:h::")) != -1) { + isExit = 0; + switch (c) { + case 'i': + InputFile = (const AX_CHAR *)optarg; + break; + case 'r': + InputResolution = (const AX_CHAR *)optarg; + break; + case 'I': + nInterval = atoi(optarg); + break; + case 'w': + ReWritePath = (const AX_CHAR *)optarg; + break; + case 'o': + SaveResultPath = (const AX_CHAR *)optarg; + break; + case 'm': + ModelsPath = (const AX_CHAR *)optarg; + break; + case 't': + nRepeatTimes = atoi(optarg); + break; + case 'd': + nDetectType = atoi(optarg); + break; + case 'u': + nPushStrategy = atoi(optarg); + break; + case 'N': + nNpuType = atoi(optarg); + break; + case 'p': + nPPL = atoi(optarg); + break; + case 'j': + nJencBufSize = atoi(optarg); + break; + case 'c': + fConfidence = atof(optarg); + break; + // case 'H': + // nHumantracksize = atoi(optarg); + // break; + // case 'V': + // nVehicletracksize = atoi(optarg); + // break; + // case 'C': + // nCycletracksize = atoi(optarg); + // break; + case 'v': + log_level = atoi(optarg); + break; + case 'h': + isExit = 1; + break; + case 'D': + nCacheListDepth = atoi(optarg); + break; + default: + isExit = 1; + break; + } + } + + if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) + || (log_level < 0 || log_level >= SKEL_LOG_MAX) + || (fConfidence < 0 || fConfidence > 1) + || (nDetectType < 0 || nDetectType > 2) + || (nPushStrategy < 1 || nPushStrategy > 3) + || (nNpuType > 3)) { + ShowUsage(); + exit(0); + } + + if (InputFile) { + AX_U32 nInputFileLen = strlen(InputFile); + + if (nInputFileLen > 4 && (strcasecmp(&InputFile[nInputFileLen - 4], ".jpg") == 0)) { + bJpgFile = AX_TRUE; + } + } + + if (nRepeatTimes <= 0) { + nRepeatTimes = 1; + } + + if (nInterval < 0) { + nInterval = 0; + } + if (nCacheListDepth == 0) { + nCacheListDepth = 1; + } + nInterval = nInterval * 1000; + + if (access(InputFile, 0) != 0) { + ALOGE("%s not exist", InputFile); + exit(0); + } + + // clear output + if (SaveResultPath) { + RUN_COMMAND("rm -rf %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_BODY_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_VEHICLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_CYCLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_FACE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_PLATE_PATH); + } + + if (ReWritePath) { + RUN_COMMAND("rm -rf %s", ReWritePath); + RUN_COMMAND("mkdir -p %s", ReWritePath); + } + + if (InputResolution) { + AX_CHAR *temp_p = strstr(InputResolution, "x"); + + if (!temp_p || strlen(temp_p) <= 1) { + ShowUsage(); + exit(0); + } + + nWidth = atoi(InputResolution); + nStride = nWidth; + nHeight = atoi(temp_p + 1); + } + + nRet = AX_SYS_Init(); + if (0 != nRet) { + ALOGE("AX_SYS_Init() fail, ret = 0x%x", nRet); + exit(0); + } + + nRet = AX_IVPS_Init(); + + ALOGI("AX_IVPS_Init nRet = 0x%x", nRet); + + AX_VENC_MOD_ATTR_T stVencModAttr; + memset(&stVencModAttr, 0x00, sizeof(stVencModAttr)); + stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; + stVencModAttr.stModThdAttr.u32TotalThreadNum = 1; + stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; + nRet = AX_VENC_Init(&stVencModAttr); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VENC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VENC_Init nRet = 0x%x", nRet); + + nRet = AX_VDEC_Init(NULL); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VDEC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VDEC_Init nRet = 0x%x", nRet); + + // init engine + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); +#if defined(CHIP_AX650) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else if (nNpuType >= 1 && nNpuType <= 4) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_STD; + } + else if (nNpuType >= 5 && nNpuType <= 7) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_BIG_LITTLE; + } +#elif defined(CHIP_AX620E) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_ENABLE; + } +#else + // #error "NO CHIP SELECTED." +#endif + + nRet = AX_ENGINE_Init(&npu_attr); + if (0 != nRet) { + ALOGE("AX_NPU_SDK_EX_Init_with_attr() fail, ret = 0x%x", nRet); + goto EXIT0; + } + + nOneSize = nWidth * nHeight * 3 / 2; + nFrameSize = nStride * nHeight * 3 / 2; + + InputFileHandle = LoadFile(InputFile, &nFileSize); + + if (bJpgFile) { + InputFileFrameCnt = 1; + } + else { + if (!InputFileHandle + || (nFileSize % nOneSize) != 0) { + ALOGE("%s file is not %dx%d", InputFile, nWidth, nHeight); + goto EXIT1; + } + + InputFileFrameCnt = nFileSize / nOneSize; + } + + + if (nWidth%2 == 1 + || nHeight%2 == 1) { + ALOGE("wxh(%dx%d) should be even", nWidth, nHeight); + goto EXIT1; + } + + // YUVDataWrVir = (AX_VOID *)malloc(nFileSize); + nRet = AX_SYS_MemAlloc(&YUVDataWrPhy, &YUVDataWrVir, nFrameSize, 128, (const AX_S8 *)"SKEL_TEST"); + + if (!YUVDataWrVir) { + ALOGE("malloc fail nRet=0x%x", nRet); + goto EXIT1; + } + + if (SaveResultPath) { + AX_CHAR szPath[512] = {0}; + snprintf(szPath, 511, "%s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_LOG_FILE); + fpResultFile = fopen(szPath, "a+"); + + if (!fpResultFile) { + ALOGE("%s file not exist", szPath); + goto EXIT1; + } + } + + nStartTime = get_tick_count(); + + AX_SKEL_INIT_PARAM_T stInitParam = {0}; + + if (ModelsPath) { + stInitParam.pStrModelDeploymentPath = ModelsPath; + } + else { + stInitParam.pStrModelDeploymentPath = "/opt/etc/skelModels"; + } + + nRet = AX_SKEL_Init(&stInitParam); + + nInitElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL init fail, ret = 0x%x", nRet); + + goto EXIT1; + } + + // get version + { + const AX_SKEL_VERSION_INFO_T *pstVersion = NULL; + + nRet = AX_SKEL_GetVersion(&pstVersion); + + if (0 != nRet) { + ALOGI("SKEL get version fail, ret = 0x%x", nRet); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + goto EXIT1; + } + + ALOGI("SKEL version: %s", pstVersion->pstrVersion); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + } + + // get capability + { + const AX_SKEL_CAPABILITY_T *pstCapability = NULL; + + nRet = AX_SKEL_GetCapability(&pstCapability); + + if (0 != nRet) { + ALOGI("SKEL get capability fail, ret = 0x%x", nRet); + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + goto EXIT1; + } + + for (AX_U32 i = 0; i < pstCapability->nPPLConfigSize; i++) { + ALOGI("SKEL capability[%d]: (ePPL: %d, PPLConfigKey: %s)", i, pstCapability->pstPPLConfig[i].ePPL, + pstCapability->pstPPLConfig[i].pstrPPLConfigKey); + } + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + } + + AX_SKEL_HANDLE_PARAM_T stHandleParam = {0}; + + stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; + stHandleParam.nFrameDepth = nFrameDepth; + stHandleParam.nFrameCacheDepth = nCacheListDepth; + stHandleParam.nIoDepth = 0; + stHandleParam.nWidth = nWidth; + stHandleParam.nHeight = nHeight; + + // config settings (if need) + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[16] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; + + // venc_attr_config + // default will be system definition: w*h*3/8 + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVencAttrConfigThreshold = {0}; + if (nJencBufSize > 0) { + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; + stVencAttrConfigThreshold.fValue = (AX_F32) nJencBufSize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stTrackEnableThreshold = {0}; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; + + // detect only (disable track + disable push) + if (nDetectType == 0) { + // track_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + stTrackEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + // detect + track (disable push) + else if (nDetectType == 1) { + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + if (itemIndex > 0) { + stConfig.nSize = itemIndex; + stHandleParam.stConfig = stConfig; + } + + // default NPU + if (nNpuType == 0 || nNpuType == 1) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_NPU_DEFAULT; + } + // STD-NPU + else if (nNpuType == 2) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_1; + } + else if (nNpuType == 3) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_2; + } + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_Create(&stHandleParam, &pHandle); + + nCreateElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Create Handle fail, ret = 0x%x", nRet); + + goto EXIT2; + } + + if (fConfidence == 0) { + fConfidence = 0.5; + } + + // set config + { + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[64] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; +#if 0 + // body_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyMaxTargetCount = {0}; + stBodyMaxTargetCount.fValue = (AX_F32)nHumantracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleMaxTargetCount = {0}; + stVehicleMaxTargetCount.fValue = (AX_F32)nVehicletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleMaxTargetCount = {0}; + stCycleMaxTargetCount.fValue = (AX_F32)nCycletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif +#if 0 + // body_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyConfidence = {0}; + stBodyConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stFaceConfidence = {0}; + stFaceConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleConfidence = {0}; + stVehicleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleConfidence = {0}; + stCycleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPlateConfidence = {0}; + stPlateConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // crop_encoder_qpLevel + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"crop_encoder_qpLevel"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCropEncoderQpLevelThreshold = {0}; + stCropEncoderQpLevelThreshold.fValue = 90; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCropEncoderQpLevelThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // body_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stBodyMinSize = {0}; + stBodyMinSize.nWidth = 0; + stBodyMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // face_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stFaceMinSize = {0}; + stFaceMinSize.nWidth = 0; + stFaceMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // vehicle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stVehicleMinSize = {0}; + stVehicleMinSize.nWidth = 0; + stVehicleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // cycle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stCycleMinSize = {0}; + stCycleMinSize.nWidth = 0; + stCycleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // plate_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stPlateMinSize = {0}; + stPlateMinSize.nWidth = 0; + stPlateMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; +#endif + + // detect_roi_polygon + AX_SKEL_ROI_POLYGON_CONFIG_T stDetectRoi = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; + stDetectRoi.bEnable = AX_FALSE; + stDetectRoi.nPointNum = 4; + AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; + stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); + itemIndex++; + + // push_strategy + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; + AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; + stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; + stPushStrategy.nIntervalTimes = 2000; + stPushStrategy.nPushCounts = 1; + stPushStrategy.bPushSameFrame = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_STRATEGY_T); + itemIndex++; + + //4 crop_encoder +#if 0 + // body_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stBodyCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_crop_encoder"; + stBodyCropEncoderThreshold.fScaleLeft = 0; + stBodyCropEncoderThreshold.fScaleRight = 0; + stBodyCropEncoderThreshold.fScaleTop = 0; + stBodyCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stVehicleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_crop_encoder"; + stVehicleCropEncoderThreshold.fScaleLeft = 0; + stVehicleCropEncoderThreshold.fScaleRight = 0; + stVehicleCropEncoderThreshold.fScaleTop = 0; + stVehicleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stCycleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_crop_encoder"; + stCycleCropEncoderThreshold.fScaleLeft = 0; + stCycleCropEncoderThreshold.fScaleRight = 0; + stCycleCropEncoderThreshold.fScaleTop = 0; + stCycleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stFaceCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_crop_encoder"; + stFaceCropEncoderThreshold.fScaleLeft = 0; + stFaceCropEncoderThreshold.fScaleRight = 0; + stFaceCropEncoderThreshold.fScaleTop = 0; + stFaceCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stPlateCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_crop_encoder"; + stPlateCropEncoderThreshold.fScaleLeft = 0; + stPlateCropEncoderThreshold.fScaleRight = 0; + stPlateCropEncoderThreshold.fScaleTop = 0; + stPlateCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + // push_panorama + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; + AX_SKEL_PUSH_PANORAMA_CONFIG_T stPushPanoramaConfig = {0}; + stPushPanoramaConfig.bEnable = AX_FALSE; + stPushPanoramaConfig.nQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); + itemIndex++; +#if 0 + // push_quality_body + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_body"; + AX_SKEL_ATTR_FILTER_CONFIG_T stBodyAttrFilter = {0}; + stBodyAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_vehicle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_vehicle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stVehicleAttrFilter = {0}; + stVehicleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_cycle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_cycle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stCycleAttrFilter = {0}; + stCycleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_face + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_face"; + AX_SKEL_ATTR_FILTER_CONFIG_T stFaceAttrFilter = {0}; + stFaceAttrFilter.stFaceAttrFilterConfig.nWidth = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.nHeight = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fPitch = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fYaw = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fRoll = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fBlur = 1.0; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceAttrFilter; + itemIndex++; + + // push_quality_plate + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_plate"; + AX_SKEL_ATTR_FILTER_CONFIG_T stPlateAttrFilter = {0}; + stPlateAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; +#endif +//#if 0 + // track_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stTrackEnable = {0}; + stTrackEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; + + //push_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushEnable = {0}; + stPushEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +//#endif +#if 0 + //target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + //push_target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + // push_attr_always + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_attr_always"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushAttrAlways = {0}; + stPushAttrAlways.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushAttrAlways; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +#endif + //analyzer_attr_config + // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; + // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; + // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; + + // stAnalyze.nSize = 1; + // AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_FACE_ATTRIBUTE; + // stAnalyze.peItems = &analyze_attribute; + + // stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stAnalyze; + // stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ANALYZER_CONFIG_T); + // itemIndex++; + + stConfig.nSize = itemIndex; + + nRet = AX_SKEL_SetConfig(pHandle, &stConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_SetConfig, ret = 0x%x", nRet); + + goto EXIT2; + } + } + + // get config + { + const AX_SKEL_CONFIG_T *pstConfig = NULL; + + nRet = AX_SKEL_GetConfig(pHandle, &pstConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_GetConfig, ret = 0x%x", nRet); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + + goto EXIT2; + } + + ParseConfigParam(pstConfig); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + } + + ALOGN("Task infomation:"); + ALOGN("\tInput file: %s", InputFile); + ALOGN("\tInput file resolution: %dx%d", nWidth, nHeight); + ALOGN("\tRepeat times: %d", nRepeatTimes); + ALOGN("SKEL Init Elapse:"); + ALOGN("\tAX_SKEL_Init: %lld ms", nInitElasped); + ALOGN("\tAX_SKEL_Create: %lld ms", nCreateElasped); + + OUTPUT_LOG_SAVE("==============================%s process result:==============================", InputFile); + + if (InputFileFrameCnt > 1) { + InputFileFrameCnt = InputFileFrameCnt + BLACK_VIDEO_FRAME_COUNT; + } + + // create frame mgr + FrameMgrCreate(nFrameSize, nFrameDepth + nCacheListDepth); + + for (AX_U32 nRepeat = 0; nRepeat < nRepeatTimes; nRepeat++) { + ALOGN("#####SKEL Process times: %d", nRepeat + 1); + + OUTPUT_LOG_SAVE("#####SKEL Process times: %d", nRepeat + 1); + + STAT_OBJECT_NUM_T tObjectTrackNum = {0}; + STAT_OBJECT_NUM_T tObjectPushNum = {0}; + + for (AX_U32 nFrameCnt = 0; nFrameCnt < InputFileFrameCnt; nFrameCnt++) { + AX_BLK blkId = AX_INVALID_BLOCKID; + + if (InputFileFrameCnt != 1) { + if (FrameSkipCtrl(nSrcFrameRate, nDstFrameRate, nSkelFrameId)) { + nSkelFrameId ++; + + LoadFileToMem(InputFileHandle, 0, NULL, nOneSize); + + if (nInterval > 0) { + usleep(nInterval); + } + continue; + } + else { + if (!FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId)) { + + ALOGE("FrameMgrGet fail"); + + goto EXIT1; + } + + if (nOneSize != nFrameSize) { + LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nOneSize); + } + } + } else { + FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId); + + if (nRet != 0 + || YUVDataPhy == 0 + || YUVDataVir == NULL) { + ALOGE("Load input file fail"); + goto EXIT1; + } + + if (bJpgFile) { + nRet = DecodeJpeg(InputFileHandle, YUVDataPhy, YUVDataVir, nWidth, nHeight); + if (0 != nRet) { + ALOGE("DecodeJpeg failed!"); + goto EXIT3; + } + } + else { + if (nOneSize != nFrameSize) { + nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, (AX_S8 *)"SKEL_TEST"); + + nRet = LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + nRet = LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nFrameSize); + } + } + } + stFrame.stFrame.u32Width = nWidth; + stFrame.stFrame.u32Height = nHeight; + stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stFrame.stFrame.u32FrameSize = nFrameSize; + stFrame.stFrame.u32PicStride[0] = nStride; + stFrame.stFrame.u32PicStride[1] = nStride; + stFrame.stFrame.u32PicStride[2] = nStride; + stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; + stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; + stFrame.stFrame.u64PhyAddr[2] = 0; + stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); + stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); + stFrame.stFrame.u64VirAddr[2] = 0; + stFrame.stFrame.u32BlkId[0] = blkId; + stFrame.stFrame.u32BlkId[1] = 0; + stFrame.stFrame.u32BlkId[2] = 0; + ALOGN("*****SKEL Frame(%d) Process", nFrameCnt + 1); + + //OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); + + stFrame.nFrameId = nSkelFrameId ++; + nStartTime = get_tick_count(); + + while (1) { + nRet = AX_SKEL_SendFrame(pHandle, &stFrame, 0); + + if (nRet == 0) { + break; + } + + usleep(1000); + } + + FrameMgrRelease(blkId); + + nProcessElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Process fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("SKEL Process Elapse:"); + ALOGN("\tAX_SKEL_SendFrame: %lld ms", nProcessElasped); + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_GetResult(pHandle, &pstResult, -1); + + nResultElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL get result fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("\tAX_SKEL_GetResult: %lld ms", nResultElasped); + + nResultElaspedTotal += nResultElasped; + + if (nProcessElasped + nResultElasped > nResultElaspedMax) { + nResultElaspedMax = nProcessElasped + nResultElasped; + } + + if (nProcessElasped + nResultElasped < nResultElaspedMin) { + nResultElaspedMin = nProcessElasped + nResultElasped; + } + + ALOGN("SKEL Process Result:"); + + ALOGI("\tFrameId: %lld", pstResult->nFrameId); + ALOGI("\tnOriginal WxH: %dx%d", pstResult->nOriginalWidth, pstResult->nOriginalHeight); + + ALOGN("\tObject Num: %d", pstResult->nObjectSize); + + AX_U32 nSkelSize = 0; + AI_Detection_SkelResult_t Skels[SKEL_SAMPLE_OBJECT_SIZE] = {0}; + for (AX_U32 i = 0; i < pstResult->nObjectSize; i++) { + AX_SKEL_OBJECT_ITEM_T *pstItems = &pstResult->pstObjectItems[i]; + + ALOGI("\t\tFrameId: %lld", pstItems->nFrameId); + ALOGI("\t\tTrackId: %lld, TrackState: %d", pstItems->nTrackId, pstItems->eTrackState); + + ALOGN("\t\tRect[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, + pstItems->stRect.fY, pstItems->stRect.fW, + pstItems->stRect.fH, pstItems->fConfidence); + StatTrackMgr(pstItems, &tObjectTrackNum); + StatPushMgr(pstItems, &tObjectPushNum); + + // get detect box only new or update state + if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW + || pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) + && nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { + Skels[nSkelSize].pstrObjectCategory = pstItems->pstrObjectCategory; + Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; + Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; + Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; + Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; + + OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", + i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, pstItems->stRect.fY, + pstItems->stRect.fW, pstItems->stRect.fH, + pstItems->fConfidence); + + ALOGN("\t\t[%d]Point Set Size: %d", i, pstItems->nPointSetSize); + + // point + Skels[nSkelSize].nPointNum = AX_MIN(DETECT_SKEL_POINT_COUNT, pstItems->nPointSetSize); + for (AX_U32 j = 0; j < Skels[i].nPointNum; j++) { + ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; + Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; + + OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", + j, + pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + } + + //face attr + if(nPPL == AX_SKEL_PPL_FACE){ + ALOGN("\t\tFace Attribute:"); + ALOGI("\t\t\tfYaw %f,",pstItems->stFaceAttr.fYaw); + ALOGI("\t\t\tfPitch %f,",pstItems->stFaceAttr.fPitch); + ALOGI("\t\t\tfRoll %f,",pstItems->stFaceAttr.fRoll); + ALOGI("\t\t\tfMask %f,",pstItems->stFaceAttr.fMask); + ALOGI("\t\t\tage %d",pstItems->stFaceAttr.nAge); + ALOGI("\t\t\tgender %d",pstItems->stFaceAttr.nGender); + } + nSkelSize ++; + } + + if (AX_SKEL_TRACK_STATUS_SELECT == pstItems->eTrackState) { + ATTRINFO_T AttrInfo; + memset(&AttrInfo, 0x00, sizeof(AttrInfo)); + AttrParser(pstItems, &AttrInfo); + + if (pstItems->bCropFrame + && pstItems->stCropFrame.pFrameData + && 0 < pstItems->stCropFrame.nFrameDataSize + && SaveResultPath) { + // save attribute + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, + AttrInfo.tFaceInfo.nAge, + AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[FACE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[PLATE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[VEHICLE ATTR] %s", arrDat); + } + else { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write crop jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + + } + + // panora frame + if (pstItems->bPanoraFrame + && pstItems->stPanoraFrame.pFrameData + && 0 < pstItems->stPanoraFrame.nFrameDataSize + && SaveResultPath) { + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write panora jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, pstItems->stPanoraFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + } + } + + // feature + ALOGI("\t\tFeature Size: %d", pstItems->nFeatureSize); + if (pstItems->pstFeatureItem + && 0 < pstItems->nFeatureSize + && SaveResultPath) { + AX_CHAR strFile[256] = {0}; + sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", + SaveResultPath, pstItems->pstrObjectCategory, + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, i); + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", + strFile, + pstItems->pstFeatureItem[0].nValueSize); + fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, pstItems->pstFeatureItem[0].nValueSize, fp_w); + fclose(fp_w); + } + } + + // meta + ALOGI("\t\tMeta Size: %d", pstItems->nMetaInfoSize); + if (pstItems->pstMetaInfo + && 0 < pstItems->nMetaInfoSize) { + for (AX_U32 i = 0; i < pstItems->nMetaInfoSize; i++) { + ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, pstItems->pstMetaInfo[i].pstrValue); + } + } + + // binds + ALOGI("\t\tBind Size: %d", pstItems->nObjectBindSize); + if (pstItems->pstObjectBind + && 0 < pstItems->nObjectBindSize) { + for (AX_U32 i = 0; i < pstItems->nObjectBindSize; i++) { + ALOGI("\t\t[%s] [TrackId] %lld bind to ObjectBind[%d]: %s [TrackId] %lld", + pstItems->pstrObjectCategory, pstItems->nTrackId, i, + pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); + } + } + } + + ALOGI("\tnCacheListSize: %d", pstResult->nCacheListSize); + if (pstResult->pstCacheList) { + for (AX_U32 i = 0; i < pstResult->nCacheListSize; i++) { + ALOGI("\t\tCacheList[%d] FrameId: %lld", i, pstResult->pstCacheList[i].nFrameId); + } + } + + if (ReWritePath && pstResult->nObjectSize > 0) { + memcpy(YUVDataWrVir, YUVDataVir, nFrameSize); + + YUV_IMAGE_T YUVImage = {0}; + YUVImage.pImage = YUVDataWrVir; + YUVImage.nWidth = nWidth; + YUVImage.nHeight = nHeight; + YUVImage.stride = nStride; + YUVImage.nSize = nFrameSize; + YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; + + for (size_t i = 0; i < nSkelSize; i++) { + // draw rect + AX_CHAR ObjectCategory = Skels[i].pstrObjectCategory[0]; + AX_S16 x0 = Skels[i].tBox.fX; + AX_S16 y0 = Skels[i].tBox.fY; + AX_U16 w = Skels[i].tBox.fW; + AX_U16 h = Skels[i].tBox.fH; + YUV_COLOR RectColor = YUV_WHITE; + + switch(ObjectCategory){ + case 'b': //body + RectColor = YUV_WHITE; + break; + case 'v': //vehicle + RectColor = YUV_PURPLE; + break; + case 'c': //cycle + RectColor = YUV_GREEN; + break; + case 'f': //face + RectColor = YUV_YELLOW; + break; + case 'p': //plate + RectColor = YUV_RED; + break; + default : + RectColor = YUV_WHITE; + break; + } + + DrawRect(&YUVImage, x0, y0, w, h, RectColor); + + if (Skels[i].nPointNum > 0) { + // draw point + for (size_t j = 0; j < Skels[i].nPointNum; j++) { + x0 = Skels[i].tPoint[j].fX; + y0 = Skels[i].tPoint[j].fY; + + DrawPoint(&YUVImage, x0, y0, 4, x0 * (4 - 1), y0 * (4 - 1), YUV_DARK_GREEN); + } + } + } + + AX_CHAR arrWrFile[512] = {0}; + sprintf(arrWrFile, "%s_result_%d_%d.jpg", InputFile, nFrameCnt + 1, nRepeat + 1); + + AX_CHAR *fileName = NULL; + AX_U32 fileNameLen = strlen(arrWrFile); + AX_CHAR fullFileName[256] = {0}; + AX_S32 i = fileNameLen - 1; + + if (fileNameLen > 0) { + for (; i >= 0; i --) { + if (arrWrFile[i] == '/') { + break; + } + } + } + + if (i < 0) { + i = 0; + } + else if (i < fileNameLen - 1) { + i ++; + } + + fileName = (AX_CHAR *)&arrWrFile[i]; + + sprintf(fullFileName, "%s/%s", ReWritePath, fileName); + + EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, nFrameSize); + } + + if (pstResult) { + AX_SKEL_Release((AX_VOID *)pstResult); + } + + OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process End (Elasped: %lld ms)", nFrameCnt + 1, nProcessElasped); + + if (nInterval > 0) { + usleep(nInterval); + } + } + + ALOGN("SKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + OUTPUT_LOG_SAVE("\nSKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + ALOGN("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + OUTPUT_LOG_SAVE("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]\n", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + if (InputFileHandle) { + ReLoadFile(InputFileHandle); + } + } + + // if(AX_SUCCESS != nRet){ + // ALOGE("AX_POOL_DestroyPool failed! Error Code:0x%X\n", nRet); + // } + + ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + + OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + +EXIT3: + if (pHandle) { + AX_SKEL_Destroy(pHandle); + } + +EXIT2: + AX_SKEL_DeInit(); + +EXIT1: + AX_ENGINE_Deinit(); + +EXIT0: + // if (YUVDataVir) { + // if (YUVDataPhy == 0) { + // free(YUVDataVir); + // } else { + // AX_SYS_MemFree(YUVDataPhy, YUVDataVir); + // } + // } + if (YUVDataWrVir) { + if (YUVDataWrPhy == 0) { + free(YUVDataWrVir); + } else { + AX_SYS_MemFree(YUVDataWrPhy, YUVDataWrVir); + } + } + + if (OneYUVDataVir) { + if (OneYUVDataPhy == 0) { + free(OneYUVDataVir); + } else { + AX_SYS_MemFree(OneYUVDataPhy, OneYUVDataVir); + } + } + + FrameMgrDestroy(); + + if (fpResultFile) { + fclose(fpResultFile); + } + + AX_VDEC_Deinit(); + + AX_VENC_Deinit(); + + AX_IVPS_Deinit(); + + AX_SYS_Deinit(); + + return (0 != nRet) ? -1 : 0; +} diff --git a/projects/llm_framework/main_skel/src/main.cpp.bak b/projects/llm_framework/main_skel/src/main.cpp.bak new file mode 100644 index 00000000..ff46984d --- /dev/null +++ b/projects/llm_framework/main_skel/src/main.cpp.bak @@ -0,0 +1,525 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "common.hpp" +#include +#include +#include + +#include "../../../../SDK/components/utilities/include/sample_log.h" + +using namespace StackFlows; + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + SLOGW("llm_yolo will be exit!"); + main_exit_flage = 1; +} + +static std::string base_model_path_; +static std::string base_model_config_path_; + +typedef struct { + std::string yolo_model; + std::string model_type = "detect"; + std::vector cls_name; + int img_h = 640; + int img_w = 640; + int cls_num = 80; + float pron_threshold = 0.45f; + float nms_threshold = 0.45; +} yolo_config; + +typedef std::function &data, bool finish)> task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: +public: + yolo_config mode_config_; + std::string model_; + // std::unique_ptr yolo_; + std::string response_format_; + std::vector inputs_; + std::vector image_data_; + bool enoutput_; + bool enstream_; + static int ax_init_flage_; + task_callback_t out_callback_; + + bool parse_config(const nlohmann::json &config_body) + { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } else + throw std::string("error"); + } catch (...) { + SLOGE("setup config_body error"); + return true; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + nlohmann::json file_body; + std::list config_file_paths; + config_file_paths.push_back(std::string("./") + model_ + ".json"); + config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + CONFIG_AUTO_SET(file_body["mode_param"], yolo_model); + CONFIG_AUTO_SET(file_body["mode_param"], img_h); + CONFIG_AUTO_SET(file_body["mode_param"], img_w); + CONFIG_AUTO_SET(file_body["mode_param"], pron_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], nms_threshold); + CONFIG_AUTO_SET(file_body["mode_param"], cls_name); + CONFIG_AUTO_SET(file_body["mode_param"], cls_num); + CONFIG_AUTO_SET(file_body["mode_param"], model_type); + mode_config_.yolo_model = base_model + mode_config_.yolo_model; + // skel_ = std::make_unique(); + // if (0 != skel_->Init(mode_config_.yolo_model.c_str())) { + // SLOGE("Init yolo_model model failed!\n"); + // return -5; + // } + } catch (...) { + SLOGE("config false"); + return -6; + } + return 0; + } + + std::string format_float(double value, int decimal_places) + { + std::ostringstream out; + out << std::fixed << std::setprecision(decimal_places) << value; + return out.str(); + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + bool inference(const std::string &msg) + { + try { + // SLOGI("msg:%s", msg.c_str()); + std::ofstream outFile("output.bin"); + outFile << msg; + outFile.close(); + cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); + if (src.empty()) return true; + SLOGI(""); + std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); + common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); + int ret = -1; + // yolo_->SetInput(image.data(), 0); + // if (0 != yolo_->RunSync()) { + // SLOGE("Run yolo model failed!\n"); + // throw std::string("yolo_ RunSync error"); + // } + // std::vector objects; + // yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, + // mode_config_.pron_threshold, mode_config_.nms_threshold, objects, + // mode_config_.model_type); + std::vector yolo_output; + // for (size_t i = 0; i < objects.size(); i++) { + // const detection::Object &obj = objects[i]; + // nlohmann::json output; + // output["class"] = mode_config_.cls_name[obj.label]; + // output["confidence"] = format_float(obj.prob, 2); + // output["bbox"] = nlohmann::json::array(); + // output["bbox"].push_back(format_float(obj.rect.x, 0)); + // output["bbox"].push_back(format_float(obj.rect.y, 0)); + // output["bbox"].push_back(format_float(obj.rect.x + obj.rect.width, 0)); + // output["bbox"].push_back(format_float(obj.rect.y + obj.rect.height, 0)); + // if (mode_config_.model_type == "segment") output["mask"] = obj.mask_feat; + // if (mode_config_.model_type == "pose") output["kps"] = obj.kps_feat; + // if (mode_config_.model_type == "obb") output["angle"] = obj.angle; + // yolo_output.push_back(output); + // if (out_callback_) out_callback_(yolo_output, false); + // } + if (out_callback_) out_callback_(yolo_output, true); + } catch (...) { + SLOGW("yolo_->Run have error!"); + return true; + } + return false; + } + + void _ax_init() + { + if (!ax_init_flage_) { + int ret = AX_SYS_Init(); + if (0 != ret) { + fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret); + } + // AX_ENGINE_NPU_ATTR_T npu_attr; + // memset(&npu_attr, 0, sizeof(npu_attr)); + // ret = AX_ENGINE_Init(&npu_attr); + if (0 != ret) { + fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret); + } + } + ax_init_flage_++; + } + + void _ax_deinit() + { + if (ax_init_flage_ > 0) { + --ax_init_flage_; + if (!ax_init_flage_) { + // AX_ENGINE_Deinit(); + AX_SYS_Deinit(); + } + } + } + + llm_task(const std::string &workid) + { + _ax_init(); + } + + ~llm_task() + { + _ax_deinit(); + } +}; +int llm_task::ax_init_flage_ = 0; +#undef CONFIG_AUTO_SET + +class llm_yolo : public StackFlow { +private: + int task_count_; + std::unordered_map> llm_task_; + int _load_config() + { + if (base_model_path_.empty()) { + base_model_path_ = sys_sql_select("config_base_mode_path"); + } + if (base_model_config_path_.empty()) { + base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); + } + if (base_model_path_.empty() || base_model_config_path_.empty()) { + return -1; + } else { + SLOGI("llm_yolo::_load_config success"); + return 0; + } + } + +public: + llm_yolo() : StackFlow("yolo") + { + task_count_ = 1; + repeat_event(1000, std::bind(&llm_yolo::_load_config, this)); + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::vector &data, + bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + for (const auto &jsonObj : data) { + data_body["delta"].push_back(jsonObj); + } + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + if (data.empty() || (data == "None")) { + error_body["code"] = -24; + error_body["message"] = "The inference data is empty."; + send("None", "None", error_body, unit_name_); + return; + } + const std::string *next_data = &data; + bool enstream = (object.find("stream") == std::string::npos) ? false : true; + int ret; + std::string tmp_msg1; + if (enstream) { + static std::unordered_map stream_buff; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg1; + } + // must encode base64 + std::string tmp_msg2; + ret = decode_base64((*next_data), tmp_msg2); + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg2; + if (llm_task_obj->inference(*next_data)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, unit_name_); + return -1; + } + + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + SLOGE("setup json format error."); + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + + llm_task_obj->set_output(std::bind(&llm_yolo::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + + for (const auto input : llm_task_obj->inputs_) { + if (input.find("yolo") != std::string::npos) { + llm_channel->subscriber_work_id( + "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + } else if (input.find("sys") != std::string::npos) { + // TODO:... + } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + return 0; + } else { + SLOGE("load_mode Failed"); + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::link:%s", data.c_str()); + int ret = 1; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + if (data.find("yolo") != std::string::npos) { + ret = llm_channel->subscriber_work_id( + "", + std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + llm_task_obj->inputs_.push_back(data); + } else if (data.find("sys") != std::string::npos) { + // TODO:... + } + if (ret) { + error_body["code"] = -20; + error_body["message"] = "link false"; + send("None", "None", error_body, work_id); + return; + } else { + send("None", "None", LLM_NO_ERROR, work_id); + } + } + + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::unlink:%s", data.c_str()); + int ret = 0; + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + auto llm_channel = get_channel(work_id); + llm_channel->stop_subscriber_work_id(data); + auto llm_task_obj = llm_task_[work_id_num]; + for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { + if (*it == data) { + it = llm_task_obj->inputs_.erase(it); + } else { + ++it; + } + } + send("None", "None", LLM_NO_ERROR, work_id); + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::taskinfo:%s", data.c_str()); + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("yolo.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs_"] = llm_task_obj->inputs_; + send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_yolo::exit:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_yolo() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main() +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_yolo llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_skel/src/runner/YuvHandler.c b/projects/llm_framework/main_skel/src/runner/YuvHandler.c new file mode 100644 index 00000000..60d1fb46 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/YuvHandler.c @@ -0,0 +1,269 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include "YuvHandler.h" +#include +#include +#include +#include + +/// +typedef struct _YUV_COLOR_S { + AX_U8 Y; + AX_U8 U; + AX_U8 V; +} YUV_COLOR_S; + +YUV_COLOR_S g_YuvColors[YUV_COLOR_MAX] = { + {0x00, 0x00, 0x00}, // green + {0x00, 0x00, 0xff}, // red + {0x00, 0xff, 0x00}, // blue + {0x00, 0xff, 0xff}, // purple + {0xff, 0x00, 0x00}, // dark green + {0xff, 0x00, 0xff}, // yellow + {0xff, 0xff, 0x00}, // light blue + {0xff, 0xff, 0xff}, // light purple + {0x00, 0x80, 0x80}, // dark black + {0x80, 0x80, 0x80}, // gray + {0xff, 0x80, 0x80} // white +}; + +static AX_VOID DrawPointInternal(YUV_IMAGE_T *pstImage, AX_U8 *y, AX_U8 *u, AX_U8 *v, AX_U16 x0, AX_U16 y0, + YUV_COLOR eColor) { + if (!pstImage) { + return; + } + + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + AX_U32 y_offset = 0; + AX_U32 u_offset = 0; + AX_U32 v_offset = 0; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: // YUV420 I420 + /* YYYY...UUUU...VVVV */ + y_offset = (AX_U32)(y0 * m_stride + x0); + u_offset = (AX_U32)((y0 / 2) * (m_stride / 2) + x0 / 2); + v_offset = u_offset; + + y[y_offset] = g_YuvColors[eColor].Y; + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV420_SEMIPLANAR: // YUV420SP NV12 + /* YYYY...UVUV */ + y_offset = (AX_U32)(y0 * m_stride + x0); + u_offset = (AX_U32)((y0 / 2) * m_stride + x0 / 2 * 2); + v_offset = u_offset + 1; + + if (g_YuvColors[eColor].Y == 0xFF) { + y[y_offset] = g_YuvColors[eColor].Y; + } else { + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + } + break; + + case AX_FORMAT_YUV420_SEMIPLANAR_VU: // YUV420SP NV21 + /* YYYY...VUVU */ + y_offset = (AX_U32)(y0 * m_stride + x0); + v_offset = (AX_U32)((y0 / 2) * m_stride + x0 / 2 * 2); + u_offset = v_offset + 1; + + y[y_offset] = g_YuvColors[eColor].Y; + u[u_offset] = g_YuvColors[eColor].U; + v[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + /* UYVYUYVY */ + u_offset = (AX_U32)(y0 * m_stride * 2 + x0 / 2 * 4); + v_offset = u_offset + 2; + y_offset = u_offset + 1; + y[y_offset] = g_YuvColors[eColor].Y; + y[y_offset + 2] = g_YuvColors[eColor].Y; + y[u_offset] = g_YuvColors[eColor].U; + y[v_offset] = g_YuvColors[eColor].V; + break; + + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + /* YUYVYUYV */ + y_offset = (AX_U32)(y0 * m_stride * 2 + x0 / 2 * 4); + u_offset = y_offset + 1; + v_offset = u_offset + 2; + + y[y_offset] = g_YuvColors[eColor].Y; + y[y_offset + 2] = g_YuvColors[eColor].Y; + y[u_offset] = g_YuvColors[eColor].U; + y[v_offset] = g_YuvColors[eColor].V; + break; + + default: + break; + } +} + +AX_VOID DrawPoint(YUV_IMAGE_T *pstImage, AX_S16 x, AX_S16 y, AX_U8 nScale /* = 1*/, AX_S16 x_offset /* = 0*/, + AX_S16 y_offset /* = 0*/, YUV_COLOR eColor /* = YUV_GREEN*/) { + if (!pstImage) { + return; + } + + AX_U8 *m_pImage = pstImage->pImage; + AX_S16 m_nWidth = pstImage->nWidth; + AX_S16 m_nHeight = pstImage->nHeight; + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + AX_U8 *pY = NULL; + AX_U8 *pU = NULL; + AX_U8 *pV = NULL; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU + m_stride * m_nHeight / 4; + break; + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU; + break; + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + pY = m_pImage; + pU = pY; + pV = pY; + break; + default: + break; + } + + AX_S16 nXStart = 0; + AX_S16 nYStart = 0; + for (AX_U32 hScale = 0; hScale < nScale; hScale++) { + for (AX_U32 wScale = 0; wScale < nScale; wScale++) { + nXStart = x * nScale + hScale - x_offset; + nYStart = y * nScale + wScale - y_offset; + if (nXStart < 0 || nXStart > m_nWidth) { + break; + } + + if (nYStart < 0 || nYStart > m_nHeight) { + break; + } + + DrawPointInternal(pstImage, pY, pU, pV, nXStart, nYStart, eColor); + } + } +} + +AX_VOID DrawLine(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_S16 x1, AX_S16 y1, YUV_COLOR eColor /* = YUV_GREEN*/, + AX_U8 nScale /* = 1*/) + +{ + if (!pstImage) { + return; + } + + AX_U8 *m_pImage = pstImage->pImage; + AX_S16 m_nWidth = pstImage->nWidth; + AX_S16 m_nHeight = pstImage->nHeight; + AX_S16 m_stride = pstImage->stride; + AX_S16 m_eType = pstImage->eType; + + x0 = (x0 < 0) ? 0 : x0; + y0 = (y0 < 0) ? 0 : y0; + x1 = (x1 < 0) ? 0 : x1; + y1 = (y1 < 0) ? 0 : y1; + + x0 = (x0 >= m_nWidth) ? m_nWidth - 1 : x0; + y0 = (y0 >= m_nHeight) ? m_nHeight - 1 : y0; + x1 = (x1 >= m_nWidth) ? m_nWidth - 1 : x1; + y1 = (y1 >= m_nHeight) ? m_nHeight - 1 : y1; + + AX_U16 dx = (x0 > x1) ? (x0 - x1) : (x1 - x0); + AX_U16 dy = (y0 > y1) ? (y0 - y1) : (y1 - y0); + + AX_S16 xstep = (x0 < x1) ? 1 : -1; + AX_S16 ystep = (y0 < y1) ? 1 : -1; + AX_S16 nstep = 0, eps = 0; + + AX_U8 *pY = NULL; + AX_U8 *pU = NULL; + AX_U8 *pV = NULL; + switch (m_eType) { + case AX_FORMAT_YUV420_PLANAR: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU + m_stride * m_nHeight / 4; + break; + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + pY = m_pImage; + pU = m_pImage + m_stride * m_nHeight; + pV = pU; + break; + case AX_FORMAT_YUV422_INTERLEAVED_UYVY: + case AX_FORMAT_YUV422_INTERLEAVED_YUYV: + pY = m_pImage; + pU = pY; + pV = pY; + break; + default: + break; + } + + AX_U16 x = x0; + AX_U16 y = y0; + if (dx > dy) { + while (nstep <= dx) { + if (nScale == 1) { + DrawPointInternal(pstImage, pY, pU, pV, x, y, eColor); + } else { + DrawPoint(pstImage, x, y, nScale, x * (nScale - 1), y * (nScale - 1), eColor); + } + eps += dy; + if ((eps << 1) >= dx) { + y += ystep; + eps -= dx; + } + x += xstep; + nstep++; + } + } else { + while (nstep <= dy) { + if (nScale == 1) { + DrawPointInternal(pstImage, pY, pU, pV, x, y, eColor); + } else { + DrawPoint(pstImage, x, y, nScale, x * (nScale - 1), y * (nScale - 1), eColor); + } + eps += dx; + if ((eps << 1) >= dy) { + x += xstep; + eps -= dy; + } + y += ystep; + nstep++; + } + } +} + +AX_VOID DrawRect(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_U16 w, AX_U16 h, YUV_COLOR eColor /* = YUV_GREEN*/) { + if (w > 0 && h > 0) { + DrawLine(pstImage, x0, y0, x0 + w, y0, eColor, 1); + DrawLine(pstImage, x0, y0, x0, y0 + h, eColor, 1); + DrawLine(pstImage, x0 + w, y0, x0 + w, y0 + h, eColor, 1); + DrawLine(pstImage, x0, y0 + h, x0 + w, y0 + h, eColor, 1); + } +} diff --git a/projects/llm_framework/main_skel/src/runner/YuvHandler.h b/projects/llm_framework/main_skel/src/runner/YuvHandler.h new file mode 100644 index 00000000..ce4f6d4f --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/YuvHandler.h @@ -0,0 +1,46 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _YUV_HANDLER_H_ +#define _YUV_HANDLER_H_ + +#include "ax_global_type.h" +#include "ax_sys_api.h" + +typedef enum _YUV_COLOR { + YUV_GREEN = 0, + YUV_RED, + YUV_BLUE, + YUV_PURPLE, + YUV_DARK_GREEN, + YUV_YELLOW, + YUV_LIGHT_BLUE, + YUV_LIGHT_PURPLE, + YUV_DARK_BLACK, + YUV_GRAY, + YUV_WHITE, + YUV_COLOR_MAX, +} YUV_COLOR; + +typedef struct _YUV_IMAGE_T { + AX_U8 *pImage; + AX_U16 nWidth; + AX_U16 nHeight; + AX_S16 stride; + AX_U32 nSize; + AX_IMG_FORMAT_E eType; +} YUV_IMAGE_T; + +AX_VOID DrawLine(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_S16 x1, AX_S16 y1, YUV_COLOR eColor, AX_U8 nScale); +AX_VOID DrawRect(YUV_IMAGE_T *pstImage, AX_S16 x0, AX_S16 y0, AX_U16 w, AX_U16 h, YUV_COLOR eColor); +AX_VOID DrawPoint(YUV_IMAGE_T *pstImage, AX_S16 x, AX_S16 y, AX_U8 nScale, AX_S16 x_offset, AX_S16 y_offset, + YUV_COLOR eColor); + +#endif /* _YUV_HANDLER_H_ */ diff --git a/projects/llm_framework/main_skel/src/runner/attrParser.cpp b/projects/llm_framework/main_skel/src/runner/attrParser.cpp new file mode 100644 index 00000000..8e386a90 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/attrParser.cpp @@ -0,0 +1,224 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include "attrParser.h" +#include "picojson.h" + +#define PICO_OBJECT get() +#define PICO_OBJECT_SIZE PICO_OBJECT.size() +#define PICO_ARRAY get() +#define PICO_ARRAY_SIZE PICO_ARRAY.size() +#define PICO_VALUE get() +#define PICO_BOOL get() +#define PICO_STRING get() +#define PICO_ROOT obj.PICO_OBJECT + +typedef struct _AI_Face_Attr_t { + AX_F32 fYaw; + AX_F32 fPitch; + AX_F32 fRoll; + AX_F32 fMask; + AX_U8 age; + AX_U8 gender; + AX_F32 fScore; +} AI_Face_Attr_t; + +typedef struct _AI_Plate_Attr_t { + AX_BOOL bExist; + AX_BOOL bValid; + /* + string: + blue + yellow + black + white + green + small_new_energy + large_new_energy + absence + unknown + */ + std::string strPlateColor; + /* + string: + one_row + two_rows + unknown + */ + std::string strPlateType; + /* string: UTF8*/ + std::string strPlateCode; + + _AI_Plate_Attr_t() { + bExist = AX_FALSE; + bValid = AX_FALSE; + strPlateColor = ""; + strPlateType = ""; + strPlateCode = ""; + } +} AI_Plate_Attr_t; + +static AX_VOID FaceAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Face_Attr_t face_attr; + + pAttrInfo->eType = ATTR_TYPE_FACE; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "face_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + // age + face_attr.age = PICO_ROOT["age"].PICO_VALUE; + printf("ageis %d\n",face_attr.age); + // gender + face_attr.gender = PICO_ROOT["gender"].PICO_VALUE; + // yaw + face_attr.fYaw = PICO_ROOT["yaw"].PICO_VALUE; + // pitch + face_attr.fPitch = PICO_ROOT["pitch"].PICO_VALUE; + // roll + face_attr.fRoll = PICO_ROOT["roll"].PICO_VALUE; + // mask + face_attr.fMask = PICO_ROOT["mask"].PICO_VALUE; + // score + face_attr.fScore = PICO_ROOT["score"].PICO_VALUE; + printf("score is %f\n",face_attr.fScore); + } + } +} + +static AX_VOID PlateAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Plate_Attr_t plat_attr; + + pAttrInfo->eType = ATTR_TYPE_PLATE; + + plat_attr.bExist = AX_FALSE; + plat_attr.bValid = AX_FALSE; + plat_attr.strPlateColor = ""; + plat_attr.strPlateType = ""; + plat_attr.strPlateCode = ""; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "plate_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + + plat_attr.bExist = AX_TRUE; + // color + plat_attr.strPlateColor = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("color")) { + plat_attr.strPlateColor = PICO_ROOT["color"].PICO_OBJECT["name"].PICO_STRING; + } + + // style + plat_attr.strPlateType = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("style")) { + plat_attr.strPlateType = PICO_ROOT["style"].PICO_OBJECT["name"].PICO_STRING; + } + + // code + plat_attr.strPlateCode = PICO_ROOT["code_result"].PICO_STRING; + + if (PICO_ROOT["code_killed"].PICO_BOOL) { + plat_attr.bValid = AX_FALSE; + } else { + plat_attr.bValid = AX_TRUE; + } + + pAttrInfo->bExist = AX_TRUE; + + strncpy(pAttrInfo->tPlateInfo.szColor, (const AX_CHAR *)plat_attr.strPlateColor.c_str(), sizeof(pAttrInfo->tPlateInfo.szColor) - 1); + + pAttrInfo->tPlateInfo.bValid = plat_attr.bValid; + strncpy(pAttrInfo->tPlateInfo.szNum, (const AX_CHAR *)plat_attr.strPlateCode.c_str(), sizeof(pAttrInfo->tPlateInfo.szNum) - 1); + } + } +} + +static AX_VOID VehicleAttrResult(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + picojson::value obj; + AI_Plate_Attr_t plat_attr; + + pAttrInfo->eType = ATTR_TYPE_VEHICLE; + + plat_attr.bExist = AX_FALSE; + plat_attr.bValid = AX_FALSE; + plat_attr.strPlateColor = ""; + plat_attr.strPlateType = ""; + plat_attr.strPlateCode = ""; + + for (size_t i = 0; i < pstObjectItems->nMetaInfoSize; i++) { + if (!strcmp(pstObjectItems->pstMetaInfo[i].pstrType, "plate_attr")) { + std::string value = pstObjectItems->pstMetaInfo[i].pstrValue; + std::string strParseRet = picojson::parse(obj, value); + if (!strParseRet.empty() || !obj.is()) { + break; + } + + plat_attr.bExist = AX_TRUE; + // color + plat_attr.strPlateColor = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("color")) { + plat_attr.strPlateColor = PICO_ROOT["color"].PICO_OBJECT["name"].PICO_STRING; + } + + // style + plat_attr.strPlateType = "unknown"; + if (PICO_ROOT.end() != PICO_ROOT.find("style")) { + plat_attr.strPlateType = PICO_ROOT["style"].PICO_OBJECT["name"].PICO_STRING; + } + + // code + plat_attr.strPlateCode = PICO_ROOT["code_result"].PICO_STRING; + + if (PICO_ROOT["code_killed"].PICO_BOOL) { + plat_attr.bValid = AX_FALSE; + } else { + plat_attr.bValid = AX_TRUE; + } + + pAttrInfo->bExist = AX_TRUE; + + strncpy(pAttrInfo->tPlateInfo.szColor, (const AX_CHAR *)plat_attr.strPlateColor.c_str(), sizeof(pAttrInfo->tPlateInfo.szColor) - 1); + + pAttrInfo->tPlateInfo.bValid = plat_attr.bValid; + strncpy(pAttrInfo->tPlateInfo.szNum, (const AX_CHAR *)plat_attr.strPlateCode.c_str(), sizeof(pAttrInfo->tPlateInfo.szNum) - 1); + } + } +} + +AX_VOID AttrParser(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo) { + if (!pstObjectItems || !pAttrInfo) { + return; + } + + pAttrInfo->bExist = AX_FALSE; + pAttrInfo->eType = ATTR_TYPE_BUTT; + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (strObjectCategory == "face") { + FaceAttrResult(pstObjectItems, pAttrInfo); + } + else if (strObjectCategory == "plate") { + PlateAttrResult(pstObjectItems, pAttrInfo); + } + else if (strObjectCategory == "vehicle") { + VehicleAttrResult(pstObjectItems, pAttrInfo); + } +} diff --git a/projects/llm_framework/main_skel/src/runner/attrParser.h b/projects/llm_framework/main_skel/src/runner/attrParser.h new file mode 100644 index 00000000..60646ef1 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/attrParser.h @@ -0,0 +1,60 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _ATTR_PARSER_H_ +#define _ATTR_PARSER_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum _ATTR_TYPE_E { + ATTR_TYPE_BODY = 0, + ATTR_TYPE_VEHICLE, + ATTR_TYPE_CYCLE, + ATTR_TYPE_FACE, + ATTR_TYPE_PLATE, + ATTR_TYPE_BUTT +} ATTR_TYPE_E; + +typedef struct _FACEINFO_T { + AX_U8 nGender; /* 0-female, 1-male */ + AX_U8 nAge; + AX_CHAR szMask[32]; + AX_CHAR szGender[32]; +} FACEINFO_T; + +typedef struct _PLATEINFO_T { + AX_BOOL bValid; + AX_CHAR szNum[16]; + AX_CHAR szColor[32]; +} PLATEINFO_T; + +typedef struct _ATTRINFO_T { + ATTR_TYPE_E eType; /* ATTR_TYPE_E */ + AX_BOOL bExist; + + union + { + FACEINFO_T tFaceInfo; + PLATEINFO_T tPlateInfo; + }; +} ATTRINFO_T; + +extern AX_VOID AttrParser(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, ATTRINFO_T *pAttrInfo); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c b/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c new file mode 100755 index 00000000..4fd522e2 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/common_vdec_utils.c @@ -0,0 +1,2233 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "common_vdec_utils.h" +#include "ax_vdec_api.h" +#include "common_arg_parse.h" +#include "common_vdec_api.h" + +#define AX_DEC_RET_STR_CASE(s32Ret) case (s32Ret): return(#s32Ret) + +const char *SampleVdecRetStr(AX_S32 s32Ret) +{ + switch (s32Ret) { + AX_DEC_RET_STR_CASE(AX_SUCCESS); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_INVALID_GRPID); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_INVALID_CHNID); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_ILLEGAL_PARAM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NULL_PTR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BAD_ADDR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_SYS_NOTREADY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUSY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_INIT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_CONFIG); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_SUPPORT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOT_PERM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_EXIST); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_UNEXIST); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOMEM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NOBUF); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUF_EMPTY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_BUF_FULL); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_QUEUE_EMPTY); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_QUEUE_FULL); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_TIMED_OUT); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_FLOW_END); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_RUN_ERROR); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_STRM_ERROR); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_UNEXIST); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_ILLEGAL_PARAM); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_NOT_SUPPORT); + AX_DEC_RET_STR_CASE(AX_ERR_POOL_NOT_PERM); + AX_DEC_RET_STR_CASE(AX_ERR_VDEC_NEED_REALLOC_BUF); + default: + SAMPLE_CRIT_LOG("Unknown return code. 0x%x", s32Ret); + return("unknown code."); + } +} + +typedef struct +{ + const uint8_t * data; // sps + int count; /* in parse */ + int index; /* in parse */ +} PARSE_STATE_T; + + + +#define PARSE_INIT(data, bytes) {(data), 8*(bytes), 0} + +#define PARSE_EOF(parse) ((parse)->index >= (parse)->count) + +static inline void __parse_init(PARSE_STATE_T *parse, const uint8_t *data, int bytes) +{ + parse->data = data; + parse->count = 8 * bytes; + parse->index = 0; +} + +static inline int __parse_get_bit(PARSE_STATE_T *parse) +{ + if (parse->index >= parse->count) + return 1; /* -> no infinite colomb's ... */ + + int bit = (parse->data[parse->index >> 3] >> (7 - (parse->index & 7))) & 1; + parse->index++; + return bit; +} + +static inline uint32_t __parse_get_bits(PARSE_STATE_T * parse, uint32_t cnt) +{ + uint32_t bit = 0; + while (cnt--) + bit = bit | (__parse_get_bit(parse) << cnt); + + return bit; +} + +#define PARSE_SKIP_BIT(parse) __parse_skip_bits(parse, 1) + +static inline void __parse_skip_bits(PARSE_STATE_T *parse, int cnt) +{ + parse->index += cnt; +} + +#define PARSE_GET_U8(parse) __parse_get_bits(parse, 8) +#define PARSE_GET_U16(parse) ((__parse_get_bits(parse, 8)<<8) | __parse_get_bits(parse, 8)) + +static inline uint32_t __parse_get_ue_golomb(PARSE_STATE_T * parse) +{ + int cnt = 0; + while (!__parse_get_bit(parse) && cnt < 32) + cnt++; + return cnt ? ((1 << cnt) - 1) + __parse_get_bits(parse, cnt) : 0; +} + +static inline int32_t __parse_get_se_golomb(PARSE_STATE_T * parse) +{ + uint32_t r = __parse_get_ue_golomb(parse) + 1; + return (r & 1) ? -(r >> 1) : (r >> 1); +} + +static inline void __parse_skip_golomb(PARSE_STATE_T * parse) +{ + int cnt = 0; + while (!__parse_get_bit(parse) && cnt < 32) + cnt++; + __parse_skip_bits(parse, cnt); +} + +#define PARSE_SKIP_UE_GOLOMB(parse) __parse_skip_golomb(parse) +#define PARSE_SKIP_SE_GOLOMB(parse) __parse_skip_golomb(parse) + +static const MPEG_RATIONAL_T aspect_ratios[] = +{ +/* page 213: */ +/* 0: unknown */ + {0, 1}, +/* 1...16: */ + {1, 1}, {12, 11 }, {10, 11}, {16, 11}, + {40, 33}, {24, 11}, {20, 11}, {32, 11}, + {80, 33}, {18, 11}, {15, 11}, {64, 33}, + {160, 99}, {4, 3}, {3, 2}, {2, 1} +}; + +int h264_parse_sps(const uint8_t *buf, int len, SAMPLE_H264_SPS_DATA_T *sps) +{ + int findSPS = -1; + int pos = 0; + + if (buf == NULL) { + SAMPLE_CRIT_LOG("buf == NULL"); + return -1; + } + + if (sps == NULL) { + SAMPLE_CRIT_LOG("sps == NULL"); + return -1; + } + + if (len < 16) { + SAMPLE_CRIT_LOG("len:%d < 16", len); + return -1; + } + + for (pos = 0; pos < len - 6; pos++) { + if ((buf[pos] == 0) && (buf[pos + 1] == 0) + && (buf[pos + 2] == 0x1) && ((buf[pos + 3] & 0x1f) == 0x7)) { + len -= (pos + 4); + buf += (pos + 4); + findSPS = AX_SUCCESS; + break; + } + } + + if (findSPS != AX_SUCCESS) { + SAMPLE_LOG_TMP("findSPS failed! Failed to parse width and height.\n" + "You can configure the '--res' parameter in case the program runs out of memory and cannot decode"); + return findSPS; + } + + PARSE_STATE_T parse = PARSE_INIT(buf, len); + int profile_idc, pic_order_cnt_type; + int frame_mbs_only; + int i, j; + + profile_idc = PARSE_GET_U8(&parse); + sps->profile = profile_idc; + SAMPLE_LOG_N("H.264 SPS: profile_idc %d", profile_idc); + /* constraint_set0_flag = __parse_get_bit(parse); */ + /* constraint_set1_flag = __parse_get_bit(parse); */ + /* constraint_set2_flag = __parse_get_bit(parse); */ + /* constraint_set3_flag = __parse_get_bit(parse); */ + /* reserved = __parse_get_bits(parse,4); */ + sps->level = PARSE_GET_U8(&parse); + __parse_skip_bits(&parse, 8); + PARSE_SKIP_UE_GOLOMB(&parse); /* seq_parameter_set_id */ + if (profile_idc >= 100) { + if (__parse_get_ue_golomb(&parse) == 3) {/* chroma_format_idc */ + PARSE_SKIP_BIT(&parse); /* residual_colour_transform_flag */ + } + PARSE_SKIP_UE_GOLOMB(&parse); /* bit_depth_luma - 8 */ + PARSE_SKIP_UE_GOLOMB(&parse); /* bit_depth_chroma - 8 */ + PARSE_SKIP_BIT(&parse); /* transform_bypass */ + if (__parse_get_bit(&parse)) { /* seq_scaling_matrix_present */ + for (i = 0; i < 8; i++) { + if (__parse_get_bit(&parse)) { + /* seq_scaling_list_present */ + int last = 8, next = 8, size = (i < 6) ? 16 : 64; + for (j = 0; j < size; j++) { + if (next) { + next = (last + __parse_get_se_golomb(&parse)) & 0xff; + } + last = next ? next : last; + } + } + } + } + } + + PARSE_SKIP_UE_GOLOMB(&parse); /* log2_max_frame_num - 4 */ + pic_order_cnt_type = __parse_get_ue_golomb(&parse); + if (pic_order_cnt_type == 0) { + PARSE_SKIP_UE_GOLOMB(&parse); /* log2_max_poc_lsb - 4 */ + } + else if (pic_order_cnt_type == 1) { + PARSE_SKIP_BIT(&parse); /* delta_pic_order_always_zero */ + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_non_ref_pic */ + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_top_to_bottom_field */ + j = __parse_get_ue_golomb(&parse); /* num_ref_frames_in_pic_order_cnt_cycle */ + for (i = 0; i < j; i++) { + PARSE_SKIP_SE_GOLOMB(&parse); /* offset_for_ref_frame[i] */ + } + + } + PARSE_SKIP_UE_GOLOMB(&parse); /* ref_frames */ + PARSE_SKIP_BIT(&parse); /* gaps_in_frame_num_allowed */ + sps->width /* mbs */ = __parse_get_ue_golomb(&parse) + 1; + sps->height /* mbs */ = __parse_get_ue_golomb(&parse) + 1; + frame_mbs_only = __parse_get_bit(&parse); + SAMPLE_LOG_N("H.264 SPS: pic_width: %u mbs", (unsigned)sps->width); + SAMPLE_LOG_N("H.264 SPS: pic_height: %u mbs", (unsigned)sps->height); + SAMPLE_LOG_N("H.264 SPS: frame only flag: %d", frame_mbs_only); + + sps->width *= 16; + sps->height *= 16 * (2 - frame_mbs_only); + + if (!frame_mbs_only) + if (__parse_get_bit(&parse)) /* mb_adaptive_frame_field_flag */ + SAMPLE_LOG_N("H.264 SPS: MBAFF"); + PARSE_SKIP_BIT(&parse); /* direct_8x8_inference_flag */ + if (__parse_get_bit(&parse)) { + /* frame_cropping_flag */ + uint32_t crop_left = __parse_get_ue_golomb(&parse); + uint32_t crop_right = __parse_get_ue_golomb(&parse); + uint32_t crop_top = __parse_get_ue_golomb(&parse); + uint32_t crop_bottom = __parse_get_ue_golomb(&parse); + SAMPLE_LOG_N("H.264 SPS: cropping %d %d %d %d", + crop_left, crop_top, crop_right, crop_bottom); + + sps->width -= 2 * (crop_left + crop_right); + if (frame_mbs_only) + sps->height -= 2 * (crop_top + crop_bottom); + else + sps->height -= 4 * (crop_top + crop_bottom); + } + + /* VUI parameters */ + sps->pixel_aspect.num = 0; + if (__parse_get_bit(&parse)) { + /* vui_parameters_present flag */ + if (__parse_get_bit(&parse)) { + /* aspect_ratio_info_present */ + uint32_t aspect_ratio_idc = PARSE_GET_U8(&parse); + SAMPLE_LOG_N("H.264 SPS: aspect_ratio_idc %d", aspect_ratio_idc); + + if (aspect_ratio_idc == 255 /* Extended_SAR */) { + sps->pixel_aspect.num = PARSE_GET_U16(&parse); /* sar_width */ + sps->pixel_aspect.den = PARSE_GET_U16(&parse); /* sar_height */ + SAMPLE_LOG_N("H.264 SPS: -> sar %dx%d", sps->pixel_aspect.num, sps->pixel_aspect.den); + } else { + if (aspect_ratio_idc < sizeof(aspect_ratios) / sizeof(aspect_ratios[0])) { + sps->pixel_aspect = aspect_ratios[aspect_ratio_idc]; + SAMPLE_LOG_N("H.264 SPS: -> aspect ratio %d / %d", sps->pixel_aspect.num, sps->pixel_aspect.den); + } else { + SAMPLE_LOG_N("H.264 SPS: aspect_ratio_idc out of range !"); + } + } + } + } + + SAMPLE_LOG_N("H.264 SPS: -> video size %dx%d, aspect %d:%d", + sps->width, sps->height, sps->pixel_aspect.num, sps->pixel_aspect.den); + + return findSPS; +} + + + +AX_PAYLOAD_TYPE_E DistinguishVideoType(const AX_U8 *stream, AX_U64 len) +{ + AX_U64 ReadLen = 0; + AX_U64 i = 0; + const AX_U8 *pu8Buf = stream; + + if (stream == NULL) { + SAMPLE_CRIT_LOG(" stream == NULL"); + return PT_BUTT; + } + + ReadLen = len; + + if (ReadLen <= 6) { + SAMPLE_CRIT_LOG(" len:%lld ReadLen:%lld", len, ReadLen); + return PT_BUTT; + } + + for (i = 0; i < ReadLen - 6; i++) + { + if ((pu8Buf[i+0] == 0) && (pu8Buf[i+1] == 0) && (pu8Buf[i+2] == 1)) { + if ((pu8Buf[i+3] & 0x1f) == 0x07) { + return PT_H264; + } + } + + if ((pu8Buf[i+0] == 0) && (pu8Buf[i+1] == 0) && (pu8Buf[i+2] == 0) && (pu8Buf[i+3] == 1)) { + if ((pu8Buf[i+4] == 0x40) && (pu8Buf[i+5] == 0x01)) { + return PT_H265; + } + } + } + + return PT_BUTT; +} + +static AX_S32 __MD5SumValidOnly(AX_U8 *p_lu, AX_U8 *p_ch, + AX_U32 coded_width, AX_U32 coded_height, + AX_U32 coded_width_ch, AX_U32 coded_height_ch, + AX_U32 pic_stride, AX_U32 pic_stride_ch, + AX_U32 planar, AX_U32 frame_number, char *md5_str) +{ +#if 0 + + unsigned char digest[16]; + MD5_CTX ctx; + int i = 0; + MD5_Init(&ctx); + AX_U8 *p_yuv = p_lu; + if (p_yuv) { + for (i = 0; i < coded_height; i++) { + MD5_Update(&ctx, p_yuv, coded_width); + p_yuv += pic_stride; + } + } + p_yuv = p_ch; + if (p_yuv) { + if (!planar) { + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch); + p_yuv += pic_stride; + } + } else { + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch / 2); + p_yuv += pic_stride_ch; + } + for (i = 0; i < coded_height_ch; i++) { + MD5_Update(&ctx, p_yuv, coded_width_ch / 2); + p_yuv += pic_stride_ch; + } + } + } + MD5_Final(digest, &ctx); + /* fprintf(f_out, "FRAME %d: ", frame_number);*/ + for (i = 0; i < 16; i++) { + snprintf(md5_str + i * 2, 2 + 1, "%02x", digest[i]); + } +#endif + return 0; +} + +int OutputFileCheckMD5(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, char *md5_str) +{ + AX_S32 sRet; + AX_S32 s32Ret = 0; + AX_VOID *pLumaVirAddr; + AX_VOID *pChromaVirAddr; + AX_U32 lumaMapSize; + AX_U32 chromaMapSize; + + if (frameInfo == NULL) { + s32Ret = AX_ERR_VDEC_NULL_PTR; + SAMPLE_CRIT_LOG("frameInfo == NULL\n"); + goto ERR_RET; + } + + if (md5_str == NULL) { + s32Ret = AX_ERR_VDEC_NULL_PTR; + SAMPLE_CRIT_LOG("md5_str == NULL\n"); + goto ERR_RET; + } + + if ((frameInfo->stVFrame.u64PhyAddr[0] == 0) || (frameInfo->stVFrame.u32PicStride[0] == 0)) { + SAMPLE_CRIT_LOG("VdGrp=%d, stVFrame.u64PhyAddr[0]:0x%llx stVFrame.u32PicStride[0]:%d\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0], frameInfo->stVFrame.u32PicStride[0]); + s32Ret = 0; + goto ERR_RET; + } + + lumaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16); + + pLumaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[0], lumaMapSize); + if (!pLumaVirAddr) { + s32Ret = AX_ERR_VDEC_BAD_ADDR; + SAMPLE_CRIT_LOG("AX_SYS_Mmap luma FAILED\n"); + goto ERR_RET; + } + + SAMPLE_LOG("AX_SYS_Mmap luma success, pLumaVirAddr=%p,lumaMapSize=%d\n", + pLumaVirAddr, lumaMapSize); + + chromaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16) / 2; + pChromaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[1], chromaMapSize); + if (!pChromaVirAddr) { + s32Ret = AX_ERR_VDEC_BAD_ADDR; + SAMPLE_CRIT_LOG("AX_SYS_Mmap chroma FAILED\n"); + goto ERR_RET_CHROMA; + } + + SAMPLE_LOG("AX_SYS_Mmap chroma success, pChromaVirAddr=%p,chromaMapSize=%d\n", + pChromaVirAddr, chromaMapSize); + + AX_VOID *p_lu = pLumaVirAddr; + AX_VOID *p_ch = pChromaVirAddr; + AX_U32 coded_width = frameInfo->stVFrame.u32Width; + AX_U32 coded_height = frameInfo->stVFrame.u32Height; + AX_U32 pic_stride = frameInfo->stVFrame.u32PicStride[0]; + AX_U32 coded_width_ch = frameInfo->stVFrame.u32Width; + AX_U32 coded_h_ch = frameInfo->stVFrame.u32Height / 2; + AX_U32 pic_stride_ch = frameInfo->stVFrame.u32PicStride[1]; + s32Ret = __MD5SumValidOnly(p_lu, p_ch, coded_width, coded_height, coded_width_ch, coded_h_ch, + pic_stride, pic_stride_ch, 0, 0, md5_str); + if (s32Ret) { + SAMPLE_CRIT_LOG("__MD5SumValidOnly FAILED! s32Ret:0x%x", s32Ret); + goto ERR_RET_LUMA; + } + +ERR_RET_LUMA: + if (pLumaVirAddr) { + sRet = AX_SYS_Munmap(pLumaVirAddr, lumaMapSize); + if (sRet) { + s32Ret = sRet; + SAMPLE_CRIT_LOG("AX_SYS_Munmap luma FAILED,sRet=0x%x\n", sRet); + } else { + SAMPLE_LOG("AX_SYS_Munmap luma success,pLumaVirAddr=%p,lumaMapSize=%d\n", + pLumaVirAddr, lumaMapSize); + } + } +ERR_RET_CHROMA: + if (pChromaVirAddr) { + sRet = AX_SYS_Munmap(pChromaVirAddr, chromaMapSize); + if (sRet) { + s32Ret = sRet; + SAMPLE_CRIT_LOG("AX_SYS_Munmap chroma FAILED,sRet=0x%x\n", sRet); + } else { + SAMPLE_LOG("AX_SYS_Munmap chroma success,pChromaVirAddr=%p,chromaMapSize=%d\n", + pChromaVirAddr, chromaMapSize); + } + } +ERR_RET: + return s32Ret; +} + + +AX_S32 LoadOneFileToMem(const AX_CHAR *ps8File, AX_U8 **ppu8Mem, size_t *pLen) +{ + AX_S32 res = 0; + AX_U64 tmp_size = 0; + AX_U64 read_size = 0; + FILE *fInput = NULL; + + if (ps8File == NULL) { + SAMPLE_CRIT_LOG("ps8File == NULL\n"); + goto ERR_RET; + } + + if (ppu8Mem == NULL) { + SAMPLE_CRIT_LOG("ppu8Mem == NULL\n"); + goto ERR_RET; + } + + if (pLen == NULL) { + SAMPLE_CRIT_LOG("pLen == NULL\n"); + goto ERR_RET; + } + + fInput = fopen(ps8File, "rb"); + if (fInput == NULL) { + SAMPLE_CRIT_LOG("Unable to open input file:%s\n", ps8File); + res = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + /* file i/o pointer to full */ + res = fseek(fInput, 0L, SEEK_END); + if (res) { + SAMPLE_CRIT_LOG("fseek FAILED! ret:%d\n", res); + goto ERR_RET; + } + + *pLen = ftello(fInput); + rewind(fInput); + + tmp_size = sizeof(AX_U8) * (*pLen); + *ppu8Mem = (AX_U8 *)malloc(tmp_size); + if (*ppu8Mem == NULL) { + SAMPLE_CRIT_LOG("malloc tmp_size:0x%llx FAILED!", tmp_size); + res = AX_ERR_VDEC_NOMEM; + goto ERR_RET_CLOSE; + } + + read_size = *pLen; + tmp_size = fread(*ppu8Mem, sizeof(AX_U8), read_size, fInput); + /* read input stream from file to buffer and close input file */ + if (tmp_size != read_size) { + SAMPLE_CRIT_LOG("fread FAILED! tmp_size:0x%llx != read_size:0x%llx\n", tmp_size, read_size); + res = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET_FREE; + } + +ERR_RET_FREE: + free((AX_VOID *)*ppu8Mem); + *ppu8Mem = NULL; +ERR_RET_CLOSE: + res = fclose(fInput); + if (res) { + SAMPLE_CRIT_LOG("fclose FAILED! ret:%d\n", res); + } +ERR_RET: + return res; +} + +FILE *OutputFileOpen(AX_CHAR **ppOutputFile, const SAMPLE_VDEC_OUTPUT_INFO_T *pInfo) +{ + FILE *fp_out = NULL; + int ret = 0; + AX_CHAR *file_path = NULL; + AX_CHAR *file_name = NULL; + AX_U32 slen = 0; + AX_CHAR *pFileOut = NULL; + + if (ppOutputFile == NULL) { + SAMPLE_CRIT_LOG("ppOutputFile == NULL"); + return NULL; + } + + if (*ppOutputFile == NULL) { + pFileOut = calloc(1, AX_VDEC_FILE_PATH_LEN); + if (pFileOut == NULL) { + SAMPLE_CRIT_LOG("malloc %d Bytes FAILED!\n", AX_VDEC_FILE_PATH_LEN); + return NULL; + } + *ppOutputFile = pFileOut; + + if (pInfo == NULL) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "%s", "./out.yuv"); + } else { + if (pInfo->bOneShot) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "oneShot_format%d_w_%d_h_%d.yuv", + pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } else { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "group%d_format%d_w_%d_h_%d.yuv", + pInfo->VdGrp, pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } + } + + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x pFileOut:%s AX_VDEC_FILE_NAME_LEN:%d\n", + ret, pFileOut, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_OUTPUTFILE; + } + + *ppOutputFile = pFileOut; + } else { + pFileOut = *ppOutputFile; + } + + SAMPLE_LOG("output_file_path >>>> pFileOut:%s", pFileOut); + slen = strlen(pFileOut); + ret = strncmp (pFileOut + slen - 1, "/", 1); + if (ret == 0) { + file_path = malloc(AX_VDEC_FILE_PATH_LEN); + if (NULL == file_path) { + SAMPLE_CRIT_LOG("malloc size:%d FAILED!", AX_VDEC_FILE_PATH_LEN); + goto ERR_RET_OUTPUTFILE; + } + + file_name = malloc(AX_VDEC_FILE_NAME_LEN); + if (NULL == file_name) { + SAMPLE_CRIT_LOG("malloc size:%d FAILED!", AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILEPATH; + } + + if (pInfo == NULL) { + ret = snprintf(file_name, AX_VDEC_FILE_NAME_LEN, "%s", "out.yuv"); + } else { + if (pInfo->bOneShot) { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "oneShot_format%d_w_%d_h_%d.yuv", + pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } else { + ret = snprintf(pFileOut, AX_VDEC_FILE_NAME_LEN, "group%d_format%d_w_%d_h_%d.yuv", + pInfo->VdGrp, pInfo->enImgFormat, pInfo->u32Width, pInfo->u32Height); + } + } + + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x file_name:%s AX_VDEC_FILE_NAME_LEN:%d\n", + ret, file_name, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + + slen = strlen(file_name); + if (slen >= AX_VDEC_FILE_NAME_LEN) { + SAMPLE_CRIT_LOG("slen:%d >= AX_VDEC_FILE_NAME_LEN:%d\n", + slen, AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + + ret = snprintf(file_path, AX_VDEC_FILE_PATH_LEN, "%s%s", + pFileOut, file_name); + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x, pFileOut:%s, file_name:%s, AX_VDEC_FILE_PATH_LEN:%d\n", + ret, pFileOut, file_name, AX_VDEC_FILE_PATH_LEN); + goto ERR_RET_FILENAME; + } + + free(file_name); + free(pFileOut); + *ppOutputFile = file_path; + pFileOut = *ppOutputFile; + } + + slen = strlen(pFileOut); + ret = strncmp (pFileOut + slen - 1, "/", 1); + if (ret == 0) { + SAMPLE_CRIT_LOG("Please check pFileOut:%s, the last character cannot be '/' \n", pFileOut); + goto ERR_RET_FILENAME; + } + + if (access(pFileOut, F_OK) == 0) { + char new_name[AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN]; + memset(new_name, 0, sizeof(new_name)); + + struct timeval current_tv; + gettimeofday(¤t_tv, NULL); + + ret = sprintf(new_name, "%s_%lds_%ldus.bak", pFileOut, current_tv.tv_sec, current_tv.tv_usec); + if (ret < 0) { + SAMPLE_CRIT_LOG("snprintf FAILED! ret:0x%x, pFileOut:%s, AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN:%d\n", + ret, pFileOut, AX_VDEC_FILE_PATH_LEN + AX_VDEC_FILE_NAME_LEN); + goto ERR_RET_FILENAME; + } + // SAMPLE_LOG("new_name:%s ", new_name); + if (chmod(pFileOut, 0x777) != 0) { + SAMPLE_CRIT_LOG("chmod:%s FAILED", pFileOut); + } + + if (rename(pFileOut, new_name) == 0) { + SAMPLE_LOG("rename(pFileOut:%s, new_name:%s) success", + pFileOut, new_name); + } else { + SAMPLE_CRIT_LOG("rename(pFileOut:%s, new_name:%s) FAILED)", pFileOut, new_name); + } + } + + if (access(pFileOut, F_OK) == 0) { + if (chmod(pFileOut, 0x777) != 0) { + SAMPLE_CRIT_LOG("chmod:%s ", pFileOut); + } + + if (remove(pFileOut) == 0) { + if (access(pFileOut, F_OK) == 0) { + SAMPLE_CRIT_LOG("remove(pFileOut:%s FAIL)", pFileOut); + } + } else { + SAMPLE_CRIT_LOG("remove(pFileOut:%s FAIL)", pFileOut); + } + } + + fp_out = fopen(pFileOut, "w"); + if (fp_out == NULL) { + SAMPLE_CRIT_LOG("VdGrp=%d, fopen pFileOut:%s FAILED!", + pInfo->VdGrp, pFileOut); + } + + SAMPLE_LOG("output file name:%s, open fp_out:%p\n", pFileOut, fp_out); + return fp_out; + +ERR_RET_FILENAME: + if (file_name != NULL) { + free(file_name); + } +ERR_RET_FILEPATH: + if (file_path != NULL) { + free(file_path); + } +ERR_RET_OUTPUTFILE: + if (*ppOutputFile != NULL) { + free(*ppOutputFile); + *ppOutputFile = NULL; + } + + return NULL; +} + +static AX_S32 OutputSaveYUVFile(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, + FILE *fp_out, AX_CHAR *pOutputFilePath) +{ + int ret = 0; + AX_S32 sRet = 0; + AX_U32 i; + AX_VOID *p_lu = NULL; + AX_VOID *p_ch = NULL; + + AX_S32 s32Ret = 0; + AX_VOID *pLumaVirAddr = NULL; + AX_VOID *pChromaVirAddr = NULL; + AX_U32 lumaMapSize = 0; + AX_U32 chromaMapSize = 0; + int tmp_size = 0; + + if (NULL == frameInfo || NULL == fp_out) { + SAMPLE_CRIT_LOG("VdGrp=%d, NULL == frameInfo || NULL == fp_out\n", VdGrp); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if ((frameInfo->stVFrame.u64PhyAddr[0] == 0) || (frameInfo->stVFrame.u32PicStride[0] == 0)) { + SAMPLE_LOG("VdGrp=%d, stVFrame.u64PhyAddr[0]:0x%llx stVFrame.u32PicStride[0]:%d\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0], frameInfo->stVFrame.u32PicStride[0]); + s32Ret = 0; + goto ERR_RET; + } + + lumaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16); + pLumaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[0], lumaMapSize); + if (NULL == pLumaVirAddr) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Mmap luma FAILED! .u64PhyAddr[0]:0x%llx\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[0]); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + SAMPLE_LOG_N("AX_SYS_Mmap luma success, .u64PhyAddr[0]:0x%llx, pLumaVirAddr=%p, lumaMapSize=%d\n", + frameInfo->stVFrame.u64PhyAddr[0], pLumaVirAddr, lumaMapSize); + + p_lu = pLumaVirAddr; + + SAMPLE_LOG_N("p_lu: %p\n", p_lu); + SAMPLE_LOG_N("lu_buss: 0x%llx\n", frameInfo->stVFrame.u64PhyAddr[0]); + SAMPLE_LOG_N("ch_buss: 0x%llx\n", frameInfo->stVFrame.u64PhyAddr[1]); + + AX_U32 coded_width = frameInfo->stVFrame.u32Width; + AX_U32 coded_height = frameInfo->stVFrame.u32Height; + AX_U32 pic_stride = frameInfo->stVFrame.u32PicStride[0]; + AX_U32 coded_width_ch = frameInfo->stVFrame.u32Width; + AX_U32 coded_h_ch = frameInfo->stVFrame.u32Height / 2; + AX_U32 pic_stride_ch = frameInfo->stVFrame.u32PicStride[1]; + AX_U32 pic_format = frameInfo->stVFrame.enImgFormat; + + if (AX_FORMAT_YUV400 != pic_format) { + chromaMapSize = frameInfo->stVFrame.u32PicStride[0] * SIZE_ALIGN(frameInfo->stVFrame.u32Height, 16) / 2; + pChromaVirAddr = AX_SYS_Mmap(frameInfo->stVFrame.u64PhyAddr[1], chromaMapSize); + if (NULL == pChromaVirAddr) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Mmap luma FAILED! .u64PhyAddr[1]:0x%llx\n", + VdGrp, frameInfo->stVFrame.u64PhyAddr[1]); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET_MUNMAP_LUMA; + } + + p_ch = pChromaVirAddr; + + SAMPLE_LOG_N("AX_SYS_Mmap chroma success, .u64PhyAddr[1]:0x%llx, pChromaVirAddr=%p, chromaMapSize=%d\n", + frameInfo->stVFrame.u64PhyAddr[1], pChromaVirAddr, chromaMapSize); + SAMPLE_LOG_N("p_ch: %p\n", p_ch); + } + + SAMPLE_LOG_N("p_lu: %p, p_ch: %p, \n", p_lu, p_ch, ); + SAMPLE_LOG_N("coded_width: %u, coded_height: %u, pic_stride: %u, \n" + "coded_width_ch: %u, coded_h_ch: %u, pic_stride_ch: %u, pixel_bytes: %u, pic_format:%d\n", + coded_width, coded_height, pic_stride, + coded_width_ch, coded_h_ch, pic_stride_ch, 1, pic_format); + + SAMPLE_LOG_N("write Y\n"); + if (AX_FORMAT_YUV420_SEMIPLANAR_10BIT_P010 == pic_format) { + coded_width = coded_width * 2; + coded_width_ch = coded_width_ch * 2; + }else if (AX_FORMAT_YUV420_SEMIPLANAR_10BIT_P101010 == pic_format) { + coded_width = coded_width * 10 / 8; + coded_width_ch = coded_width_ch * 10 / 8; + } + + for (i = 0; i < coded_height; i++) { + tmp_size = fwrite(p_lu, 1, coded_width, fp_out); + if (tmp_size != coded_width) { + SAMPLE_CRIT_LOG("VdGrp=%d, fwrite FAILED! tmp_size:0x%x != coded_width:0x%x", + VdGrp, tmp_size, coded_width); + s32Ret = AX_ERR_VDEC_NOMEM; + } + p_lu += pic_stride; + } + + if (AX_FORMAT_YUV400 != pic_format) { + SAMPLE_LOG_N("write UV\n"); + for (i = 0; i < coded_h_ch; i++) { + tmp_size = fwrite(p_ch, 1, coded_width_ch, fp_out); + if (tmp_size != coded_width_ch) { + SAMPLE_CRIT_LOG("VdGrp=%d, fwrite FAILED! tmp_size:0x%x != coded_width_ch:0x%x", + VdGrp, tmp_size, coded_width_ch); + s32Ret = AX_ERR_VDEC_NOMEM; + } + p_ch += pic_stride_ch; + } + } + + ret = fflush(fp_out); + if (ret) { + SAMPLE_CRIT_LOG("VdGrp=%d, fflush FAILED! fp_out:%p", + VdGrp, fp_out); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + } + + SAMPLE_LOG_N("VdGrp=%d, write YUV done! %s\n", VdGrp, pOutputFilePath); + + if (pChromaVirAddr) { + sRet = AX_SYS_Munmap(pChromaVirAddr, chromaMapSize); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Munmap chroma FAILED, sRet=0x%x\n", + VdGrp, sRet); + } + } + +ERR_RET_MUNMAP_LUMA: + if (pLumaVirAddr) { + sRet = AX_SYS_Munmap(pLumaVirAddr, lumaMapSize); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_SYS_Munmap luma FAILED, sRet=0x%x\n", + VdGrp, sRet); + } + } +ERR_RET: + return s32Ret || sRet; +} + +AX_S32 OutputFileSaveYUV(AX_VDEC_GRP VdGrp, const AX_VIDEO_FRAME_INFO_T *frameInfo, FILE *fp_out, AX_CHAR *pOutputFilePath) +{ + AX_S32 s32Ret = 0; + + if (NULL == frameInfo) { + SAMPLE_CRIT_LOG("VdGrp=%d, NULL == frameInfo\n", VdGrp); + return -1; + } + + if (AX_COMPRESS_MODE_NONE != frameInfo->stVFrame.stCompressInfo.enCompressMode) { + SAMPLE_CRIT_LOG("VdGrp=%d, invalid enCompressMode: %d\n", VdGrp, + frameInfo->stVFrame.stCompressInfo.enCompressMode); + return -1; + } + + s32Ret = OutputSaveYUVFile(VdGrp, frameInfo, fp_out, pOutputFilePath); + + return s32Ret; +} + +static off_t __FindFileNextStartCode(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, AX_U32 *puZeroCount, AX_U32 *pNalType) +{ + AX_S32 i; + off_t sStart; + off_t oFileStart; + off_t oFileOffset = 0; + AX_U32 uLeftFileSize, uFileReadLen = 0; + AX_CHAR tmp_buf[VDEC_BS_PARSER_BUF_SIZE] = {0}; + AX_U8 byte; + int ret_val; + int ret; + + oFileStart = ftello(pstBsInfo->fInput); + sStart = oFileStart; + *puZeroCount = 0; + + while (1) { + assert(sStart <= pstBsInfo->sFileSize); + + uLeftFileSize = pstBsInfo->sFileSize - sStart; + if (uLeftFileSize == 0) { + oFileOffset = pstBsInfo->sFileSize - 1; + break; + } + + uFileReadLen = VDEC_BS_PARSER_BUF_SIZE < uLeftFileSize ? VDEC_BS_PARSER_BUF_SIZE : uLeftFileSize; + fread(tmp_buf, 1, uFileReadLen, pstBsInfo->fInput); + /* Scan for the beginning of the packet. */ + for (i = 0; i < uFileReadLen; i++) { + ret_val = tmp_buf[i]; + oFileOffset = sStart + i; + if (ret_val == EOF) { + sStart = oFileOffset - 1; + SAMPLE_LOG_N("sStart:%ld, i:%d, oFileOffset:%d, *puZeroCount:%d", + sStart, i, oFileOffset, *puZeroCount); + + ret = fseeko(pstBsInfo->fInput, 0, SEEK_END); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + + return sStart; + } + + byte = (unsigned char)ret_val; + switch (byte) { + case 0: + *puZeroCount = *puZeroCount + 1; + break; + case 1: + /* If there's more than three leading zeros, consider only three + * of them to be part of this packet and the rest to be part of + * the previous packet. */ + if (*puZeroCount > 3) { + *puZeroCount = 3; + } + + if (*puZeroCount >= 2) { + SAMPLE_LOG_N("sStart:%ld, i:%d, oFileOffset:%d, *puZeroCount:%d", + sStart, i, oFileOffset, *puZeroCount); + if (i < uFileReadLen - 1) { + ret_val = (tmp_buf[i + 1] & 0x1f); + if (ret_val == 0x1 || ret_val == 0x7 || ret_val == 0x5) { + sStart = oFileOffset - *puZeroCount; + *pNalType = ret_val; + goto FUNC_RET; + } + } + } + *puZeroCount = 0; + break; + default: + *puZeroCount = 0; + break; + } + } + + ret = fseeko(pstBsInfo->fInput, oFileOffset, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + sStart = oFileOffset + 1; + } + +FUNC_RET: + + ret = fseeko(pstBsInfo->fInput, oFileOffset + 1, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sStart = AX_ERR_VDEC_RUN_ERROR; + } + SAMPLE_LOG_N("func end, ftello(pstBsInfo->fInput):%d uFileReadLen:%d", + ftello(pstBsInfo->fInput), uFileReadLen); + return sStart; +} + +static int __CheckFileAccessUnitBoundary(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, off_t oNalBegin, SAMPLE_BSBOUNDARY_TYPE_E *penBoundary) +{ + int ret = 0; + int iNalType, iVal; + SAMPLE_BSBOUNDARY_TYPE_E enBoundary = BSPARSER_NO_BOUNDARY; + + FILE *fInput = pstBsInfo->fInput; + off_t sStart = ftello(fInput); + + off_t tmp_offset = oNalBegin + 1; + ret = fseeko(fInput, tmp_offset, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + return ret; + } + + AX_PAYLOAD_TYPE_E enDecType = pstBsInfo->enDecType; + + if (enDecType == PT_H264) { + iNalType = (getc(fInput) & 0x1F); + + if (iNalType > NAL_CODED_SLICE_IDR) { + enBoundary = BSPARSER_BOUNDARY_NON_SLICE_NAL; + } else { + iVal = getc(fInput); + /* Check if first mb in slice is 0(ue(v)). */ + if (iVal & 0x80) { + enBoundary = BSPARSER_BOUNDARY; + } + } + } else if (enDecType == PT_H265) { + iNalType = (getc(fInput) & 0x7E) >> 1; + + if (iNalType > NAL_CODED_SLICE_CRA) { + enBoundary = BSPARSER_BOUNDARY_NON_SLICE_NAL; + } else { + iVal = getc(fInput); // nothing interesting here... + iVal = getc(fInput); + /* Check if first slice segment in picture */ + if (iVal & 0x80) { + enBoundary = BSPARSER_BOUNDARY; + } + } + } + + ret = fseeko(fInput, sStart, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + return ret; + } + + *penBoundary = enBoundary; + return 0; +} + + +static AX_S32 __StreamReadFrameInRingBuf(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, + size_t oStreamLen, size_t *pReadLen) +{ + off_t oOffset = 0; + AX_U8 *pBufRd = NULL; + AX_U8 *pBufStart = NULL; + AX_U32 uBufSize = 0; + AX_S32 sRet = 0; + AX_U32 sReadLen = 0; + AX_U32 tmp_len = 0, right_len = 0, left_len = 0; + + if (pstBsInfo == NULL) { + SAMPLE_CRIT_LOG("pstBsInfo == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pBufRd = pstStreamBuf->pBufAfterFill; + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + uBufSize = pstStreamBuf->uBufSize; + if (pBufStart == NULL) { + SAMPLE_CRIT_LOG("pBufStart == NULL\n"); + sRet = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (uBufSize < oStreamLen) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < oStreamLen:0x%x", uBufSize, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_NOMEM; + goto ERR_RET; + } + + *pReadLen = 0; + oOffset = (off_t)(pBufRd - pBufStart); + pstStreamBuf->pBufBeforeFill = pstStreamBuf->pBufAfterFill; + if ((oOffset + oStreamLen) < uBufSize) { + sReadLen = fread(pBufRd, 1, oStreamLen, pstBsInfo->fInput); + if (sReadLen != oStreamLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != oStreamLen:0x%x", sReadLen, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + pstStreamBuf->pBufAfterFill = pBufRd + sReadLen; + } else { + /* turnaround */ + right_len = uBufSize - oOffset; + sReadLen = fread(pBufRd, 1, right_len, pstBsInfo->fInput); + if (sReadLen != uBufSize - oOffset) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != (uBufSize:0x%x - oOffset:0x%llx):0x%x", + sReadLen, uBufSize, (AX_U64)oOffset, right_len); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + left_len = oStreamLen - (uBufSize - oOffset); + tmp_len = fread(pBufStart, 1, left_len, pstBsInfo->fInput); + if (tmp_len != left_len) { + SAMPLE_CRIT_LOG("fread FAILED! tmp_len:0x%x != left_len:0x%x, oStreamLen:0x%x uBufSize:0x%x oOffset:0x%llx", + tmp_len, left_len, (AX_U32)oStreamLen, uBufSize, (AX_U64)oOffset); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + sReadLen += tmp_len; + pstStreamBuf->pBufAfterFill = pBufStart + tmp_len; + } + + *pReadLen = sReadLen; +ERR_RET: + return sRet; +} + +AX_S32 StreamFileParserReadFrame(const SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + int ret = 0; + AX_S32 sRet = 0; + AX_U32 uZeroCount = 0; + SAMPLE_BSBOUNDARY_TYPE_E enBoundary = BSPARSER_NO_BOUNDARY; + off_t oNalBegin, oStreamLen; + off_t oBegin, oEnd, oTmpEnd; + AX_U32 sReadLen = 0; + AX_U32 uBufSize; + AX_U8 *pBufStart = NULL; + AX_U32 u32NalType = 0; + + if (pstBsInfo == NULL) { + SAMPLE_CRIT_LOG("pstBsInfo == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf->tBufAddr.pVirAddr == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf->tBufAddr.pVirAddr == NULL\n"); + sRet = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (pReadLen == NULL) { + SAMPLE_CRIT_LOG("pReadLen == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + oBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + /* Check for non-slice type in current NAL. non slice NALs are + * decoded one-by-one */ + oNalBegin = oBegin + uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (u32NalType == 0x7) { /* get i frm video nal */ + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + /* Check for non-slice type in current NAL. non slice NALs are + * decoded one-by-one */ + oNalBegin = oNalBegin + uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + } + + oEnd = oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + + if (oEnd == oBegin) { + *pReadLen = 0; + return 0; /* End of stream */ + } + + /* if there is more stream and a slice type NAL */ + if (enBoundary != BSPARSER_BOUNDARY_NON_SLICE_NAL) { + while (1) { + oEnd = oNalBegin; + oNalBegin += uZeroCount; + + /* Check access unit boundary for next NAL */ + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (enBoundary == BSPARSER_NO_BOUNDARY) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + } + else if (enBoundary == BSPARSER_BOUNDARY_NON_SLICE_NAL) { + while (1) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + oTmpEnd = oNalBegin; + oNalBegin += uZeroCount; + ret = __CheckFileAccessUnitBoundary(pstBsInfo, oNalBegin, &enBoundary); + if (ret) { + SAMPLE_CRIT_LOG("__CheckFileAccessUnitBoundary FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + if (enBoundary != BSPARSER_BOUNDARY_NON_SLICE_NAL) { + break; + } + + if (oTmpEnd == oNalBegin) { + break; + } + } + + if (oTmpEnd == oNalBegin) { + break; + } + + if (enBoundary == BSPARSER_NO_BOUNDARY) { + oNalBegin = __FindFileNextStartCode(pstBsInfo, &uZeroCount, &u32NalType); + } + } + + if (enBoundary == BSPARSER_BOUNDARY) { + break; + } + + if (oEnd == oNalBegin) { + break; + } + } + } + + ret = fseeko(pstBsInfo->fInput, oBegin, SEEK_SET); + if (ret) { + SAMPLE_CRIT_LOG("fseeko FAILED! ret:0x%x", ret); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + oStreamLen = oEnd - oBegin; + uBufSize = pstStreamBuf->uBufSize; + if (uBufSize < oStreamLen) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < oStreamLen:0x%x. bufSize is not enough, please increase STREAM_BUFFER_MAX_SIZE", + uBufSize, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_NOMEM; + goto ERR_RET; + } + + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + + sReadLen = fread(pBufStart, 1, oStreamLen, pstBsInfo->fInput); + if (sReadLen != oStreamLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != oStreamLen:0x%x", sReadLen, (AX_U32)oStreamLen); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + pstStreamBuf->pBufBeforeFill = pBufStart; + pstStreamBuf->pBufAfterFill = pBufStart + sReadLen; + + SAMPLE_LOG("sReadLen:0x%lx, uBufSize:0x%x", sReadLen, uBufSize); + *pReadLen = sReadLen; + return 0; +ERR_RET: + return sRet; +} + +static AX_U8 imgGetBytes(FILE *fInput, size_t pos) +{ + AX_U8 data = 0; + AX_U8 readLen = 0; + + fseeko(fInput, pos, SEEK_SET); + readLen = fread(&data, 1, 1, fInput); + if(1 != readLen) + SAMPLE_CRIT_LOG(" read file failed\n"); + + return data; +} + +AX_S32 StreamParserReadFrameJpeg(SAMPLE_INPUT_FILE_INFO_T *pstBsInfo, SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + size_t i,j; + AX_U32 jpeg_thumb_in_stream = 0; + AX_U64 tmp, tmp1, tmp_total = 0; + size_t curPos = 0; + AX_U32 imgLen = 0; + AX_S32 s32Ret = 0; + size_t stream_length = 0; + AX_U8 *pBufStart = NULL; + size_t sReadLen = 0; + + pBufStart = pstStreamBuf->tBufAddr.pVirAddr; + if (pBufStart == NULL) { + SAMPLE_CRIT_LOG("pBufStart == NULL\n"); + return -1; + } + + stream_length = pstBsInfo->sFileSize; + for (i = pstBsInfo->curPos; i < stream_length; ++i) { + if (0xFF == imgGetBytes(pstBsInfo->fInput, i)) { + /* if 0xFFE1 to 0xFFFD ==> skip */ + if ((((i + 1) < stream_length) && + 0xE1 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE2 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE3 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE4 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE5 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE6 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE7 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE8 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xE9 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEA == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEB == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEC == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xED == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEE == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xEF == imgGetBytes(pstBsInfo->fInput, i + 1))) { + /* increase counter */ + i += 2; + + /* check length vs. data */ + if ((i + 1) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + + /* get length */ + tmp = imgGetBytes(pstBsInfo->fInput, i); + tmp1 = imgGetBytes(pstBsInfo->fInput, i + 1); + tmp_total = (tmp << 8) | tmp1; + + /* check length vs. data */ + if ((tmp_total + i) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + /* update */ + i += tmp_total-1; + continue; + } + + /* if 0xFFC2 to 0xFFCB ==> skip */ + if ((((i + 1) < stream_length) && + 0xC1 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC2 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC3 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC5 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC6 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC7 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC8 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xC9 == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xCA == imgGetBytes(pstBsInfo->fInput, i + 1)) || + (((i + 1) < stream_length) && + 0xCB == imgGetBytes(pstBsInfo->fInput, i + 1)) ) { + /* increase counter */ + i += 2; + + /* check length vs. data */ + if ((i + 1) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + + /* get length */ + tmp = imgGetBytes(pstBsInfo->fInput, i); + tmp1 = imgGetBytes(pstBsInfo->fInput, i + 1); + tmp_total = (tmp << 8) | tmp1; + + /* check length vs. data */ + if ((tmp_total + i) > (stream_length)) { + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ret; + } + /* update */ + i += tmp_total-1; + + /* look for EOI */ + for(j = i; j < stream_length; ++j) { + if (0xFF == imgGetBytes(pstBsInfo->fInput, j)) { + /* EOI */ + if (((j + 1) < stream_length) && + 0xD9 == imgGetBytes(pstBsInfo->fInput, j + 1)) { + /* check length vs. data */ + if ((j + 2) >= (stream_length)) { + curPos = j + 2; + s32Ret = 0; + goto ret; + } + /* update */ + i = j; + /* stil data left ==> continue */ + continue; + } + } + } + } + + /* check if thumbnails in stream */ + if (((i + 1) < stream_length) && + 0xE0 == imgGetBytes(pstBsInfo->fInput, i + 1)) { + if (((i + 9) < stream_length) && + 0x4A == imgGetBytes(pstBsInfo->fInput, i + 4) && + 0x46 == imgGetBytes(pstBsInfo->fInput, i + 5) && + 0x58 == imgGetBytes(pstBsInfo->fInput, i + 6) && + 0x58 == imgGetBytes(pstBsInfo->fInput, i + 7) && + 0x00 == imgGetBytes(pstBsInfo->fInput, i + 8) && + 0x10 == imgGetBytes(pstBsInfo->fInput, i + 9)) { + jpeg_thumb_in_stream = 1; + } + } + + /* EOI */ + if (((i + 1) < stream_length) && + 0xD9 == imgGetBytes(pstBsInfo->fInput, i + 1)) { + curPos = i + 2; + /* update amount of thumbnail or full resolution image */ + if (jpeg_thumb_in_stream) { + jpeg_thumb_in_stream = 0; + } else { + s32Ret = 0; + goto ret; + } + } + } + } + +ret: + imgLen = curPos > pstBsInfo->curPos ? curPos - pstBsInfo->curPos : 0; + if(0 == s32Ret) { + fseeko(pstBsInfo->fInput, pstBsInfo->curPos, SEEK_SET); + if (pstStreamBuf->bRingbuf == AX_TRUE) { + s32Ret = __StreamReadFrameInRingBuf(pstBsInfo, pstStreamBuf, imgLen, &sReadLen); + if (s32Ret) { + SAMPLE_CRIT_LOG("__StreamReadFrameInRingBuf FAILED! ret:0x%x", s32Ret); + return -1; + } + } else { + if (imgLen > pstStreamBuf->uBufSize) { + SAMPLE_CRIT_LOG("bufSize is not enough(imgLen %d > bufSize %d), please increase STREAM_BUFFER_MAX_SIZE", + imgLen, pstStreamBuf->uBufSize); + return -1; + } + sReadLen = fread(pBufStart, 1, imgLen, pstBsInfo->fInput); + if (sReadLen != imgLen) { + SAMPLE_CRIT_LOG("fread FAILED! sReadLen:0x%x != imgLen:0x%x", (AX_U32)sReadLen, imgLen); + return -1; + } + + pstStreamBuf->pBufBeforeFill = pBufStart; + pstStreamBuf->pBufAfterFill = pBufStart + sReadLen; + } + pstBsInfo->curPos += sReadLen; + } + + *pReadLen = imgLen; + return s32Ret; +} + +static AX_S32 __VdecUsrPicRead(FILE *pFileIn, AX_VIDEO_FRAME_T *pstFrame) +{ + AX_S32 sRet = 0; + AX_U32 i = 0, rows = 0, realRead = 0; + AX_S32 widthSrc = 0, strideSrc = 0, heightSrc = 0; + AX_IMG_FORMAT_E eFmt; + AX_VOID *pVaddr = NULL; + + if (pFileIn == NULL) { + SAMPLE_CRIT_LOG("pFileIn == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstFrame == NULL) { + SAMPLE_CRIT_LOG("pstFrame == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + widthSrc = pstFrame->u32Width; + heightSrc = pstFrame->u32Height; + strideSrc = pstFrame->u32PicStride[0]; + eFmt = pstFrame->enImgFormat; + pVaddr = (AX_VOID *)(AX_ULONG)pstFrame->u64VirAddr[0]; + switch (eFmt) { + case AX_FORMAT_YUV420_SEMIPLANAR: + case AX_FORMAT_YUV420_SEMIPLANAR_VU: + rows = heightSrc * 3 / 2; + for (i = 0; i < rows; i++) { + realRead = fread(pVaddr, 1, widthSrc, pFileIn); + if (realRead < widthSrc) { + SAMPLE_CRIT_LOG("fread failed! line %d realRead=%d < widthSrc=%d\n", i, realRead, widthSrc); + sRet = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + pVaddr += strideSrc; + } + break; + default: + SAMPLE_CRIT_LOG("Invalid format, eFmt = %d\n", eFmt); + } + +ERR_RET: + return sRet; +} + +static AX_S32 __VdecUsrPicInfoFill(SAMPLE_VDEC_USRPIC_ARGS_T *pstUsrPicArgs, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_POOL_CONFIG_T stPoolConfig; + AX_U32 FrameSize = 0; + AX_BLK blkId = AX_INVALID_BLOCKID; + AX_VDEC_USRPIC_T *pstUserPic = NULL; + FILE *fpUserYUV = NULL; + AX_S32 sRet = 0; + AX_S32 ret = 0; + AX_VDEC_GRP VdGrp = 0; + AX_U32 uWidth = 0; + AX_U32 u32FrameStride = 0; + + if (pstUsrPicArgs == NULL) { + SAMPLE_CRIT_LOG("pstUsrPicArgs == NULL\n"); + sRet = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstUsrPicArgs->VdGrp; + SAMPLE_LOG("VdGrp=%d begin\n", VdGrp); + + memset(&stPoolConfig, 0x0, sizeof(AX_POOL_CONFIG_T)); + + if (pstUsrPicArgs->tPicParam.bUserPicEnable) { + if ((AX_FORMAT_YUV420_SEMIPLANAR != pstUsrPicArgs->tPicParam.enImgFormat) && + (AX_FORMAT_YUV420_SEMIPLANAR_VU != pstUsrPicArgs->tPicParam.enImgFormat)) { + SAMPLE_CRIT_LOG("VdGrp=%d, unsupport enImgFormat:%d\n", + VdGrp, pstUsrPicArgs->tPicParam.enImgFormat); + sRet = AX_ERR_VDEC_NOT_SUPPORT; + goto ERR_RET; + } + uWidth = pstUsrPicArgs->tPicParam.u32PicWidth; + u32FrameStride = AX_COMM_ALIGN(uWidth, AX_VDEC_WIDTH_ALIGN); + pstVdecUserPic->pUsrPicFilePath = pstUsrPicArgs->tPicParam.pUsrPicFilePath; + FrameSize = AX_VDEC_GetPicBufferSize(u32FrameStride, + pstUsrPicArgs->tPicParam.u32PicHeight, + pstUsrPicArgs->enDecType); + + stPoolConfig.MetaSize = 512; + stPoolConfig.BlkCnt = 1; + + stPoolConfig.BlkSize = FrameSize; + stPoolConfig.CacheMode = AX_POOL_CACHE_MODE_NONCACHE; + snprintf((AX_CHAR *)stPoolConfig.PartitionName, AX_MAX_PARTITION_NAME_LEN, "anonymous"); + pstVdecUserPic->PoolId = AX_POOL_CreatePool(&stPoolConfig); + if (AX_INVALID_POOLID == pstVdecUserPic->PoolId) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_POOL_CreatePool FAILED! BlkCnt:%d, BlkSize:0x%llx\n", + VdGrp, stPoolConfig.BlkCnt, stPoolConfig.BlkSize); + sRet = AX_ERR_VDEC_NOBUF; + goto ERR_RET_DESTROY; + } + + blkId = AX_POOL_GetBlock(pstVdecUserPic->PoolId, FrameSize, NULL); + if (AX_INVALID_BLOCKID == blkId) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_POOL_GetBlock FAILED! PoolId:%d, BlkSize:0x%llx\n", + VdGrp, pstVdecUserPic->PoolId, stPoolConfig.BlkSize); + sRet = AX_ERR_VDEC_NOBUF; + goto ERR_RET_DESTROY; + } + + pstVdecUserPic->BlkId = blkId; + pstUserPic = &pstVdecUserPic->stUserPic; + pstUserPic->bEnable = AX_TRUE; + pstUserPic->bInstant = pstUsrPicArgs->bUsrInstant; + pstUserPic->stFrmInfo.bEndOfStream = AX_TRUE; + pstUserPic->stFrmInfo.enModId = AX_ID_VDEC; + pstUserPic->stFrmInfo.stVFrame.u32BlkId[0] = blkId; + pstUserPic->stFrmInfo.stVFrame.u32FrameSize = FrameSize; + pstUserPic->stFrmInfo.stVFrame.u32Width = pstUsrPicArgs->tPicParam.u32PicWidth; + pstUserPic->stFrmInfo.stVFrame.u32Height = pstUsrPicArgs->tPicParam.u32PicHeight; + pstUserPic->stFrmInfo.stVFrame.enImgFormat = pstUsrPicArgs->tPicParam.enImgFormat; + pstUserPic->stFrmInfo.stVFrame.enVscanFormat = AX_VSCAN_FORMAT_RASTER; + pstUserPic->stFrmInfo.stVFrame.stCompressInfo.enCompressMode = AX_COMPRESS_MODE_NONE; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0] = AX_POOL_Handle2PhysAddr(blkId); + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0] = (AX_ULONG)AX_POOL_GetBlockVirAddr(blkId); + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] = u32FrameStride; + pstUserPic->stFrmInfo.stVFrame.u32PicStride[1] = u32FrameStride; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[1] = pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0] + + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] * + pstUserPic->stFrmInfo.stVFrame.u32Height; + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[2] = 0; + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[1] = pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0] + + pstUserPic->stFrmInfo.stVFrame.u32PicStride[0] * + pstUserPic->stFrmInfo.stVFrame.u32Height; + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[2] = 0; + pstUserPic->stFrmInfo.stVFrame.u64PTS = 0; + } + + if (pstUsrPicArgs->tPicParam.enImgFormat) { + uWidth = pstUserPic->stFrmInfo.stVFrame.u32Width; + pstVdecUserPic->fpUsrPic = fopen(pstVdecUserPic->pUsrPicFilePath,"rb"); + if (pstVdecUserPic->fpUsrPic == NULL) { + SAMPLE_CRIT_LOG("VdGrp=%d, can't open file %s in VDEC_PREPARE_USERPIC.\n", + VdGrp, pstVdecUserPic->pUsrPicFilePath); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET_FREE; + } + + fpUserYUV = pstVdecUserPic->fpUsrPic; + + sRet = __VdecUsrPicRead(fpUserYUV, &pstUserPic->stFrmInfo.stVFrame); + if (sRet) { + SAMPLE_CRIT_LOG("__VdecUsrPicRead FAILED! ret:%d\n", sRet); + goto ERR_RET_FREE; + } + + sRet = fclose(fpUserYUV); + if (sRet) { + SAMPLE_CRIT_LOG("fclose FAILED! ret:%d\n", sRet); + sRet = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET_FREE; + } + + fpUserYUV = NULL; + SAMPLE_LOG("VdGrp=%d, .u64PhyAddr[0]:%llx .u64VirAddr[0]:%llx .u64PhyAddr[1]:%llx .u64VirAddr[1]:%llx\n", + VdGrp, + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[0], + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[0], + pstUserPic->stFrmInfo.stVFrame.u64PhyAddr[1], + pstUserPic->stFrmInfo.stVFrame.u64VirAddr[1]); + } + + return AX_SUCCESS; + +ERR_RET_FREE: + if (fpUserYUV) { + fclose(fpUserYUV); + fpUserYUV = NULL; + } + +ERR_RET_DESTROY: + if (pstUsrPicArgs->tPicParam.bUserPicEnable) { + if (pstVdecUserPic->BlkId != AX_INVALID_BLOCKID) { + ret = AX_POOL_ReleaseBlock(pstVdecUserPic->BlkId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolI:%d, AX_POOL_ReleaseBlock FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->BlkId, ret, SampleVdecRetStr(ret)); + } + if (pstVdecUserPic->PoolId != AX_INVALID_POOLID) { + ret = AX_POOL_DestroyPool(pstVdecUserPic->PoolId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolId:%d, AX_POOL_DestroyPool FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->PoolId, ret, SampleVdecRetStr(ret)); + } + } + +ERR_RET: + return sRet; +} + +AX_S32 __VdecUsrPicCreat(SAMPLE_VDEC_USRPIC_ARGS_T *pstUsrPicArgs, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_S32 s32Ret = AX_SUCCESS; + AX_VDEC_GRP VdGrp = 0; + + if (pstVdecUserPic == NULL) { + SAMPLE_CRIT_LOG("null pointer\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstUsrPicArgs == NULL) { + SAMPLE_CRIT_LOG("pstUsrPicArgs == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstUsrPicArgs->VdGrp; + SAMPLE_LOG("VdGrp=%d begin\n", VdGrp); + + if (pstUsrPicArgs->tPicParam.bUserPicEnable == AX_FALSE) { + pstVdecUserPic->PoolId = AX_INVALID_POOLID; + goto ERR_RET; + } + + s32Ret = __VdecUsrPicInfoFill(pstUsrPicArgs, pstVdecUserPic); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, __VdecUsrPicInfoFill FAILED!\n",VdGrp); + goto ERR_RET; + } + + s32Ret = AX_VDEC_DisableUserPic(VdGrp); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_DisableUserPic FAILED! ret:0x%x %s\n", + VdGrp, s32Ret, SampleVdecRetStr(s32Ret)); + goto ERR_RET; + } + + s32Ret = AX_VDEC_SetUserPic(VdGrp, &pstVdecUserPic->stUserPic); + if (s32Ret != AX_SUCCESS) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_SetUserPic FAILED! ret:0x%x %s\n", + VdGrp, s32Ret, SampleVdecRetStr(s32Ret)); + goto ERR_RET; + } + +ERR_RET: + return s32Ret; +} + +AX_S32 VdecUserPicEnable(AX_VDEC_GRP VdGrp, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic, + AX_BOOL *pContSendStm, SAMPLE_VDEC_CONTEXT_T *pstCtx) +{ + AX_S32 sRet = AX_SUCCESS; + AX_VDEC_RECV_PIC_PARAM_T tRecvParam; + + if (pstVdecUserPic->stUserPic.bEnable) { + sRet = AX_VDEC_StopRecvStream(VdGrp); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_StopRecvStream FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } else { + pstCtx->GrpStatus[VdGrp] = AX_VDEC_GRP_STOP_RECV; + } + + if (!pstVdecUserPic->stUserPic.bInstant) + sleep(1); + + sRet = AX_VDEC_EnableUserPic(VdGrp); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_EnableUserPic FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } + + if (pstVdecUserPic->recvStmAfUsrPic) { + *pContSendStm = AX_TRUE; + } else { + *pContSendStm = AX_FALSE; + } + + SAMPLE_LOG("VdGrp=%d, AX_VDEC_EnableUserPic finish!\n", VdGrp); + + memset(&tRecvParam, 0, sizeof(tRecvParam)); + tRecvParam.s32RecvPicNum = pstVdecUserPic->s32RecvPicNumBak; + sRet = AX_VDEC_StartRecvStream(VdGrp, &tRecvParam); + if (sRet) { + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_StartRecvStream FAILED! ret:0x%x %s\n", + VdGrp, sRet, SampleVdecRetStr(sRet)); + goto ERR_RET; + } else { + pstCtx->GrpStatus[VdGrp] = AX_VDEC_GRP_START_RECV; + } + } + +ERR_RET: + return sRet; +} + +AX_VOID VdecUserPicDestroy(AX_VDEC_GRP VdGrp, SAMPLE_VDEC_USERPIC_T *pstVdecUserPic) +{ + AX_S32 ret = AX_SUCCESS; + + if (pstVdecUserPic->stUserPic.bEnable) { + if (pstVdecUserPic->BlkId != AX_INVALID_BLOCKID) { + ret = AX_POOL_ReleaseBlock(pstVdecUserPic->BlkId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, BlkId:%d, AX_POOL_ReleaseBlock FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->BlkId, ret, SampleVdecRetStr(ret)); + } + if (pstVdecUserPic->PoolId != AX_INVALID_POOLID) { + ret = AX_POOL_DestroyPool(pstVdecUserPic->PoolId); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, PoolId:%d, AX_POOL_DestroyPool FAILED! ret:0x%x %s", + VdGrp, pstVdecUserPic->PoolId, ret, SampleVdecRetStr(ret)); + } + } + + if (pstVdecUserPic->stUserPic.bEnable) { + ret = AX_VDEC_DisableUserPic(VdGrp); + if (ret) + SAMPLE_CRIT_LOG("VdGrp=%d, AX_VDEC_DisableUserPic FAILED! ret:0x%x %s", + VdGrp, ret, SampleVdecRetStr(ret)); + } +} + + +int VdecCommonPoolPrintf(AX_VOID) +{ + return 0; + AX_S32 s32Ret = 0; + AX_POOL_FLOORPLAN_T PoolFloorPlan; + + memset(&PoolFloorPlan, 0, sizeof(AX_POOL_FLOORPLAN_T)); + + s32Ret = AX_POOL_GetConfig(&PoolFloorPlan); + if (s32Ret) { + SAMPLE_CRIT_LOG("AX_POOL_SetConfig FAILED! 0x%x\n", s32Ret); + goto ERR_RET; + } + + for (int pi = 0; pi < AX_MAX_COMM_POOLS; pi++) { + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].MetaSize:0x%llx", pi, PoolFloorPlan.CommPool[pi].MetaSize); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].BlkSize:0x%llx", pi, PoolFloorPlan.CommPool[pi].BlkSize); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].BlkCnt:0x%x", pi, PoolFloorPlan.CommPool[pi].BlkCnt); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].IsMergeMode:0x%x", pi, PoolFloorPlan.CommPool[pi].IsMergeMode); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].CacheMode:0x%x", pi, PoolFloorPlan.CommPool[pi].CacheMode); + SAMPLE_LOG("PoolFloorPlan.CommPool[%d].PartitionName:%s", pi, PoolFloorPlan.CommPool[pi].PartitionName); + } + +ERR_RET: + return s32Ret; +} + +#ifdef AX_VDEC_FFMPEG_ENABLE +AX_S32 SampleVdecFfmpegDeinit(SAMPLE_FFMPEG_T *pstFfmpeg, AX_VDEC_GRP VdGrp) +{ + AX_S32 s32Ret = AX_SUCCESS; + + SAMPLE_LOG("stream %d +++", VdGrp); + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstFfmpeg->pstAvPkt) { + // av_packet_unref(pstFfmpeg->pstAvPkt); + av_packet_free(&pstFfmpeg->pstAvPkt); + pstFfmpeg->pstAvPkt = NULL; + } + + if (pstFfmpeg->pstAvBSFCtx) { + av_bsf_free(&pstFfmpeg->pstAvBSFCtx); + pstFfmpeg->pstAvBSFCtx = NULL; + } + + if (pstFfmpeg->pstAvFmtCtx) { + avformat_close_input(&pstFfmpeg->pstAvFmtCtx); + pstFfmpeg->pstAvFmtCtx = NULL; + } + + SAMPLE_LOG("stream %d ---", VdGrp); + +ERR_RET: + return s32Ret; +} + +AX_S32 SampleVdecFfmpegInit(SAMPLE_FFMPEG_T *pstFfmpeg, const AX_CHAR *pcInputFilePath, + SAMPLE_BITSTREAM_INFO_T *pstBitStreamInfo) +{ + AX_S32 s32Ret = AX_SUCCESS; + int ret; + enum AVCodecID eCodecID = AV_CODEC_ID_H264; + AX_VDEC_GRP VdGrp = 0; + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pcInputFilePath == NULL) { + SAMPLE_CRIT_LOG("pcInputFilePath == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstBitStreamInfo == NULL) { + SAMPLE_CRIT_LOG("pstBitStreamInfo == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstBitStreamInfo->VdGrp; + + pstFfmpeg->s32VideoIndex = -1; + pstFfmpeg->pstAvFmtCtx = avformat_alloc_context(); + if (pstFfmpeg->pstAvFmtCtx == NULL) { + SAMPLE_CRIT_LOG("avformat_alloc_context() failed!"); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + ret = avformat_open_input(&pstFfmpeg->pstAvFmtCtx, pcInputFilePath, NULL, NULL); + if (ret < 0) { + AX_CHAR szError[64] = {0}; + av_strerror(ret, szError, 64); + SAMPLE_CRIT_LOG("open %s fail, error: %d, %s", pcInputFilePath, ret, szError); + goto ERR_RET; + } + + ret = avformat_find_stream_info(pstFfmpeg->pstAvFmtCtx, NULL); + if (ret < 0) { + SAMPLE_CRIT_LOG("avformat_find_stream_info fail, error = %d", ret); + goto ERR_RET; + } + + for (int i = 0; i < pstFfmpeg->pstAvFmtCtx->nb_streams; i++) { + if (AVMEDIA_TYPE_VIDEO == pstFfmpeg->pstAvFmtCtx->streams[i]->codecpar->codec_type) { + pstFfmpeg->s32VideoIndex = i; + break; + } + } + + if (-1 == pstFfmpeg->s32VideoIndex) { + SAMPLE_CRIT_LOG("%s has no video stream!", pcInputFilePath); + goto ERR_RET; + } else { + AVStream *pAvs = pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]; + eCodecID = pAvs->codecpar->codec_id; + switch (eCodecID) { + case AV_CODEC_ID_H264: + pstBitStreamInfo->eVideoType = PT_H264; + break; + case AV_CODEC_ID_HEVC: + pstBitStreamInfo->eVideoType = PT_H265; + break; + default: + SAMPLE_CRIT_LOG("Current Only support H264 or HEVC stream %d!", VdGrp); + goto ERR_RET; + } + + pstBitStreamInfo->nWidth = pAvs->codecpar->width; + pstBitStreamInfo->nHeight = pAvs->codecpar->height; + pstBitStreamInfo->nFps = av_q2d(pAvs->r_frame_rate); + if (0 == pstBitStreamInfo->nFps) { + pstBitStreamInfo->nFps = 30; + SAMPLE_LOG("stream %d fps is 0, set to %d fps", VdGrp, pstBitStreamInfo->nFps); + } + + SAMPLE_LOG("stream %d: vcodec %d, %dx%d, fps %d", + VdGrp, pstBitStreamInfo->eVideoType, pstBitStreamInfo->nWidth, pstBitStreamInfo->nHeight, + pstBitStreamInfo->nFps); + } + + pstFfmpeg->pstAvPkt = av_packet_alloc(); + if (!pstFfmpeg->pstAvPkt) { + SAMPLE_CRIT_LOG("Create packet(stream %d) fail!", VdGrp); + goto ERR_RET; + } + + if ((AV_CODEC_ID_H264 == eCodecID) || (AV_CODEC_ID_HEVC == eCodecID)) { + const AVBitStreamFilter *pstBSFilter = av_bsf_get_by_name((AV_CODEC_ID_H264 == eCodecID) ? + "h264_mp4toannexb" : "hevc_mp4toannexb"); + if (!pstBSFilter) { + SAMPLE_CRIT_LOG("av_bsf_get_by_name(stream %d) fail!", VdGrp); + goto ERR_RET; + } + + ret = av_bsf_alloc(pstBSFilter, &pstFfmpeg->pstAvBSFCtx); + if (ret < 0) { + SAMPLE_CRIT_LOG("av_bsf_alloc(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } + + ret = avcodec_parameters_copy(pstFfmpeg->pstAvBSFCtx->par_in, + pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]->codecpar); + if (ret < 0) { + SAMPLE_CRIT_LOG("avcodec_parameters_copy(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } else { + pstFfmpeg->pstAvBSFCtx->time_base_in = pstFfmpeg->pstAvFmtCtx->streams[pstFfmpeg->s32VideoIndex]->time_base; + } + + ret = av_bsf_init(pstFfmpeg->pstAvBSFCtx); + if (ret < 0) { + SAMPLE_CRIT_LOG("av_bsf_init(stream %d) fail, error:%d", VdGrp, ret); + goto ERR_RET; + } + } + + return s32Ret; + +ERR_RET: + SampleVdecFfmpegDeinit(pstFfmpeg, VdGrp); + return s32Ret; +} + +AX_S32 SampleVdecFfmpegExtractOnePic(SAMPLE_FFMPEG_T *pstFfmpeg, SAMPLE_BITSTREAM_INFO_T *pstBitStreamInfo, + SAMPLE_STREAM_BUF_T *pstStreamBuf, size_t *pReadLen) +{ + AX_S32 s32Ret = AX_SUCCESS; + int ret; + AVFormatContext *pstAvFmtCtx = NULL; + AVBSFContext *pstAvBSFCtx = NULL; + AVPacket *pstAvPkt = NULL; + static AX_U64 u64FrameCnt = 0; + AX_VDEC_GRP VdGrp = 0; + + if (pstFfmpeg == NULL) { + SAMPLE_CRIT_LOG("pstFfmpeg == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvFmtCtx = pstFfmpeg->pstAvFmtCtx; + if (pstAvFmtCtx == NULL) { + SAMPLE_CRIT_LOG("pstAvFmtCtx == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvBSFCtx = pstFfmpeg->pstAvBSFCtx; + if (pstAvBSFCtx == NULL) { + SAMPLE_CRIT_LOG("pstAvBSFCtx == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + pstAvPkt = pstFfmpeg->pstAvPkt; + if (pstAvPkt == NULL) { + SAMPLE_CRIT_LOG("pstAvPkt == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstBitStreamInfo == NULL) { + SAMPLE_CRIT_LOG("pstBitStreamInfo == NULL"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + if (pstStreamBuf->tBufAddr.pVirAddr == NULL) { + SAMPLE_CRIT_LOG("pstStreamBuf->tBufAddr.pVirAddr == NULL\n"); + s32Ret = AX_ERR_VDEC_BAD_ADDR; + goto ERR_RET; + } + + if (pReadLen == NULL) { + SAMPLE_CRIT_LOG("pReadLen == NULL\n"); + s32Ret = AX_ERR_VDEC_NULL_PTR; + goto ERR_RET; + } + + VdGrp = pstBitStreamInfo->VdGrp; + + while (1) { + ret = av_read_frame(pstAvFmtCtx, pstAvPkt); + if (ret < 0) { + if (AVERROR_EOF == ret) { + SAMPLE_LOG("reach eof of stream %d ", VdGrp); + *pReadLen = 0; + break; + } else { + SAMPLE_CRIT_LOG("av_read_frame(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_STRM_ERROR; + break; + } + } else { + if (pstAvPkt->stream_index == pstFfmpeg->s32VideoIndex) { + ret = av_bsf_send_packet(pstAvBSFCtx, pstAvPkt); + if (ret < 0) { + av_packet_unref(pstAvPkt); + SAMPLE_CRIT_LOG("av_bsf_send_packet(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + break; + } + + while (ret >= 0) { + ret = av_bsf_receive_packet(pstAvBSFCtx, pstAvPkt); + if (ret < 0) { + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + + av_packet_unref(pstAvPkt); + SAMPLE_CRIT_LOG("av_bsf_receive_packet(stream %d) fail, error: %d", VdGrp, ret); + s32Ret = AX_ERR_VDEC_RUN_ERROR; + goto ERR_RET; + } + + if (pstStreamBuf->uBufSize < pstAvPkt->size) { + SAMPLE_CRIT_LOG("uBufSize:0x%x < pstAvPkt->size:0x%x. " + "bufSize is not enough, please increase STREAM_BUFFER_MAX_SIZE", + pstStreamBuf->uBufSize, pstAvPkt->size); + s32Ret = AX_ERR_VDEC_STRM_ERROR; + goto ERR_RET; + } + + u64FrameCnt++; + *pReadLen = pstAvPkt->size; + memcpy(pstStreamBuf->tBufAddr.pVirAddr, pstAvPkt->data, pstAvPkt->size); + + SAMPLE_LOG_N("u64FrameCnt:%lld, pstAvPkt->size:%d", u64FrameCnt, pstAvPkt->size); + } + } + + av_packet_unref(pstAvPkt); + } + + break; + } + +ERR_RET: + return s32Ret; +} +#endif + +AX_U32 SampleVdecSearchStartCode(AX_U8 *uStrAddr, AX_U32 uLen, AX_U32 *puReadBytes, AX_U32 *puZeroCount) +{ + AX_U32 i = 0; + + if (AX_NULL == uStrAddr) { + return AX_FALSE; + } + + if (uLen < 4) { + return AX_FALSE; + } + + if (AX_NULL == puReadBytes) { + return AX_FALSE; + } + + if (AX_NULL == puZeroCount) { + return AX_FALSE; + } + + *puReadBytes = 0; + *puZeroCount = 0; + + for (i = 0; i < uLen - 4; i++) { + if ((uStrAddr[i] == 0) + && (uStrAddr[i + 1] == 0) + && (uStrAddr[i + 2] == 0) + && (uStrAddr[i + 3] == 1)) { + *puReadBytes = i; + *puZeroCount = 3; + return AX_TRUE; + } else if ((uStrAddr[i] == 0) + && (uStrAddr[i + 1] == 0) + && (uStrAddr[i + 2] == 1)) { + *puReadBytes = i; + *puZeroCount = 2; + return AX_TRUE; + } + } + + return AX_FALSE; +} + +AX_VOID SampelVdecSetThreadName(const char *nameFmt, ...) +{ + AX_CHAR name[16]; + va_list args; + + va_start(args, nameFmt); + vsnprintf(name, sizeof(name), nameFmt, args); + va_end(args); + + prctl(PR_SET_NAME, name, NULL, NULL, NULL); +} + +AX_U64 SampleGetFileSize(char * pFileName) +{ + AX_CHAR cmd[256] = {0}; + FILE *pInfo = NULL; + AX_U64 fileSize = 0; + AX_S32 ret = 0; + AX_U32 i = 0; + + SAMPLE_LOG_TMP("calu file: %s\n", pFileName); + sprintf(cmd, "stat %s > fileInfo.txt", pFileName); + + ret = system(cmd); + if (ret) + return -1; + + pInfo = fopen("fileInfo.txt", "rb"); + if (NULL == pInfo) + return -1; + + fgets(cmd, 256, pInfo); + SAMPLE_LOG_TMP("file: %s\n", cmd); + fgets(cmd, 256, pInfo); + SAMPLE_LOG_TMP("info: %s\n", cmd); + fclose(pInfo); + + if (strlen(cmd) < 10) { + return -1; + } + + while(cmd[i]) { + if (('0' <= cmd[i]) && ('9' >= cmd[i])) + break; + + SAMPLE_LOG_TMP("pos %d char %c\n", i, cmd[i]); + i++; + } + + while(cmd[i]) { + SAMPLE_LOG_TMP("pos %d char %c\n", i, cmd[i]); + if (('0' > cmd[i]) || ('9' < cmd[i])) + break; + + fileSize = fileSize * 10 + (cmd[i] - '0'); + i++; + } + + return fileSize; +} diff --git a/projects/llm_framework/main_skel/src/runner/frameMgr.cpp b/projects/llm_framework/main_skel/src/runner/frameMgr.cpp new file mode 100644 index 00000000..bb5138d9 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/frameMgr.cpp @@ -0,0 +1,72 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include "frameMgr.h" +#include + +AX_BOOL g_bSkel_frame_mgr_inited = AX_FALSE; +AX_POOL g_skel_frame_poolId = AX_INVALID_POOLID; + +AX_VOID FrameMgrCreate(AX_U32 nFrameSize, AX_U32 nDepth) { + AX_POOL_CONFIG_T stPoolConfig; + + memset(&stPoolConfig, 0, sizeof(AX_POOL_CONFIG_T)); + stPoolConfig.MetaSize = 4096; + stPoolConfig.BlkCnt = nDepth; + stPoolConfig.BlkSize = nFrameSize; + stPoolConfig.CacheMode = AX_POOL_CACHE_MODE_NONCACHE; + memset(stPoolConfig.PartitionName, 0, sizeof(stPoolConfig.PartitionName)); + strcpy((AX_CHAR *)stPoolConfig.PartitionName, "anonymous"); + + g_skel_frame_poolId = AX_POOL_CreatePool(&stPoolConfig); + g_bSkel_frame_mgr_inited = AX_TRUE; +} + +AX_BOOL FrameMgrGet(AX_U64 *YUVDataPhy, AX_VOID **YUVDataVir, AX_U32 nFrameSize, AX_U64 nFrameId, AX_BLK *nBlkId) { + if (!g_bSkel_frame_mgr_inited + || g_skel_frame_poolId == AX_INVALID_POOLID) { + return AX_FALSE; + } + + AX_BLK blkId = AX_POOL_GetBlock(g_skel_frame_poolId, nFrameSize, NULL); + + if (blkId != AX_INVALID_BLOCKID) { + *YUVDataPhy = AX_POOL_Handle2PhysAddr(blkId); + *YUVDataVir = AX_POOL_GetBlockVirAddr(blkId); + *nBlkId = blkId; + + return AX_TRUE; + } + + return AX_FALSE; +} + +AX_VOID FrameMgrRelease(AX_BLK blkId) { + if (blkId != AX_INVALID_BLOCKID) { + blkId = AX_POOL_ReleaseBlock(blkId); + } +} + +AX_VOID FrameMgrDestroy(AX_VOID) { + if (!g_bSkel_frame_mgr_inited + || g_skel_frame_poolId == AX_INVALID_POOLID) { + return; + } + + AX_POOL_DestroyPool(g_skel_frame_poolId); + + g_skel_frame_poolId = AX_INVALID_POOLID; + g_bSkel_frame_mgr_inited = AX_FALSE; +} diff --git a/projects/llm_framework/main_skel/src/runner/frameMgr.h b/projects/llm_framework/main_skel/src/runner/frameMgr.h new file mode 100644 index 00000000..ce3fbd42 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/frameMgr.h @@ -0,0 +1,33 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _FRAME_MGR_H_ +#define _FRAME_MGR_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SKEL_FRAME_BUF_DEFAULT_DEPTH 2 + +AX_VOID FrameMgrCreate(AX_U32 nFrameSize, AX_U32 nDepth); +AX_BOOL FrameMgrGet(AX_U64 *YUVDataPhy, AX_VOID **YUVDataVir, AX_U32 nFrameSize, AX_U64 nFrameId, AX_BLK *nBlkId); +AX_VOID FrameMgrRelease(AX_BLK nBlkId); +AX_VOID FrameMgrDestroy(AX_VOID); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/picojson.h b/projects/llm_framework/main_skel/src/runner/picojson.h new file mode 100644 index 00000000..04dab8ba --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/picojson.h @@ -0,0 +1,1151 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef picojson_h +#define picojson_h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// for isnan/isinf +#if __cplusplus >= 201103L +#include +#else +extern "C" { +#ifdef _MSC_VER +#include +#elif defined(__INTEL_COMPILER) +#include +#else +#include +#endif +} +#endif + +#ifndef PICOJSON_USE_RVALUE_REFERENCE +#if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || (defined(_MSC_VER) && _MSC_VER >= 1600) +#define PICOJSON_USE_RVALUE_REFERENCE 1 +#else +#define PICOJSON_USE_RVALUE_REFERENCE 0 +#endif +#endif // PICOJSON_USE_RVALUE_REFERENCE + +#ifndef PICOJSON_NOEXCEPT +#if PICOJSON_USE_RVALUE_REFERENCE +#define PICOJSON_NOEXCEPT noexcept +#else +#define PICOJSON_NOEXCEPT throw() +#endif +#endif + +// experimental support for int64_t (see README.mkdn for detail) +#ifdef PICOJSON_USE_INT64 +#define __STDC_FORMAT_MACROS +#include +#include +#endif + +// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 +#ifndef PICOJSON_USE_LOCALE +#define PICOJSON_USE_LOCALE 1 +#endif +#if PICOJSON_USE_LOCALE +extern "C" { +#include +} +#endif + +#ifndef PICOJSON_ASSERT +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) \ + throw std::runtime_error(#e); \ + } while (0) +#endif + +#ifdef _MSC_VER +#define SNPRINTF _snprintf_s +#pragma warning(push) +#pragma warning(disable : 4244) // conversion from int to char +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4702) // unreachable code +#else +#define SNPRINTF snprintf +#endif + +namespace picojson { + +enum { + null_type, + boolean_type, + number_type, + string_type, + array_type, + object_type +#ifdef PICOJSON_USE_INT64 + , + int64_type +#endif +}; + +enum { INDENT_WIDTH = 2 }; + +struct null {}; + +class value { +public: + typedef std::vector array; + typedef std::map object; + union _storage { + bool boolean_; + double number_; +#ifdef PICOJSON_USE_INT64 + int64_t int64_; +#endif + std::string *string_; + array *array_; + object *object_; + }; + +protected: + int type_; + _storage u_; + +public: + value(); + value(int type, bool); + explicit value(bool b); +#ifdef PICOJSON_USE_INT64 + explicit value(int64_t i); +#endif + explicit value(double n); + explicit value(const std::string &s); + explicit value(const array &a); + explicit value(const object &o); +#if PICOJSON_USE_RVALUE_REFERENCE + explicit value(std::string &&s); + explicit value(array &&a); + explicit value(object &&o); +#endif + explicit value(const char *s); + value(const char *s, size_t len); + ~value(); + value(const value &x); + value &operator=(const value &x); +#if PICOJSON_USE_RVALUE_REFERENCE + value(value &&x) PICOJSON_NOEXCEPT; + value &operator=(value &&x) PICOJSON_NOEXCEPT; +#endif + void swap(value &x) PICOJSON_NOEXCEPT; + template bool is() const; + template const T &get() const; + template T &get(); + template void set(const T &); +#if PICOJSON_USE_RVALUE_REFERENCE + template void set(T &&); +#endif + bool evaluate_as_boolean() const; + const value &get(const size_t idx) const; + const value &get(const std::string &key) const; + value &get(const size_t idx); + value &get(const std::string &key); + + bool contains(const size_t idx) const; + bool contains(const std::string &key) const; + std::string to_str() const; + template void serialize(Iter os, bool prettify = false) const; + std::string serialize(bool prettify = false) const; + +private: + template value(const T *); // intentionally defined to block implicit conversion of pointer to bool + template static void _indent(Iter os, int indent); + template void _serialize(Iter os, int indent) const; + std::string _serialize(int indent) const; + void clear(); +}; + +typedef value::array array; +typedef value::object object; + +inline value::value() : type_(null_type), u_() { +} + +inline value::value(int type, bool) : type_(type), u_() { + switch (type) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(boolean_, false); + INIT(number_, 0.0); +#ifdef PICOJSON_USE_INT64 + INIT(int64_, 0); +#endif + INIT(string_, new std::string()); + INIT(array_, new array()); + INIT(object_, new object()); +#undef INIT + default: + break; + } +} + +inline value::value(bool b) : type_(boolean_type), u_() { + u_.boolean_ = b; +} + +#ifdef PICOJSON_USE_INT64 +inline value::value(int64_t i) : type_(int64_type), u_() { + u_.int64_ = i; +} +#endif + +inline value::value(double n) : type_(number_type), u_() { + if ( +#ifdef _MSC_VER + !_finite(n) +#elif __cplusplus >= 201103L || !(defined(isnan) && defined(isinf)) + std::isnan(n) || std::isinf(n) +#else + isnan(n) || isinf(n) +#endif + ) { + throw std::overflow_error(""); + } + u_.number_ = n; +} + +inline value::value(const std::string &s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const array &a) : type_(array_type), u_() { + u_.array_ = new array(a); +} + +inline value::value(const object &o) : type_(object_type), u_() { + u_.object_ = new object(o); +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(std::string &&s) : type_(string_type), u_() { + u_.string_ = new std::string(std::move(s)); +} + +inline value::value(array &&a) : type_(array_type), u_() { + u_.array_ = new array(std::move(a)); +} + +inline value::value(object &&o) : type_(object_type), u_() { + u_.object_ = new object(std::move(o)); +} +#endif + +inline value::value(const char *s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const char *s, size_t len) : type_(string_type), u_() { + u_.string_ = new std::string(s, len); +} + +inline void value::clear() { + switch (type_) { +#define DEINIT(p) \ + case p##type: \ + delete u_.p; \ + break + DEINIT(string_); + DEINIT(array_); + DEINIT(object_); +#undef DEINIT + default: + break; + } +} + +inline value::~value() { + clear(); +} + +inline value::value(const value &x) : type_(x.type_), u_() { + switch (type_) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(string_, new std::string(*x.u_.string_)); + INIT(array_, new array(*x.u_.array_)); + INIT(object_, new object(*x.u_.object_)); +#undef INIT + default: + u_ = x.u_; + break; + } +} + +inline value &value::operator=(const value &x) { + if (this != &x) { + value t(x); + swap(t); + } + return *this; +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(value &&x) PICOJSON_NOEXCEPT : type_(null_type), u_() { + swap(x); +} +inline value &value::operator=(value &&x) PICOJSON_NOEXCEPT { + swap(x); + return *this; +} +#endif +inline void value::swap(value &x) PICOJSON_NOEXCEPT { + std::swap(type_, x.type_); + std::swap(u_, x.u_); +} + +#define IS(ctype, jtype) \ + template <> inline bool value::is() const { \ + return type_ == jtype##_type; \ + } +IS(null, null) +IS(bool, boolean) +#ifdef PICOJSON_USE_INT64 +IS(int64_t, int64) +#endif +IS(std::string, string) +IS(array, array) +IS(object, object) +#undef IS +template <> inline bool value::is() const { + return type_ == number_type +#ifdef PICOJSON_USE_INT64 + || type_ == int64_type +#endif + ; +} + +#define GET(ctype, var) \ + template <> inline const ctype &value::get() const { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } \ + template <> inline ctype &value::get() { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } +GET(bool, u_.boolean_) +GET(std::string, *u_.string_) +GET(array, *u_.array_) +GET(object, *u_.object_) +#ifdef PICOJSON_USE_INT64 +GET(double, + (type_ == int64_type && (const_cast(this)->type_ = number_type, const_cast(this)->u_.number_ = u_.int64_), + u_.number_)) +GET(int64_t, u_.int64_) +#else +GET(double, u_.number_) +#endif +#undef GET + +#define SET(ctype, jtype, setter) \ + template <> inline void value::set(const ctype &_val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +SET(bool, boolean, u_.boolean_ = _val;) +SET(std::string, string, u_.string_ = new std::string(_val);) +SET(array, array, u_.array_ = new array(_val);) +SET(object, object, u_.object_ = new object(_val);) +SET(double, number, u_.number_ = _val;) +#ifdef PICOJSON_USE_INT64 +SET(int64_t, int64, u_.int64_ = _val;) +#endif +#undef SET + +#if PICOJSON_USE_RVALUE_REFERENCE +#define MOVESET(ctype, jtype, setter) \ + template <> inline void value::set(ctype && _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) +MOVESET(array, array, u_.array_ = new array(std::move(_val));) +MOVESET(object, object, u_.object_ = new object(std::move(_val));) +#undef MOVESET +#endif + +inline bool value::evaluate_as_boolean() const { + switch (type_) { + case null_type: + return false; + case boolean_type: + return u_.boolean_; + case number_type: + return u_.number_ != 0; +#ifdef PICOJSON_USE_INT64 + case int64_type: + return u_.int64_ != 0; +#endif + case string_type: + return !u_.string_->empty(); + default: + return true; + } +} + +inline const value &value::get(const size_t idx) const { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline value &value::get(const size_t idx) { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline const value &value::get(const std::string &key) const { + static value s_null; + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline value &value::get(const std::string &key) { + static value s_null; + PICOJSON_ASSERT(is()); + object::iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline bool value::contains(const size_t idx) const { + PICOJSON_ASSERT(is()); + return idx < u_.array_->size(); +} + +inline bool value::contains(const std::string &key) const { + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end(); +} + +inline std::string value::to_str() const { + switch (type_) { + case null_type: + return "null"; + case boolean_type: + return u_.boolean_ ? "true" : "false"; +#ifdef PICOJSON_USE_INT64 + case int64_type: { + char buf[sizeof("-9223372036854775808")]; + SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); + return buf; + } +#endif + case number_type: { + char buf[256]; + double tmp; + SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); +#if PICOJSON_USE_LOCALE + char *decimal_point = localeconv()->decimal_point; + if (strcmp(decimal_point, ".") != 0) { + size_t decimal_point_len = strlen(decimal_point); + for (char *p = buf; *p != '\0'; ++p) { + if (strncmp(p, decimal_point, decimal_point_len) == 0) { + return std::string(buf, p) + "." + (p + decimal_point_len); + } + } + } +#endif + return buf; + } + case string_type: + return *u_.string_; + case array_type: + return "array"; + case object_type: + return "object"; + default: + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + } + return std::string(); +} + +template void copy(const std::string &s, Iter oi) { + std::copy(s.begin(), s.end(), oi); +} + +template struct serialize_str_char { + Iter oi; + void operator()(char c) { + switch (c) { +#define MAP(val, sym) \ + case val: \ + copy(sym, oi); \ + break + MAP('"', "\\\""); + MAP('\\', "\\\\"); + MAP('/', "\\/"); + MAP('\b', "\\b"); + MAP('\f', "\\f"); + MAP('\n', "\\n"); + MAP('\r', "\\r"); + MAP('\t', "\\t"); +#undef MAP + default: + if (static_cast(c) < 0x20 || c == 0x7f) { + char buf[7]; + SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); + copy(buf, buf + 6, oi); + } else { + *oi++ = c; + } + break; + } + } +}; + +template void serialize_str(const std::string &s, Iter oi) { + *oi++ = '"'; + serialize_str_char process_char = {oi}; + std::for_each(s.begin(), s.end(), process_char); + *oi++ = '"'; +} + +template void value::serialize(Iter oi, bool prettify) const { + return _serialize(oi, prettify ? 0 : -1); +} + +inline std::string value::serialize(bool prettify) const { + return _serialize(prettify ? 0 : -1); +} + +template void value::_indent(Iter oi, int indent) { + *oi++ = '\n'; + for (int i = 0; i < indent * INDENT_WIDTH; ++i) { + *oi++ = ' '; + } +} + +template void value::_serialize(Iter oi, int indent) const { + switch (type_) { + case string_type: + serialize_str(*u_.string_, oi); + break; + case array_type: { + *oi++ = '['; + if (indent != -1) { + ++indent; + } + for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { + if (i != u_.array_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + i->_serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.array_->empty()) { + _indent(oi, indent); + } + } + *oi++ = ']'; + break; + } + case object_type: { + *oi++ = '{'; + if (indent != -1) { + ++indent; + } + for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { + if (i != u_.object_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(i->first, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + i->second._serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.object_->empty()) { + _indent(oi, indent); + } + } + *oi++ = '}'; + break; + } + default: + copy(to_str(), oi); + break; + } + if (indent == 0) { + *oi++ = '\n'; + } +} + +inline std::string value::_serialize(int indent) const { + std::string s; + _serialize(std::back_inserter(s), indent); + return s; +} + +template class input { +protected: + Iter cur_, end_; + bool consumed_; + int line_; + +public: + input(const Iter &first, const Iter &last) : cur_(first), end_(last), consumed_(false), line_(1) { + } + int getc() { + if (consumed_) { + if (*cur_ == '\n') { + ++line_; + } + ++cur_; + } + if (cur_ == end_) { + consumed_ = false; + return -1; + } + consumed_ = true; + return *cur_ & 0xff; + } + void ungetc() { + consumed_ = false; + } + Iter cur() const { + if (consumed_) { + input *self = const_cast *>(this); + self->consumed_ = false; + ++self->cur_; + } + return cur_; + } + int line() const { + return line_; + } + void skip_ws() { + while (1) { + int ch = getc(); + if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { + ungetc(); + break; + } + } + } + bool expect(const int expected) { + skip_ws(); + if (getc() != expected) { + ungetc(); + return false; + } + return true; + } + bool match(const std::string &pattern) { + for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { + if (getc() != *pi) { + ungetc(); + return false; + } + } + return true; + } +}; + +template inline int _parse_quadhex(input &in) { + int uni_ch = 0, hex; + for (int i = 0; i < 4; i++) { + if ((hex = in.getc()) == -1) { + return -1; + } + if ('0' <= hex && hex <= '9') { + hex -= '0'; + } else if ('A' <= hex && hex <= 'F') { + hex -= 'A' - 0xa; + } else if ('a' <= hex && hex <= 'f') { + hex -= 'a' - 0xa; + } else { + in.ungetc(); + return -1; + } + uni_ch = uni_ch * 16 + hex; + } + return uni_ch; +} + +template inline bool _parse_codepoint(String &out, input &in) { + int uni_ch; + if ((uni_ch = _parse_quadhex(in)) == -1) { + return false; + } + if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { + if (0xdc00 <= uni_ch) { + // a second 16-bit of a surrogate pair appeared + return false; + } + // first 16-bit of surrogate pair, get the next one + if (in.getc() != '\\' || in.getc() != 'u') { + in.ungetc(); + return false; + } + int second = _parse_quadhex(in); + if (!(0xdc00 <= second && second <= 0xdfff)) { + return false; + } + uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); + uni_ch += 0x10000; + } + if (uni_ch < 0x80) { + out.push_back(static_cast(uni_ch)); + } else { + if (uni_ch < 0x800) { + out.push_back(static_cast(0xc0 | (uni_ch >> 6))); + } else { + if (uni_ch < 0x10000) { + out.push_back(static_cast(0xe0 | (uni_ch >> 12))); + } else { + out.push_back(static_cast(0xf0 | (uni_ch >> 18))); + out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); + } + out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); + } + out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); + } + return true; +} + +template inline bool _parse_string(String &out, input &in) { + while (1) { + int ch = in.getc(); + if (ch < ' ') { + in.ungetc(); + return false; + } else if (ch == '"') { + return true; + } else if (ch == '\\') { + if ((ch = in.getc()) == -1) { + return false; + } + switch (ch) { +#define MAP(sym, val) \ + case sym: \ + out.push_back(val); \ + break + MAP('"', '\"'); + MAP('\\', '\\'); + MAP('/', '/'); + MAP('b', '\b'); + MAP('f', '\f'); + MAP('n', '\n'); + MAP('r', '\r'); + MAP('t', '\t'); +#undef MAP + case 'u': + if (!_parse_codepoint(out, in)) { + return false; + } + break; + default: + return false; + } + } else { + out.push_back(static_cast(ch)); + } + } + return false; +} + +template inline bool _parse_array(Context &ctx, input &in) { + if (!ctx.parse_array_start()) { + return false; + } + size_t idx = 0; + if (in.expect(']')) { + return ctx.parse_array_stop(idx); + } + do { + if (!ctx.parse_array_item(in, idx)) { + return false; + } + idx++; + } while (in.expect(',')); + return in.expect(']') && ctx.parse_array_stop(idx); +} + +template inline bool _parse_object(Context &ctx, input &in) { + if (!ctx.parse_object_start()) { + return false; + } + if (in.expect('}')) { + return true; + } + do { + std::string key; + if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { + return false; + } + if (!ctx.parse_object_item(in, key)) { + return false; + } + } while (in.expect(',')); + return in.expect('}'); +} + +template inline std::string _parse_number(input &in) { + std::string num_str; + while (1) { + int ch = in.getc(); + if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { + num_str.push_back(static_cast(ch)); + } else if (ch == '.') { +#if PICOJSON_USE_LOCALE + num_str += localeconv()->decimal_point; +#else + num_str.push_back('.'); +#endif + } else { + in.ungetc(); + break; + } + } + return num_str; +} + +template inline bool _parse(Context &ctx, input &in) { + in.skip_ws(); + int ch = in.getc(); + switch (ch) { +#define IS(ch, text, op) \ + case ch: \ + if (in.match(text) && op) { \ + return true; \ + } else { \ + return false; \ + } + IS('n', "ull", ctx.set_null()); + IS('f', "alse", ctx.set_bool(false)); + IS('t', "rue", ctx.set_bool(true)); +#undef IS + case '"': + return ctx.parse_string(in); + case '[': + return _parse_array(ctx, in); + case '{': + return _parse_object(ctx, in); + default: + if (('0' <= ch && ch <= '9') || ch == '-') { + double f; + char *endp; + in.ungetc(); + std::string num_str(_parse_number(in)); + if (num_str.empty()) { + return false; + } +#ifdef PICOJSON_USE_INT64 + { + errno = 0; + intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); + if (errno == 0 && std::numeric_limits::min() <= ival && ival <= std::numeric_limits::max() && + endp == num_str.c_str() + num_str.size()) { + ctx.set_int64(ival); + return true; + } + } +#endif + f = strtod(num_str.c_str(), &endp); + if (endp == num_str.c_str() + num_str.size()) { + ctx.set_number(f); + return true; + } + return false; + } + break; + } + in.ungetc(); + return false; +} + +class deny_parse_context { +public: + bool set_null() { + return false; + } + bool set_bool(bool) { + return false; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { + return false; + } +#endif + bool set_number(double) { + return false; + } + template bool parse_string(input &) { + return false; + } + bool parse_array_start() { + return false; + } + template bool parse_array_item(input &, size_t) { + return false; + } + bool parse_array_stop(size_t) { + return false; + } + bool parse_object_start() { + return false; + } + template bool parse_object_item(input &, const std::string &) { + return false; + } +}; + +class default_parse_context { +protected: + value *out_; + +public: + default_parse_context(value *out) : out_(out) { + } + bool set_null() { + *out_ = value(); + return true; + } + bool set_bool(bool b) { + *out_ = value(b); + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t i) { + *out_ = value(i); + return true; + } +#endif + bool set_number(double f) { + *out_ = value(f); + return true; + } + template bool parse_string(input &in) { + *out_ = value(string_type, false); + return _parse_string(out_->get(), in); + } + bool parse_array_start() { + *out_ = value(array_type, false); + return true; + } + template bool parse_array_item(input &in, size_t) { + array &a = out_->get(); + a.push_back(value()); + default_parse_context ctx(&a.back()); + return _parse(ctx, in); + } + bool parse_array_stop(size_t) { + return true; + } + bool parse_object_start() { + *out_ = value(object_type, false); + return true; + } + template bool parse_object_item(input &in, const std::string &key) { + object &o = out_->get(); + default_parse_context ctx(&o[key]); + return _parse(ctx, in); + } + +private: + default_parse_context(const default_parse_context &); + default_parse_context &operator=(const default_parse_context &); +}; + +class null_parse_context { +public: + struct dummy_str { + void push_back(int) { + } + }; + +public: + null_parse_context() { + } + bool set_null() { + return true; + } + bool set_bool(bool) { + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { + return true; + } +#endif + bool set_number(double) { + return true; + } + template bool parse_string(input &in) { + dummy_str s; + return _parse_string(s, in); + } + bool parse_array_start() { + return true; + } + template bool parse_array_item(input &in, size_t) { + return _parse(*this, in); + } + bool parse_array_stop(size_t) { + return true; + } + bool parse_object_start() { + return true; + } + template bool parse_object_item(input &in, const std::string &) { + return _parse(*this, in); + } + +private: + null_parse_context(const null_parse_context &); + null_parse_context &operator=(const null_parse_context &); +}; + +// obsolete, use the version below +template inline std::string parse(value &out, Iter &pos, const Iter &last) { + std::string err; + pos = parse(out, pos, last, &err); + return err; +} + +template inline Iter _parse(Context &ctx, const Iter &first, const Iter &last, std::string *err) { + input in(first, last); + if (!_parse(ctx, in) && err != NULL) { + char buf[64]; + SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); + *err = buf; + while (1) { + int ch = in.getc(); + if (ch == -1 || ch == '\n') { + break; + } else if (ch >= ' ') { + err->push_back(static_cast(ch)); + } + } + } + return in.cur(); +} + +template inline Iter parse(value &out, const Iter &first, const Iter &last, std::string *err) { + default_parse_context ctx(&out); + return _parse(ctx, first, last, err); +} + +inline std::string parse(value &out, const std::string &s) { + std::string err; + parse(out, s.begin(), s.end(), &err); + return err; +} + +inline std::string parse(value &out, std::istream &is) { + std::string err; + parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); + return err; +} + +template struct last_error_t { static std::string s; }; +template std::string last_error_t::s; + +inline void set_last_error(const std::string &s) { + last_error_t::s = s; +} + +inline const std::string &get_last_error() { + return last_error_t::s; +} + +inline bool operator==(const value &x, const value &y) { + if (x.is()) + return y.is(); +#define PICOJSON_CMP(type) \ + if (x.is()) \ + return y.is() && x.get() == y.get() + PICOJSON_CMP(bool); + PICOJSON_CMP(double); + PICOJSON_CMP(std::string); + PICOJSON_CMP(array); + PICOJSON_CMP(object); +#undef PICOJSON_CMP + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + return false; +} + +inline bool operator!=(const value &x, const value &y) { + return !(x == y); +} +} + +#if !PICOJSON_USE_RVALUE_REFERENCE +namespace std { +template <> inline void swap(picojson::value &x, picojson::value &y) { + x.swap(y); +} +} +#endif + +inline std::istream &operator>>(std::istream &is, picojson::value &x) { + picojson::set_last_error(std::string()); + const std::string err(picojson::parse(x, is)); + if (!err.empty()) { + picojson::set_last_error(err); + is.setstate(std::ios::failbit); + } + return is; +} + +inline std::ostream &operator<<(std::ostream &os, const picojson::value &x) { + x.serialize(std::ostream_iterator(os)); + return os; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#endif diff --git a/projects/llm_framework/main_skel/src/runner/sample_skel.c b/projects/llm_framework/main_skel/src/runner/sample_skel.c new file mode 100644 index 00000000..b1e9364a --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/sample_skel.c @@ -0,0 +1,2285 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "YuvHandler.h" +#include "ax_skel_api.h" +#include "ax_sys_api.h" +#include "ax_ivps_api.h" +#include "ax_venc_api.h" +#include "ax_buffer_tool.h" +#include "skel_log.h" +#include "ax_engine_api.h" +#include "frameMgr.h" +#include "attrParser.h" +#include "statMgr.h" +#include "common_vdec_api.h" + +#ifndef ALIGN_UP +#define ALIGN_UP(x, align) ((((x) + ((align) - 1)) / (align)) * (align)) +#endif + +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, align) (((x) / (align)) * (align)) +#endif + +#ifndef AX_MAX +#define AX_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef AX_MIN +#define AX_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" +#define SKEL_SAMPLE_OUTPUT_VEHICLE_PATH "vehicle" +#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" +#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" +#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" +#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" + +#define SAMPLE_SKEL_SHIFT_LEFT_ALIGN(a) (1 << (a)) +#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) +#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) + +#define RUN_COMMAND(format, ...) do { \ + char cmd[512]; \ + snprintf(cmd, 512, format, ##__VA_ARGS__); \ + system(cmd); \ + } while(0); + +AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) { + if (file) { + va_list args; + char szLog[1024] = {0}; + + va_start(args, fmt); + + AX_U32 len = vsnprintf(szLog, sizeof(szLog), (char *)fmt, args); + if (len < (AX_U32)sizeof(szLog)) { + szLog[len] = '\0'; + } else { + szLog[sizeof(szLog) - 1] = '\0'; + } + + fwrite((AX_U8 *)szLog, 1, strlen(szLog), file); + + va_end(args); + } +} +#define OUTPUT_LOG_SAVE(fmt, ...) LogSaveToFile(fpResultFile, fmt "\n", ##__VA_ARGS__) +#define BLACK_VIDEO_FRAME_COUNT 60 +#define SKEL_SAMPLE_OBJECT_SIZE 512 + +struct skeleton { + int connection[2]; + int left_right_neutral; +}; + +struct skeleton pairs[] = {{{15, 13}, 0}, {{13, 11}, 0}, {{16, 14}, 0}, {{14, 12}, 0}, {{11, 12}, 0}, + {{5, 11}, 0}, {{6, 12}, 0}, {{5, 6}, 0}, {{5, 7}, 0}, {{6, 8}, 0}, + {{7, 9}, 0}, {{8, 10}, 0}, {{1, 2}, 0}, {{0, 1}, 0}, {{0, 2}, 0}, + {{1, 3}, 0}, {{2, 4}, 0}, {{0, 5}, 0}, {{0, 6}, 0}}; + +#define DETECT_SKEL_POINT_COUNT 256 + +typedef struct _AI_Detection_Box_t { + AX_F32 fX, fY, fW, fH; +} AI_Detection_Box_t; + +typedef struct _AI_Detection_Point_t { + AX_F32 fX, fY; +} AI_Detection_Point_t; + +typedef struct _AI_Detection_SkelResult_t { + const AX_CHAR *pstrObjectCategory; + AX_U8 nPointNum; + AI_Detection_Point_t tPoint[DETECT_SKEL_POINT_COUNT]; + AI_Detection_Box_t tBox; +} AI_Detection_SkelResult_t; + +static AX_U64 get_tick_count(AX_VOID) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + +// static AX_U64 get_tick_count_us(AX_VOID) { +// struct timespec ts; +// clock_gettime(CLOCK_MONOTONIC, &ts); +// return (ts.tv_sec * 1000000 + ts.tv_nsec / 1000); +// } + +static AX_VOID ShowUsage(AX_VOID) { + // printf("usage: ./%s ...\n", SAMPLE_SKEL_NAME); + printf("options:\n"); + printf("-i, \tInput File(yuv)\n"); + printf("-r, \tInput File Resolution(wxh)(yuv: should be input)\n"); + printf("-w, \tWrite result image to new jpg file((unsigned int), default=0)\n"); + printf("-o, \tSave result to file(file name)\n"); + printf("-m, \tModels deployment path(path name)\n"); + printf("-t, \tRepeat times((unsigned int), default=1)\n"); + printf("-I, \tInterval repeat time((unsigned int)ms, default=0)\n"); + printf("-c, \tConfidence((float: 0-1), default=0)\n"); + printf("-H, \tHuman track size limit((unsigned int), default=0)\n"); + printf("-V, \tVehicle track size limit((unsigned int), default=0)\n"); + printf("-C, \tCylcle track size limit((unsigned int), default=0)\n"); + printf("-d, \tSkel detect type((unsigned int), default=2)\n" + "\t\t0: detect only\n" + "\t\t1: detect + track\n" + "\t\t2: detect + track + push\n"); + printf("-u, \tSkel push strategy((unsigned int), default=3)\n" + "\t\t1: fast push strategy\n" + "\t\t2: push strategy\n" + "\t\t3: best push strategy\n"); + printf("-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" + "\t\t0: VNPU Disable\n" + "\t\t1: STD-VNPU Default\n" + "\t\t2: STD-VNPU1\n" + "\t\t3: STD-VNPU2\n"); + printf("-p, \tSkel PPL((unsigned int), default=1)\n" + "\t\t1: AX_SKEL_PPL_HVCP\n" + "\t\t2: AX_SKEL_PPL_FACE\n"); + printf("-v, \tLog level((unsigned int), default=5)\n" + "\t\t0: LOG_EMERGENCY_LEVEL\n" + "\t\t1: LOG_ALERT_LEVEL\n" + "\t\t2: LOG_CRITICAL_LEVEL\n" + "\t\t3: LOG_ERROR_LEVEL\n" + "\t\t4: LOG_WARN_LEVEL\n" + "\t\t5: LOG_NOTICE_LEVEL\n" + "\t\t6: LOG_INFO_LEVEL\n" + "\t\t7: LOG_DEBUG_LEVEL\n"); + printf("-h, \tprint this message\n"); +} + +AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { + if (pstConfig->nSize > 0 && pstConfig->pstItems) { + for (size_t i = 0; i < pstConfig->nSize; i++) { + if (pstConfig->pstItems[i].pstrType && pstConfig->pstItems[i].pstrValue) { + // cmd: "body_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + if (strcmp(pstConfig->pstItems[i].pstrType, "body_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_max_target_count") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_confidence") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "crop_encoder_qpLevel", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "crop_encoder_qpLevel") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_min_size") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T)) { + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *pstConf = + (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "detect_roi_polygon") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T)) { + AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, pstConf->nPointNum); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_strategy", value_type: AX_SKEL_PUSH_STRATEGY_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_strategy") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_STRATEGY_T)) { + AX_SKEL_PUSH_STRATEGY_T *pstConf = (AX_SKEL_PUSH_STRATEGY_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, pstConf->ePushMode, pstConf->nIntervalTimes, + pstConf->nPushCounts, pstConf->bPushSameFrame); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "body_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "body_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "vehicle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "cycle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "face_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "face_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "plate_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_crop_encoder") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, + pstConf->fScaleLeft, pstConf->fScaleRight, + pstConf->fScaleTop, pstConf->fScaleBottom); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_panorama", value_type: AX_SKEL_PUSH_PANORAMA_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_panorama") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T)) { + AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Enable: %d, Quality: %d]", pstConfig->pstItems[i].pstrType, + pstConf->bEnable, pstConf->nQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_body", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_body") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_vehicle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_vehicle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_cycle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_cycle") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_face", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_face") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [W: %d, H: %d, P: %f, Y: %f, R: %f, B: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, + pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, + pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_quality_plate", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_plate") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_bind_enable", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_bind_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "track_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_enable") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "target_config", value_type: AX_SKEL_TARGET_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_target_config", value_type: AX_SKEL_TARGET_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_target_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { + AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->pstItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + if (pstConf->pstItems[j].pstrObjectCategory) { + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + } + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "analyzer_attr_config", value_type: AX_SKEL_ANALYZER_CONFIG_T * + else if (strcmp(pstConfig->pstItems[i].pstrType, "analyzer_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ANALYZER_CONFIG_T)) { + AX_SKEL_ANALYZER_CONFIG_T *pstConf = (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + if (pstConf->peItems) { + for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + ALOGI("SKEL set %s: [%d]%d", pstConfig->pstItems[i].pstrType, j, pstConf->peItems[j]); + } + } + else { + ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); + } + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "venc_attr_config", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "venc_attr_config") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, (AX_U32)pstConf->fValue); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + // cmd: "push_attr_always", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + else if (strcmp(pstConfig->pstItems[i].pstrType, "push_attr_always") == 0) { + if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); + } else { + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + } + } + else { + ALOGE("SKEL cmd: %s not support", pstConfig->pstItems[i].pstrType); + } + } + } + } + + return 0; +} + +FILE* LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + + if (f_in) { + if (pLen) { + fseeko(f_in, 0L, SEEK_END); + AX_U64 nFileSize = ftello(f_in); + rewind(f_in); + + *pLen = (AX_U64)nFileSize; + } + } + + return f_in; +} + +int ReLoadFile(FILE* pFile) +{ + if (pFile) { + rewind(pFile); + return 0; + } + + return -1; +} + +int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, + AX_VOID *pSrcVirAddr, + AX_U32 nSrcSize, + AX_U32 nSrcStride, + AX_U32 nSrcHeight, + AX_U64 nDstPhyAddr, + AX_VOID *pDstVirAddr, + AX_U32 nDstSize, + AX_U32 nDstStride, + AX_U32 nDstHeight + ) { + AX_U64 nStartTime = get_tick_count(); + + // src check + if (!pSrcVirAddr) { + return -1; + } + + if (nSrcSize != nSrcStride * nSrcHeight * 3 /2) { + return -1; + } + + // dst check + if (!pDstVirAddr) { + return -1; + } + + if (nDstSize != nDstStride * nDstHeight * 3 /2) { + return -1; + } + + if (nDstSize < nSrcSize) { + return -1; + } + + if (nDstStride < nSrcStride) { + return -1; + } + + if (nDstHeight != nSrcHeight) { + return -1; + } + + AX_VOID *src = pSrcVirAddr; + AX_VOID *dst = pDstVirAddr; + for(AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { + memcpy(dst, src, nSrcStride); + src += nSrcStride; + dst += nDstStride; + } + + AX_U64 nProcessElasped = get_tick_count() - nStartTime; + + ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, nProcessElasped); + + return 0; +} + +int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) +{ + AX_S32 nReadSize = 0; + + if (pFile) { + if (nPhyAddr != 0 && pVirAddr) { + memset((AX_U8 *)pVirAddr, 0x00, nSize); + nReadSize = fread((AX_U8 *)pVirAddr, 1, nSize, pFile); + } + else { + fseeko(pFile, nSize, SEEK_CUR); + } + } + + return nReadSize; +} + +int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) +{ + SAMPLE_INPUT_FILE_INFO_T stStreamInfo; + SAMPLE_STREAM_BUF_T stStreamBuf; + size_t nReadLen = 0; + AX_U64 streamPhyAddr = 0; + AX_VOID *pStreamVirAddr = NULL; + AX_VDEC_DEC_ONE_FRM_T decOneFrmParam; + size_t nFileSize = 0; + AX_U32 heightAlign = 0; + AX_U32 frmStride = 0; + frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; + heightAlign = ALIGN_UP(nHeight, 16); + AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; + + memset(&stStreamInfo, 0, sizeof(SAMPLE_INPUT_FILE_INFO_T)); + memset(&stStreamBuf, 0, sizeof(SAMPLE_STREAM_BUF_T)); + memset(&decOneFrmParam, 0, sizeof(AX_VDEC_DEC_ONE_FRM_T)); + + rewind(InputFileHandle); + fseek(InputFileHandle, 0, SEEK_END); + nFileSize = (size_t)ftello(InputFileHandle); + + stStreamInfo.fInput = InputFileHandle; + stStreamInfo.sFileSize = nFileSize; + + stStreamBuf.uBufSize = ((AX_U32)nFileSize) > STREAM_BUFFER_MAX_SIZE ? STREAM_BUFFER_MAX_SIZE : nFileSize; + int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, + stStreamBuf.uBufSize, 0x100, (AX_S8 *)"vdec_input_stream"); + if (ret != AX_SUCCESS) { + ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", + stStreamBuf.uBufSize, ret); + return -1; + } + + stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; + stStreamBuf.tBufAddr.u64PhyAddr = streamPhyAddr; + + ret = StreamParserReadFrameJpeg(&stStreamInfo, &stStreamBuf, &nReadLen); + if (0 != ret) { + ALOGE("StreamParserReadFrameJpeg failed! ret = 0x%x", ret); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return -1; + } + + if (!nReadLen) { + ALOGE("read jpeg frame FAILED!\n"); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + ret = AX_ERR_VDEC_STRM_ERROR; + return ret; + } + + decOneFrmParam.stStream.pu8Addr = (AX_U8*)stStreamBuf.tBufAddr.pVirAddr; + decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; + decOneFrmParam.stStream.u32StreamPackLen = (AX_U32)nReadLen; + + decOneFrmParam.stFrame.u64VirAddr[0] = (AX_ULONG)OneYUVDataVir; + decOneFrmParam.stFrame.u64VirAddr[1] = (AX_ULONG)OneYUVDataVir + frmStride * heightAlign; + decOneFrmParam.stFrame.u64PhyAddr[0] = OneYUVDataPhy; + decOneFrmParam.stFrame.u64PhyAddr[1] = OneYUVDataPhy + frmStride * heightAlign; + decOneFrmParam.stFrame.u32FrameSize = nFrameSize; + + ret = AX_VDEC_JpegDecodeOneFrame(&decOneFrmParam); + if (ret != AX_SUCCESS) { + ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", + ret, SampleVdecRetStr(ret)); + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + return ret; + } + + AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); + + return 0; +} + +int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr, AX_U32 *pLen) +{ + /* Reading input file */ + FILE *f_in = fopen(pFile, "rb"); + if (f_in == NULL) { + ALOGE("Unable to open input file\n"); + return -1; + } + + /* file i/o pointer to full */ + fseek(f_in, 0L, SEEK_END); + AX_U32 nFileSize = ftell(f_in); + rewind(f_in); + + if (pPhyAddr && ppVirAddr) { + AX_U64 nPhyAddr = 0; + AX_VOID *pVirAddr = NULL; + + // pVirAddr = (AX_VOID *)malloc(nFileSize); + AX_S32 nRet = AX_SYS_MemAlloc(&nPhyAddr, (AX_VOID **)&pVirAddr, nFileSize, 128, (AX_S8 *)"SKEL_TEST"); + + if (nRet != 0) { + fclose(f_in); + ALOGE("AX_SYS_MemAlloc failed"); + return -1; + } + + fread((AX_U8 *)pVirAddr, nFileSize, 1, f_in); + + *pPhyAddr = nPhyAddr; + *ppVirAddr = pVirAddr; + } + + if (pLen) { + *pLen = nFileSize; + } + + fclose(f_in); + + return 0; +} + +int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_U32 nLen) { + AX_S32 s32Ret = AX_SUCCESS; + + AX_U32 frameSize = nLen; + AX_JPEG_ENCODE_ONCE_PARAMS_T stJpegEncodeOnceParam; + memset(&stJpegEncodeOnceParam, 0, sizeof(stJpegEncodeOnceParam)); + + AX_U64 outPhyAddr = 0; + AX_VOID *outVirAddr = NULL; + + s32Ret = AX_SYS_MemAlloc(&outPhyAddr, &outVirAddr, frameSize, 128, (AX_S8 *)"SKEL-VENC"); + if (s32Ret) { + ALOGE("alloc mem err, size(%d).", frameSize); + return -1; + } + + // output + stJpegEncodeOnceParam.u32OutBufSize = frameSize; + stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; + stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; + + // input + stJpegEncodeOnceParam.stJpegParam.u32Qfactor = 90; + stJpegEncodeOnceParam.u32Width = nWidth; + stJpegEncodeOnceParam.u32Height = nHeight; + stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stJpegEncodeOnceParam.u32PicStride[0] = nStride; + stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; + stJpegEncodeOnceParam.u32PicStride[2] = 0; + + stJpegEncodeOnceParam.u64PhyAddr[0] = nPhyAddr; + stJpegEncodeOnceParam.u64PhyAddr[1] = stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64PhyAddr[2] = 0; + + stJpegEncodeOnceParam.u64VirAddr[0] = (AX_ULONG)pVirAddr; + stJpegEncodeOnceParam.u64VirAddr[1] = stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64VirAddr[2] = 0; + + s32Ret = AX_VENC_JpegEncodeOneFrame(&stJpegEncodeOnceParam); + if (AX_SUCCESS != s32Ret) { + ALOGE("AX_VENC_JpegEncodeOneFrame fail, ret=0x%x", s32Ret); + s32Ret = -1; + goto JENC_EXIT; + } + +JENC_EXIT: + if (0 == s32Ret && dstFile) { + FILE *fp_w = fopen(dstFile, "wb"); + + if (fp_w) { + ALOGN("\tWrite new JPG result image to file: %s", dstFile); + fwrite((AX_U8 *)stJpegEncodeOnceParam.pu8Addr, stJpegEncodeOnceParam.u32Len, 1, fp_w); + fclose(fp_w); + } + } + + if (outPhyAddr && (NULL != outVirAddr)) { + AX_SYS_MemFree(outPhyAddr, outVirAddr); + } + + return s32Ret; +} + +AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) { + if (nFrameSeqNum < 1) { + nFrameSeqNum = 1; + } + + if (nSrcFrameRate == nDstFrameRate) { + return AX_FALSE; + } + + if (nDstFrameRate > nSrcFrameRate) { + return AX_FALSE; + } + + if ((nFrameSeqNum * nDstFrameRate / (nSrcFrameRate)) > ((nFrameSeqNum - 1) * nDstFrameRate / (nSrcFrameRate))) { + return AX_FALSE; + } else { + return AX_TRUE; + } +} + +AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { + AX_S32 nRet = 0; + AX_S32 c; + AX_S32 isExit = 0; + AX_BOOL bJpgFile = AX_FALSE; + FILE *InputFileHandle = NULL; + const AX_CHAR *InputFile = NULL; + const AX_CHAR *InputResolution = NULL; + AX_S32 InputFileFrameCnt = 1; + const AX_CHAR *ReWritePath = NULL; + const AX_CHAR *SaveResultPath = NULL; + const AX_CHAR *ModelsPath = NULL; + FILE *fpResultFile = NULL; + AX_S32 nRepeatTimes = 1; + AX_S32 nPPL = AX_SKEL_PPL_HVCP; + AX_S32 nDetectType = 2; + AX_S32 nPushStrategy = 3; + AX_S32 nNpuType = 0; + AX_S32 nInterval = 0; + AX_U32 nStride = 1920; + AX_U32 nWidth = 1920; + AX_U32 nHeight = 1080; + AX_U32 nJencBufSize = 0; + AX_U32 nFrameDepth = 1; + AX_U32 nOneSize = 0; + AX_U64 OneYUVDataPhy = 0; + AX_VOID *OneYUVDataVir = NULL; + AX_U32 nFrameSize = nStride * nHeight * 3 / 2; + AX_U32 nCacheListDepth = 1; + AX_S32 nSrcFrameRate = 25; + AX_S32 nDstFrameRate = 25; + AX_VOID *YUVDataVir = NULL; + AX_U64 YUVDataPhy = 0; + AX_VOID *YUVDataWrVir = NULL; + AX_U64 YUVDataWrPhy = 0; + AX_SKEL_HANDLE pHandle = NULL; + AX_SKEL_FRAME_T stFrame = {0}; + AX_SKEL_RESULT_T *pstResult = NULL; + AX_U64 nFileSize = 0; + AX_U64 nStartTime = 0; + AX_U64 nInitElasped = 0; + AX_U64 nCreateElasped = 0; + AX_U64 nProcessElasped = 0; + AX_U64 nResultElasped = 0; + AX_U64 nResultElaspedMin = (AX_U64)-1; + AX_U64 nResultElaspedMax = 0; + AX_U64 nResultElaspedTotal = 0; + AX_F32 fConfidence = 0.0; + // AX_U32 nHumantracksize = 0; + // AX_U32 nVehicletracksize = 0; + // AX_U32 nCycletracksize = 0; + AX_U64 nSkelFrameId = 1; + +#if defined(SAMPLE_SKEL_BUILD_VERSION) + printf("SKEL sample: %s build: %s %s\n", SAMPLE_SKEL_BUILD_VERSION, __DATE__, __TIME__); +#endif + + while ((c = getopt(argc, argv, "i:r:I:w:o:m:t:d:u:p:j:v:c:N:H:V:C:h::")) != -1) { + isExit = 0; + switch (c) { + case 'i': + InputFile = (const AX_CHAR *)optarg; + break; + case 'r': + InputResolution = (const AX_CHAR *)optarg; + break; + case 'I': + nInterval = atoi(optarg); + break; + case 'w': + ReWritePath = (const AX_CHAR *)optarg; + break; + case 'o': + SaveResultPath = (const AX_CHAR *)optarg; + break; + case 'm': + ModelsPath = (const AX_CHAR *)optarg; + break; + case 't': + nRepeatTimes = atoi(optarg); + break; + case 'd': + nDetectType = atoi(optarg); + break; + case 'u': + nPushStrategy = atoi(optarg); + break; + case 'N': + nNpuType = atoi(optarg); + break; + case 'p': + nPPL = atoi(optarg); + break; + case 'j': + nJencBufSize = atoi(optarg); + break; + case 'c': + fConfidence = atof(optarg); + break; + // case 'H': + // nHumantracksize = atoi(optarg); + // break; + // case 'V': + // nVehicletracksize = atoi(optarg); + // break; + // case 'C': + // nCycletracksize = atoi(optarg); + // break; + case 'v': + log_level = atoi(optarg); + break; + case 'h': + isExit = 1; + break; + case 'D': + nCacheListDepth = atoi(optarg); + break; + default: + isExit = 1; + break; + } + } + + if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) + || (log_level < 0 || log_level >= SKEL_LOG_MAX) + || (fConfidence < 0 || fConfidence > 1) + || (nDetectType < 0 || nDetectType > 2) + || (nPushStrategy < 1 || nPushStrategy > 3) + || (nNpuType > 3)) { + ShowUsage(); + exit(0); + } + + if (InputFile) { + AX_U32 nInputFileLen = strlen(InputFile); + + if (nInputFileLen > 4 && (strcasecmp(&InputFile[nInputFileLen - 4], ".jpg") == 0)) { + bJpgFile = AX_TRUE; + } + } + + if (nRepeatTimes <= 0) { + nRepeatTimes = 1; + } + + if (nInterval < 0) { + nInterval = 0; + } + if (nCacheListDepth == 0) { + nCacheListDepth = 1; + } + nInterval = nInterval * 1000; + + if (access(InputFile, 0) != 0) { + ALOGE("%s not exist", InputFile); + exit(0); + } + + // clear output + if (SaveResultPath) { + RUN_COMMAND("rm -rf %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s", SaveResultPath); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_BODY_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_VEHICLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_CYCLE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_FACE_PATH); + RUN_COMMAND("mkdir -p %s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_PLATE_PATH); + } + + if (ReWritePath) { + RUN_COMMAND("rm -rf %s", ReWritePath); + RUN_COMMAND("mkdir -p %s", ReWritePath); + } + + if (InputResolution) { + AX_CHAR *temp_p = strstr(InputResolution, "x"); + + if (!temp_p || strlen(temp_p) <= 1) { + ShowUsage(); + exit(0); + } + + nWidth = atoi(InputResolution); + nStride = nWidth; + nHeight = atoi(temp_p + 1); + } + + nRet = AX_SYS_Init(); + if (0 != nRet) { + ALOGE("AX_SYS_Init() fail, ret = 0x%x", nRet); + exit(0); + } + + nRet = AX_IVPS_Init(); + + ALOGI("AX_IVPS_Init nRet = 0x%x", nRet); + + AX_VENC_MOD_ATTR_T stVencModAttr; + memset(&stVencModAttr, 0x00, sizeof(stVencModAttr)); + stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; + stVencModAttr.stModThdAttr.u32TotalThreadNum = 1; + stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; + nRet = AX_VENC_Init(&stVencModAttr); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VENC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VENC_Init nRet = 0x%x", nRet); + + nRet = AX_VDEC_Init(NULL); + if (AX_SUCCESS != nRet) { + ALOGE("AX_VDEC_Init FAILED! ret:0x%x\n", nRet); + goto EXIT0; + } + + ALOGI("AX_VDEC_Init nRet = 0x%x", nRet); + + // init engine + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); +#if defined(CHIP_AX650) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else if (nNpuType >= 1 && nNpuType <= 4) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_STD; + } + else if (nNpuType >= 5 && nNpuType <= 7) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_BIG_LITTLE; + } +#elif defined(CHIP_AX620E) + if (nNpuType == 0) { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; + } + else { + npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_ENABLE; + } +#else + // #error "NO CHIP SELECTED." +#endif + + nRet = AX_ENGINE_Init(&npu_attr); + if (0 != nRet) { + ALOGE("AX_NPU_SDK_EX_Init_with_attr() fail, ret = 0x%x", nRet); + goto EXIT0; + } + + nOneSize = nWidth * nHeight * 3 / 2; + nFrameSize = nStride * nHeight * 3 / 2; + + InputFileHandle = LoadFile(InputFile, &nFileSize); + + if (bJpgFile) { + InputFileFrameCnt = 1; + } + else { + if (!InputFileHandle + || (nFileSize % nOneSize) != 0) { + ALOGE("%s file is not %dx%d", InputFile, nWidth, nHeight); + goto EXIT1; + } + + InputFileFrameCnt = nFileSize / nOneSize; + } + + + if (nWidth%2 == 1 + || nHeight%2 == 1) { + ALOGE("wxh(%dx%d) should be even", nWidth, nHeight); + goto EXIT1; + } + + // YUVDataWrVir = (AX_VOID *)malloc(nFileSize); + nRet = AX_SYS_MemAlloc(&YUVDataWrPhy, &YUVDataWrVir, nFrameSize, 128, (const AX_S8 *)"SKEL_TEST"); + + if (!YUVDataWrVir) { + ALOGE("malloc fail nRet=0x%x", nRet); + goto EXIT1; + } + + if (SaveResultPath) { + AX_CHAR szPath[512] = {0}; + snprintf(szPath, 511, "%s/%s", SaveResultPath, SKEL_SAMPLE_OUTPUT_LOG_FILE); + fpResultFile = fopen(szPath, "a+"); + + if (!fpResultFile) { + ALOGE("%s file not exist", szPath); + goto EXIT1; + } + } + + nStartTime = get_tick_count(); + + AX_SKEL_INIT_PARAM_T stInitParam = {0}; + + if (ModelsPath) { + stInitParam.pStrModelDeploymentPath = ModelsPath; + } + else { + stInitParam.pStrModelDeploymentPath = "/opt/etc/skelModels"; + } + + nRet = AX_SKEL_Init(&stInitParam); + + nInitElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL init fail, ret = 0x%x", nRet); + + goto EXIT1; + } + + // get version + { + const AX_SKEL_VERSION_INFO_T *pstVersion = NULL; + + nRet = AX_SKEL_GetVersion(&pstVersion); + + if (0 != nRet) { + ALOGI("SKEL get version fail, ret = 0x%x", nRet); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + goto EXIT1; + } + + ALOGI("SKEL version: %s", pstVersion->pstrVersion); + + if (pstVersion) { + AX_SKEL_Release((AX_VOID *)pstVersion); + } + } + + // get capability + { + const AX_SKEL_CAPABILITY_T *pstCapability = NULL; + + nRet = AX_SKEL_GetCapability(&pstCapability); + + if (0 != nRet) { + ALOGI("SKEL get capability fail, ret = 0x%x", nRet); + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + goto EXIT1; + } + + for (AX_U32 i = 0; i < pstCapability->nPPLConfigSize; i++) { + ALOGI("SKEL capability[%d]: (ePPL: %d, PPLConfigKey: %s)", i, pstCapability->pstPPLConfig[i].ePPL, + pstCapability->pstPPLConfig[i].pstrPPLConfigKey); + } + + if (pstCapability) { + AX_SKEL_Release((AX_VOID *)pstCapability); + } + } + + AX_SKEL_HANDLE_PARAM_T stHandleParam = {0}; + + stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; + stHandleParam.nFrameDepth = nFrameDepth; + stHandleParam.nFrameCacheDepth = nCacheListDepth; + stHandleParam.nIoDepth = 0; + stHandleParam.nWidth = nWidth; + stHandleParam.nHeight = nHeight; + + // config settings (if need) + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[16] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; + + // venc_attr_config + // default will be system definition: w*h*3/8 + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVencAttrConfigThreshold = {0}; + if (nJencBufSize > 0) { + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; + stVencAttrConfigThreshold.fValue = (AX_F32) nJencBufSize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stTrackEnableThreshold = {0}; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; + + // detect only (disable track + disable push) + if (nDetectType == 0) { + // track_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + stTrackEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + // detect + track (disable push) + else if (nDetectType == 1) { + // push_disable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + } + + if (itemIndex > 0) { + stConfig.nSize = itemIndex; + stHandleParam.stConfig = stConfig; + } + + // default NPU + if (nNpuType == 0 || nNpuType == 1) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_NPU_DEFAULT; + } + // STD-NPU + else if (nNpuType == 2) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_1; + } + else if (nNpuType == 3) { + stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_2; + } + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_Create(&stHandleParam, &pHandle); + + nCreateElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Create Handle fail, ret = 0x%x", nRet); + + goto EXIT2; + } + + if (fConfidence == 0) { + fConfidence = 0.5; + } + + // set config + { + AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_ITEM_T stItems[64] = {0}; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; +#if 0 + // body_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyMaxTargetCount = {0}; + stBodyMaxTargetCount.fValue = (AX_F32)nHumantracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleMaxTargetCount = {0}; + stVehicleMaxTargetCount.fValue = (AX_F32)nVehicletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_max_target_count + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_max_target_count"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleMaxTargetCount = {0}; + stCycleMaxTargetCount.fValue = (AX_F32)nCycletracksize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMaxTargetCount; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif +#if 0 + // body_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stBodyConfidence = {0}; + stBodyConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stFaceConfidence = {0}; + stFaceConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVehicleConfidence = {0}; + stVehicleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCycleConfidence = {0}; + stCycleConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_confidence + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_confidence"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPlateConfidence = {0}; + stPlateConfidence.fValue = fConfidence; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateConfidence; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // crop_encoder_qpLevel + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"crop_encoder_qpLevel"; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stCropEncoderQpLevelThreshold = {0}; + stCropEncoderQpLevelThreshold.fValue = 90; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCropEncoderQpLevelThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + +#if 0 + // body_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stBodyMinSize = {0}; + stBodyMinSize.nWidth = 0; + stBodyMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // face_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stFaceMinSize = {0}; + stFaceMinSize.nWidth = 0; + stFaceMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // vehicle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stVehicleMinSize = {0}; + stVehicleMinSize.nWidth = 0; + stVehicleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // cycle_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stCycleMinSize = {0}; + stCycleMinSize.nWidth = 0; + stCycleMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; + + // plate_min_size + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_min_size"; + AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T stPlateMinSize = {0}; + stPlateMinSize.nWidth = 0; + stPlateMinSize.nHeight = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateMinSize; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T); + itemIndex++; +#endif + + // detect_roi_polygon + AX_SKEL_ROI_POLYGON_CONFIG_T stDetectRoi = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; + stDetectRoi.bEnable = AX_FALSE; + stDetectRoi.nPointNum = 4; + AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; + stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); + itemIndex++; + + // push_strategy + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; + AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; + stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; + stPushStrategy.nIntervalTimes = 2000; + stPushStrategy.nPushCounts = 1; + stPushStrategy.bPushSameFrame = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_STRATEGY_T); + itemIndex++; + + //4 crop_encoder +#if 0 + // body_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stBodyCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_crop_encoder"; + stBodyCropEncoderThreshold.fScaleLeft = 0; + stBodyCropEncoderThreshold.fScaleRight = 0; + stBodyCropEncoderThreshold.fScaleTop = 0; + stBodyCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // vehicle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stVehicleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"vehicle_crop_encoder"; + stVehicleCropEncoderThreshold.fScaleLeft = 0; + stVehicleCropEncoderThreshold.fScaleRight = 0; + stVehicleCropEncoderThreshold.fScaleTop = 0; + stVehicleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // cycle_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stCycleCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"cycle_crop_encoder"; + stCycleCropEncoderThreshold.fScaleLeft = 0; + stCycleCropEncoderThreshold.fScaleRight = 0; + stCycleCropEncoderThreshold.fScaleTop = 0; + stCycleCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // face_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stFaceCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"face_crop_encoder"; + stFaceCropEncoderThreshold.fScaleLeft = 0; + stFaceCropEncoderThreshold.fScaleRight = 0; + stFaceCropEncoderThreshold.fScaleTop = 0; + stFaceCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; + + // plate_crop_encoder + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stPlateCropEncoderThreshold = {0}; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"plate_crop_encoder"; + stPlateCropEncoderThreshold.fScaleLeft = 0; + stPlateCropEncoderThreshold.fScaleRight = 0; + stPlateCropEncoderThreshold.fScaleTop = 0; + stPlateCropEncoderThreshold.fScaleBottom = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateCropEncoderThreshold; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T); + itemIndex++; +#endif + // push_panorama + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; + AX_SKEL_PUSH_PANORAMA_CONFIG_T stPushPanoramaConfig = {0}; + stPushPanoramaConfig.bEnable = AX_FALSE; + stPushPanoramaConfig.nQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); + itemIndex++; +#if 0 + // push_quality_body + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_body"; + AX_SKEL_ATTR_FILTER_CONFIG_T stBodyAttrFilter = {0}; + stBodyAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stBodyAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_vehicle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_vehicle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stVehicleAttrFilter = {0}; + stVehicleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVehicleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_cycle + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_cycle"; + AX_SKEL_ATTR_FILTER_CONFIG_T stCycleAttrFilter = {0}; + stCycleAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stCycleAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; + + // push_quality_face + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_face"; + AX_SKEL_ATTR_FILTER_CONFIG_T stFaceAttrFilter = {0}; + stFaceAttrFilter.stFaceAttrFilterConfig.nWidth = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.nHeight = 0; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fPitch = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fYaw = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fRoll = 180; + stFaceAttrFilter.stFaceAttrFilterConfig.stPoseblur.fBlur = 1.0; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stFaceAttrFilter; + itemIndex++; + + // push_quality_plate + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_quality_plate"; + AX_SKEL_ATTR_FILTER_CONFIG_T stPlateAttrFilter = {0}; + stPlateAttrFilter.stCommonAttrFilterConfig.fQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPlateAttrFilter; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); + itemIndex++; +#endif +//#if 0 + // track_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stTrackEnable = {0}; + stTrackEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; + + //push_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushEnable = {0}; + stPushEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +//#endif +#if 0 + //target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + //push_target_config + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_target_config"; + AX_SKEL_TARGET_CONFIG_T stTarget = {0}; + stTarget.nSize = 1; + AX_SKEL_TARGET_ITEM_T stItem = {0}; + stItem.pstrObjectCategory = "body"; + stTarget.pstItems = &stItem; + + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTarget; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_TARGET_CONFIG_T); + itemIndex++; + + // push_attr_always + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_attr_always"; + AX_SKEL_COMMON_ENABLE_CONFIG_T stPushAttrAlways = {0}; + stPushAttrAlways.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushAttrAlways; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + itemIndex++; +#endif + //analyzer_attr_config + // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; + // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; + // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; + + // stAnalyze.nSize = 1; + // AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_FACE_ATTRIBUTE; + // stAnalyze.peItems = &analyze_attribute; + + // stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stAnalyze; + // stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ANALYZER_CONFIG_T); + // itemIndex++; + + stConfig.nSize = itemIndex; + + nRet = AX_SKEL_SetConfig(pHandle, &stConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_SetConfig, ret = 0x%x", nRet); + + goto EXIT2; + } + } + + // get config + { + const AX_SKEL_CONFIG_T *pstConfig = NULL; + + nRet = AX_SKEL_GetConfig(pHandle, &pstConfig); + + if (0 != nRet) { + ALOGE("SKEL AX_SKEL_GetConfig, ret = 0x%x", nRet); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + + goto EXIT2; + } + + ParseConfigParam(pstConfig); + + if (pstConfig) { + AX_SKEL_Release((AX_VOID *)pstConfig); + } + } + + ALOGN("Task infomation:"); + ALOGN("\tInput file: %s", InputFile); + ALOGN("\tInput file resolution: %dx%d", nWidth, nHeight); + ALOGN("\tRepeat times: %d", nRepeatTimes); + ALOGN("SKEL Init Elapse:"); + ALOGN("\tAX_SKEL_Init: %lld ms", nInitElasped); + ALOGN("\tAX_SKEL_Create: %lld ms", nCreateElasped); + + OUTPUT_LOG_SAVE("==============================%s process result:==============================", InputFile); + + if (InputFileFrameCnt > 1) { + InputFileFrameCnt = InputFileFrameCnt + BLACK_VIDEO_FRAME_COUNT; + } + + // create frame mgr + FrameMgrCreate(nFrameSize, nFrameDepth + nCacheListDepth); + + for (AX_U32 nRepeat = 0; nRepeat < nRepeatTimes; nRepeat++) { + ALOGN("#####SKEL Process times: %d", nRepeat + 1); + + OUTPUT_LOG_SAVE("#####SKEL Process times: %d", nRepeat + 1); + + STAT_OBJECT_NUM_T tObjectTrackNum = {0}; + STAT_OBJECT_NUM_T tObjectPushNum = {0}; + + for (AX_U32 nFrameCnt = 0; nFrameCnt < InputFileFrameCnt; nFrameCnt++) { + AX_BLK blkId = AX_INVALID_BLOCKID; + + if (InputFileFrameCnt != 1) { + if (FrameSkipCtrl(nSrcFrameRate, nDstFrameRate, nSkelFrameId)) { + nSkelFrameId ++; + + LoadFileToMem(InputFileHandle, 0, NULL, nOneSize); + + if (nInterval > 0) { + usleep(nInterval); + } + continue; + } + else { + if (!FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId)) { + + ALOGE("FrameMgrGet fail"); + + goto EXIT1; + } + + if (nOneSize != nFrameSize) { + LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nOneSize); + } + } + } else { + FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId); + + if (nRet != 0 + || YUVDataPhy == 0 + || YUVDataVir == NULL) { + ALOGE("Load input file fail"); + goto EXIT1; + } + + if (bJpgFile) { + nRet = DecodeJpeg(InputFileHandle, YUVDataPhy, YUVDataVir, nWidth, nHeight); + if (0 != nRet) { + ALOGE("DecodeJpeg failed!"); + goto EXIT3; + } + } + else { + if (nOneSize != nFrameSize) { + nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, (AX_S8 *)"SKEL_TEST"); + + nRet = LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); + + NV12ToStrideNV12(OneYUVDataPhy, + OneYUVDataVir, + nOneSize, + nWidth, + nHeight, + YUVDataPhy, + YUVDataVir, + nFrameSize, + nStride, + nHeight + ); + } + else { + nRet = LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nFrameSize); + } + } + } + stFrame.stFrame.u32Width = nWidth; + stFrame.stFrame.u32Height = nHeight; + stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stFrame.stFrame.u32FrameSize = nFrameSize; + stFrame.stFrame.u32PicStride[0] = nStride; + stFrame.stFrame.u32PicStride[1] = nStride; + stFrame.stFrame.u32PicStride[2] = nStride; + stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; + stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; + stFrame.stFrame.u64PhyAddr[2] = 0; + stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); + stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); + stFrame.stFrame.u64VirAddr[2] = 0; + stFrame.stFrame.u32BlkId[0] = blkId; + stFrame.stFrame.u32BlkId[1] = 0; + stFrame.stFrame.u32BlkId[2] = 0; + ALOGN("*****SKEL Frame(%d) Process", nFrameCnt + 1); + + //OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); + + stFrame.nFrameId = nSkelFrameId ++; + nStartTime = get_tick_count(); + + while (1) { + nRet = AX_SKEL_SendFrame(pHandle, &stFrame, 0); + + if (nRet == 0) { + break; + } + + usleep(1000); + } + + FrameMgrRelease(blkId); + + nProcessElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL Process fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("SKEL Process Elapse:"); + ALOGN("\tAX_SKEL_SendFrame: %lld ms", nProcessElasped); + + nStartTime = get_tick_count(); + + nRet = AX_SKEL_GetResult(pHandle, &pstResult, -1); + + nResultElasped = get_tick_count() - nStartTime; + + if (0 != nRet) { + ALOGE("SKEL get result fail, ret = 0x%x", nRet); + + goto EXIT3; + } + + ALOGN("\tAX_SKEL_GetResult: %lld ms", nResultElasped); + + nResultElaspedTotal += nResultElasped; + + if (nProcessElasped + nResultElasped > nResultElaspedMax) { + nResultElaspedMax = nProcessElasped + nResultElasped; + } + + if (nProcessElasped + nResultElasped < nResultElaspedMin) { + nResultElaspedMin = nProcessElasped + nResultElasped; + } + + ALOGN("SKEL Process Result:"); + + ALOGI("\tFrameId: %lld", pstResult->nFrameId); + ALOGI("\tnOriginal WxH: %dx%d", pstResult->nOriginalWidth, pstResult->nOriginalHeight); + + ALOGN("\tObject Num: %d", pstResult->nObjectSize); + + AX_U32 nSkelSize = 0; + AI_Detection_SkelResult_t Skels[SKEL_SAMPLE_OBJECT_SIZE] = {0}; + for (AX_U32 i = 0; i < pstResult->nObjectSize; i++) { + AX_SKEL_OBJECT_ITEM_T *pstItems = &pstResult->pstObjectItems[i]; + + ALOGI("\t\tFrameId: %lld", pstItems->nFrameId); + ALOGI("\t\tTrackId: %lld, TrackState: %d", pstItems->nTrackId, pstItems->eTrackState); + + ALOGN("\t\tRect[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, + pstItems->stRect.fY, pstItems->stRect.fW, + pstItems->stRect.fH, pstItems->fConfidence); + StatTrackMgr(pstItems, &tObjectTrackNum); + StatPushMgr(pstItems, &tObjectPushNum); + + // get detect box only new or update state + if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW + || pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) + && nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { + Skels[nSkelSize].pstrObjectCategory = pstItems->pstrObjectCategory; + Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; + Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; + Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; + Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; + + OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", + i, pstItems->pstrObjectCategory, + pstItems->stRect.fX, pstItems->stRect.fY, + pstItems->stRect.fW, pstItems->stRect.fH, + pstItems->fConfidence); + + ALOGN("\t\t[%d]Point Set Size: %d", i, pstItems->nPointSetSize); + + // point + Skels[nSkelSize].nPointNum = AX_MIN(DETECT_SKEL_POINT_COUNT, pstItems->nPointSetSize); + for (AX_U32 j = 0; j < Skels[i].nPointNum; j++) { + ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; + Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; + + OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", + j, + pstItems->pstPointSet[j].pstrObjectCategory, + pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].fConfidence); + } + + //face attr + if(nPPL == AX_SKEL_PPL_FACE){ + ALOGN("\t\tFace Attribute:"); + ALOGI("\t\t\tfYaw %f,",pstItems->stFaceAttr.fYaw); + ALOGI("\t\t\tfPitch %f,",pstItems->stFaceAttr.fPitch); + ALOGI("\t\t\tfRoll %f,",pstItems->stFaceAttr.fRoll); + ALOGI("\t\t\tfMask %f,",pstItems->stFaceAttr.fMask); + ALOGI("\t\t\tage %d",pstItems->stFaceAttr.nAge); + ALOGI("\t\t\tgender %d",pstItems->stFaceAttr.nGender); + } + nSkelSize ++; + } + + if (AX_SKEL_TRACK_STATUS_SELECT == pstItems->eTrackState) { + ATTRINFO_T AttrInfo; + memset(&AttrInfo, 0x00, sizeof(AttrInfo)); + AttrParser(pstItems, &AttrInfo); + + if (pstItems->bCropFrame + && pstItems->stCropFrame.pFrameData + && 0 < pstItems->stCropFrame.nFrameDataSize + && SaveResultPath) { + // save attribute + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, + AttrInfo.tFaceInfo.nAge, + AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[FACE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[PLATE ATTR] %s", arrDat); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, + AttrInfo.tPlateInfo.szNum, + AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + + OUTPUT_LOG_SAVE("\t\t[VEHICLE ATTR] %s", arrDat); + } + else { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, + pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write crop jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + + } + + // panora frame + if (pstItems->bPanoraFrame + && pstItems->stPanoraFrame.pFrameData + && 0 < pstItems->stPanoraFrame.nFrameDataSize + && SaveResultPath) { + AX_CHAR arrDat[256] = {0}; + AX_CHAR strFile[512] = {0}; + if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + else { + sprintf(arrDat, "frame%lld_panora_%s[%lld]_Score[%f]_%dx%d.jpg", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + } + + sprintf(strFile, "%s/%s/%s.jpg", + SaveResultPath, + pstItems->pstrObjectCategory, + arrDat); + + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("Write panora jpg to file: %s", strFile); + fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, pstItems->stPanoraFrame.nFrameDataSize, fp_w); + fclose(fp_w); + } + } + } + + // feature + ALOGI("\t\tFeature Size: %d", pstItems->nFeatureSize); + if (pstItems->pstFeatureItem + && 0 < pstItems->nFeatureSize + && SaveResultPath) { + AX_CHAR strFile[256] = {0}; + sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", + SaveResultPath, pstItems->pstrObjectCategory, + pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, i); + FILE *fp_w = fopen(strFile, "wb"); + + if (fp_w) { + ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", + strFile, + pstItems->pstFeatureItem[0].nValueSize); + fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, pstItems->pstFeatureItem[0].nValueSize, fp_w); + fclose(fp_w); + } + } + + // meta + ALOGI("\t\tMeta Size: %d", pstItems->nMetaInfoSize); + if (pstItems->pstMetaInfo + && 0 < pstItems->nMetaInfoSize) { + for (AX_U32 i = 0; i < pstItems->nMetaInfoSize; i++) { + ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, pstItems->pstMetaInfo[i].pstrValue); + } + } + + // binds + ALOGI("\t\tBind Size: %d", pstItems->nObjectBindSize); + if (pstItems->pstObjectBind + && 0 < pstItems->nObjectBindSize) { + for (AX_U32 i = 0; i < pstItems->nObjectBindSize; i++) { + ALOGI("\t\t[%s] [TrackId] %lld bind to ObjectBind[%d]: %s [TrackId] %lld", + pstItems->pstrObjectCategory, pstItems->nTrackId, i, + pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); + } + } + } + + ALOGI("\tnCacheListSize: %d", pstResult->nCacheListSize); + if (pstResult->pstCacheList) { + for (AX_U32 i = 0; i < pstResult->nCacheListSize; i++) { + ALOGI("\t\tCacheList[%d] FrameId: %lld", i, pstResult->pstCacheList[i].nFrameId); + } + } + + if (ReWritePath && pstResult->nObjectSize > 0) { + memcpy(YUVDataWrVir, YUVDataVir, nFrameSize); + + YUV_IMAGE_T YUVImage = {0}; + YUVImage.pImage = YUVDataWrVir; + YUVImage.nWidth = nWidth; + YUVImage.nHeight = nHeight; + YUVImage.stride = nStride; + YUVImage.nSize = nFrameSize; + YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; + + for (size_t i = 0; i < nSkelSize; i++) { + // draw rect + AX_CHAR ObjectCategory = Skels[i].pstrObjectCategory[0]; + AX_S16 x0 = Skels[i].tBox.fX; + AX_S16 y0 = Skels[i].tBox.fY; + AX_U16 w = Skels[i].tBox.fW; + AX_U16 h = Skels[i].tBox.fH; + YUV_COLOR RectColor = YUV_WHITE; + + switch(ObjectCategory){ + case 'b': //body + RectColor = YUV_WHITE; + break; + case 'v': //vehicle + RectColor = YUV_PURPLE; + break; + case 'c': //cycle + RectColor = YUV_GREEN; + break; + case 'f': //face + RectColor = YUV_YELLOW; + break; + case 'p': //plate + RectColor = YUV_RED; + break; + default : + RectColor = YUV_WHITE; + break; + } + + DrawRect(&YUVImage, x0, y0, w, h, RectColor); + + if (Skels[i].nPointNum > 0) { + // draw point + for (size_t j = 0; j < Skels[i].nPointNum; j++) { + x0 = Skels[i].tPoint[j].fX; + y0 = Skels[i].tPoint[j].fY; + + DrawPoint(&YUVImage, x0, y0, 4, x0 * (4 - 1), y0 * (4 - 1), YUV_DARK_GREEN); + } + } + } + + AX_CHAR arrWrFile[512] = {0}; + sprintf(arrWrFile, "%s_result_%d_%d.jpg", InputFile, nFrameCnt + 1, nRepeat + 1); + + AX_CHAR *fileName = NULL; + AX_U32 fileNameLen = strlen(arrWrFile); + AX_CHAR fullFileName[256] = {0}; + AX_S32 i = fileNameLen - 1; + + if (fileNameLen > 0) { + for (; i >= 0; i --) { + if (arrWrFile[i] == '/') { + break; + } + } + } + + if (i < 0) { + i = 0; + } + else if (i < fileNameLen - 1) { + i ++; + } + + fileName = (AX_CHAR *)&arrWrFile[i]; + + sprintf(fullFileName, "%s/%s", ReWritePath, fileName); + + EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, nFrameSize); + } + + if (pstResult) { + AX_SKEL_Release((AX_VOID *)pstResult); + } + + OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process End (Elasped: %lld ms)", nFrameCnt + 1, nProcessElasped); + + if (nInterval > 0) { + usleep(nInterval); + } + } + + ALOGN("SKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + OUTPUT_LOG_SAVE("\nSKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectTrackNum.nBodyNum, + tObjectTrackNum.nVehicleNum, + tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, + tObjectTrackNum.nPlateNum); + + ALOGN("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + OUTPUT_LOG_SAVE("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]\n", + tObjectPushNum.nBodyNum, + tObjectPushNum.nVehicleNum, + tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); + + if (InputFileHandle) { + ReLoadFile(InputFileHandle); + } + } + + // if(AX_SUCCESS != nRet){ + // ALOGE("AX_POOL_DestroyPool failed! Error Code:0x%X\n", nRet); + // } + + ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + + OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", + nRepeatTimes, + nResultElaspedMin, + (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + nResultElaspedMax); + +EXIT3: + if (pHandle) { + AX_SKEL_Destroy(pHandle); + } + +EXIT2: + AX_SKEL_DeInit(); + +EXIT1: + AX_ENGINE_Deinit(); + +EXIT0: + // if (YUVDataVir) { + // if (YUVDataPhy == 0) { + // free(YUVDataVir); + // } else { + // AX_SYS_MemFree(YUVDataPhy, YUVDataVir); + // } + // } + if (YUVDataWrVir) { + if (YUVDataWrPhy == 0) { + free(YUVDataWrVir); + } else { + AX_SYS_MemFree(YUVDataWrPhy, YUVDataWrVir); + } + } + + if (OneYUVDataVir) { + if (OneYUVDataPhy == 0) { + free(OneYUVDataVir); + } else { + AX_SYS_MemFree(OneYUVDataPhy, OneYUVDataVir); + } + } + + FrameMgrDestroy(); + + if (fpResultFile) { + fclose(fpResultFile); + } + + AX_VDEC_Deinit(); + + AX_VENC_Deinit(); + + AX_IVPS_Deinit(); + + AX_SYS_Deinit(); + + return (0 != nRet) ? -1 : 0; +} diff --git a/projects/llm_framework/main_skel/src/runner/skel_log.h b/projects/llm_framework/main_skel/src/runner/skel_log.h new file mode 100644 index 00000000..ffb495e6 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/skel_log.h @@ -0,0 +1,61 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _SAMPLE_SKEL_LOG_H_ +#define _SAMPLE_SKEL_LOG_H_ + +#include + +typedef enum { + SKEL_LOG_MIN = -1, + SKEL_LOG_EMERGENCY = 0, + SKEL_LOG_ALERT = 1, + SKEL_LOG_CRITICAL = 2, + SKEL_LOG_ERROR = 3, + SKEL_LOG_WARN = 4, + SKEL_LOG_NOTICE = 5, + SKEL_LOG_INFO = 6, + SKEL_LOG_DEBUG = 7, + SKEL_LOG_MAX +} SKEL_LOG_LEVEL_E; + +static SKEL_LOG_LEVEL_E log_level = SKEL_LOG_NOTICE; + +#if 1 +#define MACRO_BLACK "\033[1;30;30m" +#define MACRO_RED "\033[1;30;31m" +#define MACRO_GREEN "\033[1;30;32m" +#define MACRO_YELLOW "\033[1;30;33m" +#define MACRO_BLUE "\033[1;30;34m" +#define MACRO_PURPLE "\033[1;30;35m" +#define MACRO_WHITE "\033[1;30;37m" +#define MACRO_END "\033[0m" +#else +#define MACRO_BLACK +#define MACRO_RED +#define MACRO_GREEN +#define MACRO_YELLOW +#define MACRO_BLUE +#define MACRO_PURPLE +#define MACRO_WHITE +#define MACRO_END +#endif + +#define ALOGE(fmt, ...) printf(MACRO_RED "[E][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGW(fmt, ...) if (log_level >= SKEL_LOG_WARN) \ + printf(MACRO_YELLOW "[W][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGI(fmt, ...) if (log_level >= SKEL_LOG_INFO) \ + printf(MACRO_GREEN "[I][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGD(fmt, ...) if (log_level >= SKEL_LOG_DEBUG) \ + printf(MACRO_WHITE "[D][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define ALOGN(fmt, ...) if (log_level >= SKEL_LOG_NOTICE) \ + printf(MACRO_PURPLE "[N][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) + +#endif /* _SAMPLE_SKEL_TRACE_H_ */ diff --git a/projects/llm_framework/main_skel/src/runner/statMgr.cpp b/projects/llm_framework/main_skel/src/runner/statMgr.cpp new file mode 100644 index 00000000..c3a71010 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/statMgr.cpp @@ -0,0 +1,65 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#include +#include "statMgr.h" +#include + +AX_VOID StatTrackMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum) { + if (!pstObjectItems || !pObjectNum) { + return; + } + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (pstObjectItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW) { + if (strObjectCategory == "body") { + pObjectNum->nBodyNum ++; + } + else if (strObjectCategory == "vehicle") { + pObjectNum->nVehicleNum ++; + } + else if (strObjectCategory == "cycle") { + pObjectNum->nCycleNum ++; + } + else if (strObjectCategory == "face") { + pObjectNum->nFaceNum ++; + } + else if (strObjectCategory == "plate") { + pObjectNum->nPlateNum ++; + } + } +} + +AX_VOID StatPushMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum) { + if (!pstObjectItems || !pObjectNum) { + return; + } + + std::string strObjectCategory = pstObjectItems->pstrObjectCategory; + + if (pstObjectItems->eTrackState == AX_SKEL_TRACK_STATUS_SELECT) { + if (strObjectCategory == "body") { + pObjectNum->nBodyNum ++; + } + else if (strObjectCategory == "vehicle") { + pObjectNum->nVehicleNum ++; + } + else if (strObjectCategory == "cycle") { + pObjectNum->nCycleNum ++; + } + else if (strObjectCategory == "face") { + pObjectNum->nFaceNum ++; + } + else if (strObjectCategory == "plate") { + pObjectNum->nPlateNum ++; + } + } +} diff --git a/projects/llm_framework/main_skel/src/runner/statMgr.h b/projects/llm_framework/main_skel/src/runner/statMgr.h new file mode 100644 index 00000000..8cad1f60 --- /dev/null +++ b/projects/llm_framework/main_skel/src/runner/statMgr.h @@ -0,0 +1,36 @@ +/************************************************************************************************** + * + * Copyright (c) 2019-2024 Axera Semiconductor Co., Ltd. All Rights Reserved. + * + * This source file is the property of Axera Semiconductor Co., Ltd. and + * may not be copied or distributed in any isomorphic form without the prior + * written consent of Axera Semiconductor Co., Ltd. + * + **************************************************************************************************/ + +#ifndef _STAT_MGR_H_ +#define _STAT_MGR_H_ +#include "ax_global_type.h" +#include "ax_sys_api.h" +#include "ax_skel_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _STAT_OBJECT_NUM_T { + AX_U32 nBodyNum; + AX_U32 nVehicleNum; + AX_U32 nCycleNum; + AX_U32 nFaceNum; + AX_U32 nPlateNum; +} STAT_OBJECT_NUM_T; + +extern AX_VOID StatTrackMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum); +extern AX_VOID StatPushMgr(const AX_SKEL_OBJECT_ITEM_T *pstObjectItems, STAT_OBJECT_NUM_T *pObjectNum); + +#ifdef __cplusplus +} +#endif + +#endif From f649b9244e096e215f0064e1038c25d55f36a821 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 29 Nov 2024 16:56:09 +0800 Subject: [PATCH 28/44] [update] add llm_camera! --- SDK | 2 +- .../StackFlow/stackflow/StackFlowUtil.cpp | 185 +----- ext_components/StackFlow/stackflow/pzmq.hpp | 3 + projects/llm_framework/.gitignore | 1 + projects/llm_framework/config_defaults.mk | 2 +- projects/llm_framework/main/src/main.cpp | 2 +- projects/llm_framework/main_asr/src/main.cpp | 26 +- projects/llm_framework/main_camera/Kconfig | 0 projects/llm_framework/main_camera/SConstruct | 60 ++ .../llm_framework/main_camera/camera.json | 18 + .../llm_framework/main_camera/src/camera.c | 589 ++++++++++++++++++ .../llm_framework/main_camera/src/camera.h | 87 +++ .../llm_framework/main_camera/src/main.cpp | 288 +++++++++ .../llm_framework/main_camera_test/Kconfig | 0 .../llm_framework/main_camera_test/SConstruct | 68 ++ .../main_camera_test/camera.json | 18 + .../main_camera_test/src/main.cpp | 59 ++ projects/llm_framework/main_kws/src/main.cpp | 22 +- projects/llm_framework/main_llm/src/main.cpp | 2 +- .../llm_framework/main_melotts/src/main.cpp | 2 +- .../llm_framework/main_sys/src/event_loop.cpp | 2 + projects/llm_framework/main_tts/src/main.cpp | 2 +- projects/llm_framework/main_vlm/src/main.cpp | 2 +- projects/llm_framework/main_yolo/src/main.cpp | 86 ++- 24 files changed, 1324 insertions(+), 202 deletions(-) create mode 100644 projects/llm_framework/main_camera/Kconfig create mode 100644 projects/llm_framework/main_camera/SConstruct create mode 100644 projects/llm_framework/main_camera/camera.json create mode 100644 projects/llm_framework/main_camera/src/camera.c create mode 100644 projects/llm_framework/main_camera/src/camera.h create mode 100644 projects/llm_framework/main_camera/src/main.cpp create mode 100644 projects/llm_framework/main_camera_test/Kconfig create mode 100644 projects/llm_framework/main_camera_test/SConstruct create mode 100644 projects/llm_framework/main_camera_test/camera.json create mode 100644 projects/llm_framework/main_camera_test/src/main.cpp diff --git a/SDK b/SDK index 23712b73..e6f2432c 160000 --- a/SDK +++ b/SDK @@ -1 +1 @@ -Subproject commit 23712b73c833f5109f1a3a64e960ed7a244c4315 +Subproject commit e6f2432c564324360b762d738698cc170b95f68d diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index b0bc14c0..756889ec 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -106,33 +106,19 @@ std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &u { return unit_name + "." + std::to_string(work_id_num); } - +// clang-format off std::string StackFlows::sample_escapeString(const std::string &input) { std::string escaped; for (char c : input) { switch (c) { - case '\n': - escaped += "\\n"; - break; - case '\t': - escaped += "\\t"; - break; - case '\\': - escaped += "\\\\"; - break; - case '\"': - escaped += "\\\""; - break; - case '\r': - escaped += "\\r"; - break; - case '\b': - escaped += "\\b"; - break; - default: - escaped += c; - break; + case '\n':escaped += "\\n" ;break; + case '\t':escaped += "\\t" ;break; + case '\\':escaped += "\\\\";break; + case '\"':escaped += "\\\"";break; + case '\r':escaped += "\\r" ;break; + case '\b':escaped += "\\b" ;break; + default :escaped += c ;break; } } return escaped; @@ -144,33 +130,13 @@ std::string StackFlows::sample_unescapeString(const std::string &input) for (size_t i = 0; i < input.length(); ++i) { if (input[i] == '\\' && i + 1 < input.length()) { switch (input[i + 1]) { - case 'n': - unescaped += '\n'; - ++i; - break; - case 't': - unescaped += '\t'; - ++i; - break; - case '\\': - unescaped += '\\'; - ++i; - break; - case '\"': - unescaped += '\"'; - ++i; - break; - case 'r': - unescaped += '\r'; - ++i; - break; - case 'b': - unescaped += '\b'; - ++i; - break; - default: - unescaped += input[i]; - break; + case 'n' :unescaped += '\n';++i;break; + case 't' :unescaped += '\t';++i;break; + case '\\':unescaped += '\\';++i;break; + case '\"':unescaped += '\"';++i;break; + case 'r' :unescaped += '\r';++i;break; + case 'b' :unescaped += '\b';++i;break; + default :unescaped += input[i];break; } } else { unescaped += input[i]; @@ -178,7 +144,7 @@ std::string StackFlows::sample_unescapeString(const std::string &input) } return unescaped; } - +// clang-format on bool StackFlows::decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff) { @@ -200,125 +166,23 @@ bool StackFlows::decode_stream(const std::string &in, std::string &out, #define BASE64_DECODE_OUT_SIZE(s) (((s)) / 4 * 3) #include /* BASE 64 encode table */ -static const char base64en[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', -}; +static const char base64en[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; #define BASE64_PAD '=' #define BASE64DE_FIRST '+' #define BASE64DE_LAST 'z' /* ASCII order for BASE 64 decode, -1 in unused character */ static const signed char base64de[] = { - /* '+', ',', '-', '.', '/', '0', '1', '2', */ - 62, - -1, - -1, - -1, - 63, - 52, - 53, - 54, - - /* '3', '4', '5', '6', '7', '8', '9', ':', */ - 55, - 56, - 57, - 58, - 59, - 60, - 61, - -1, - - /* ';', '<', '=', '>', '?', '@', 'A', 'B', */ - -1, - -1, - -1, - -1, - -1, - -1, - 0, - 1, - - /* 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', */ - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - - /* 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', */ - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - - /* 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', */ - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - - /* '[', '\', ']', '^', '_', '`', 'a', 'b', */ - -1, - -1, - -1, - -1, - -1, - -1, - 26, - 27, - - /* 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', */ - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - - /* 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', */ - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - - /* 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', */ - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, + 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, }; static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) { unsigned int i = 0, j = 0; - for (; i < inlen; i++) { int s = i % 3; - switch (s) { case 0: out[j++] = base64en[(in[i] >> 2) & 0x3F]; @@ -331,10 +195,8 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) out[j++] = base64en[in[i] & 0x3F]; } } - /* move back */ i -= 1; - /* check the last and add padding */ if ((i % 3) == 0) { out[j++] = base64en[(in[i] & 0x3) << 4]; @@ -344,35 +206,28 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) out[j++] = base64en[(in[i] & 0xF) << 2]; out[j++] = BASE64_PAD; } - return j; } static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) { unsigned int i = 0, j = 0; - for (; i < inlen; i++) { int c; int s = i % 4; - if (in[i] == '=') return j; - if (in[i] < BASE64DE_FIRST || in[i] > BASE64DE_LAST || (c = base64de[in[i] - BASE64DE_FIRST]) == -1) return -1; - switch (s) { case 0: out[j] = ((unsigned int)c << 2) & 0xFF; continue; case 1: out[j++] += ((unsigned int)c >> 4) & 0x3; - /* if not last char with padding */ if (i < (inlen - 3) || in[inlen - 2] != '=') out[j] = ((unsigned int)c & 0xF) << 4; continue; case 2: out[j++] += ((unsigned int)c >> 2) & 0xF; - /* if not last char with padding */ if (i < (inlen - 2) || in[inlen - 1] != '=') out[j] = ((unsigned int)c & 0x3) << 6; continue; diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index 19421093..1a7baf90 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -59,6 +59,9 @@ class pzmq { pzmq(const std::string &url, int mode, const msg_callback_fun &raw_call = nullptr) : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) { + if ((url[0] != 'i') && (url[1] != 'p')) { + rpc_url_head_.clear(); + } if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call); } void set_timeout(int ms) diff --git a/projects/llm_framework/.gitignore b/projects/llm_framework/.gitignore index e31df2ba..20085e28 100644 --- a/projects/llm_framework/.gitignore +++ b/projects/llm_framework/.gitignore @@ -8,3 +8,4 @@ static_lib** *.tar.gz m5stack_* debian* +gitignore_* \ No newline at end of file diff --git a/projects/llm_framework/config_defaults.mk b/projects/llm_framework/config_defaults.mk index 7076e548..a61fc047 100644 --- a/projects/llm_framework/config_defaults.mk +++ b/projects/llm_framework/config_defaults.mk @@ -13,7 +13,7 @@ CONFIG_DEVICE_UART_ENABLED=y # CONFIG_LHV_WITH_EVPP=y CONFIG_UTILITIES_ENABLED=y CONFIG_BACKWARD_CPP_ENABLED=y -CONFIG_TOOLCHAIN_FLAGS="-O2" +CONFIG_TOOLCHAIN_FLAGS="-O0" CONFIG_EVENTPP_ENABLED=y CONFIG_LHV_ENABLED=y CONFIG_LHV_WITH_EVPP=y diff --git a/projects/llm_framework/main/src/main.cpp b/projects/llm_framework/main/src/main.cpp index 9004bd60..116a52da 100644 --- a/projects/llm_framework/main/src/main.cpp +++ b/projects/llm_framework/main/src/main.cpp @@ -229,7 +229,7 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 27e3798e..9c4d39e6 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -179,7 +179,7 @@ class llm_task { out_callback_ = out_callback; } - void sys_pcm_on_data(pzmq *_pzmq, const std::string &raw) + void sys_pcm_on_data(const std::string &raw) { static int count = 0; if (count < delay_audio_frame_) { @@ -403,7 +403,7 @@ class llm_asr : public StackFlow { } next_data = &tmp_msg4; } - llm_task_obj->sys_pcm_on_data(nullptr, (*next_data)); + llm_task_obj->sys_pcm_on_data((*next_data)); } void _task_pause(const std::string &work_id, const std::string &data) @@ -435,8 +435,10 @@ class llm_asr : public StackFlow { } llm_task_obj->kws_awake(); if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), - std::placeholders::_1, std::placeholders::_2)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } } @@ -521,9 +523,11 @@ class llm_asr : public StackFlow { for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { - audio_url_ = unit_call("audio", "cap", input); - llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), - std::placeholders::_1, std::placeholders::_2)); + audio_url_ = unit_call("audio", "cap", input); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } else if (input.find("asr") != std::string::npos) { llm_channel->subscriber_work_id( @@ -568,8 +572,10 @@ class llm_asr : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; if (data.find("sys") != std::string::npos) { if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data); - llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), - std::placeholders::_1, std::placeholders::_2)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; llm_task_obj->inputs_.push_back(data); } else if (data.find("kws") != std::string::npos) { @@ -638,7 +644,7 @@ class llm_asr : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("asr.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_camera/Kconfig b/projects/llm_framework/main_camera/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_camera/SConstruct b/projects/llm_framework/main_camera/SConstruct new file mode 100644 index 00000000..28bc9381 --- /dev/null +++ b/projects/llm_framework/main_camera/SConstruct @@ -0,0 +1,60 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +# SRCS = append_srcs_dir(ADir('src')) +SRCS = Glob('src/*.c*') +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +# INCLUDE += [ADir('../include'), ADir('../include/cppzmq'), ADir('../include/libzmq')] +LINK_SEARCH_PATH += [ADir('../static_lib')] + +REQUIREMENTS += ['ax_sys', 'ax_interpreter'] + +INCLUDE += [ADir('../include/opencv4')] +static_file = [] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] +STATIC_LIB += static_file * 4 + +STATIC_FILES += [AFile('camera.json')] + +env['COMPONENTS'].append({'target':'llm_camera', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_camera/camera.json b/projects/llm_framework/main_camera/camera.json new file mode 100644 index 00000000..608b163c --- /dev/null +++ b/projects/llm_framework/main_camera/camera.json @@ -0,0 +1,18 @@ +{ + "mode": "None", + "type": "camera", + "capabilities": [ + "play", + "cap" + ], + "input_type": [ + "rpc.camera.wav.base64", + "rpc.camera.pcm.base64" + ], + "output_type": [ + "camera.pcm.stream" + ], + "cap_param": { + "None": "None" + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/camera.c b/projects/llm_framework/main_camera/src/camera.c new file mode 100644 index 00000000..0150857c --- /dev/null +++ b/projects/llm_framework/main_camera/src/camera.c @@ -0,0 +1,589 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "camera.h" + +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" +#include +#include +#include +#include +#include +#include + +/** + * Name of storage device + */ +static int DevNameSet(struct Camera_t* Camera, const char* pDevName) +{ + int CopyLen = 0; + CopyLen = strlen(pDevName); + if (CopyLen > CONFIG_DEVNAME_LEN - 1) { + SLOGE("Error: device name length over limit: %d", CopyLen); + return -1; + } + memset(Camera->DevName, 0, CONFIG_DEVNAME_LEN); + memcpy(Camera->DevName, pDevName, CopyLen); + return 0; +} + +static int CameraCaptureCallbackSet(struct Camera_t* Camera, vCameraFrameGet pCallback) +{ + if (Camera->State == CAMERA_SATTE_CAP) { + SLOGW("Set capture callback failed"); + return -1; + } + Camera->pCallback = pCallback; + return 0; +} + +static void* CameraCaptureThread(void* pParam) +{ + int Ret = -1; + Camera_t* Camera = (Camera_t*)pParam; + struct v4l2_buffer EnQueueBuf; + struct v4l2_buffer DeQueueBuf; + + SLOGI("Start capture"); + + while (Camera->State == CAMERA_SATTE_CAP) { + memset(&DeQueueBuf, 0, sizeof(struct v4l2_buffer)); + DeQueueBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + DeQueueBuf.memory = V4L2_MEMORY_MMAP; + + memset(&EnQueueBuf, 0, sizeof(struct v4l2_buffer)); + EnQueueBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + EnQueueBuf.memory = V4L2_MEMORY_MMAP; + + // Retrieve data from the queue + Ret = ioctl(Camera->DevFd, VIDIOC_DQBUF, &DeQueueBuf); + if (Ret) { + perror("Dequeue fail"); + break; + } + Camera->pCallback(Camera->pCameraBuffer[DeQueueBuf.index].pStart, Camera->Width, Camera->Height, + Camera->pCameraBuffer[DeQueueBuf.index].Length, Camera->ctx_); + + // Re-queue the data + EnQueueBuf.index = DeQueueBuf.index; + + if (ioctl(Camera->DevFd, VIDIOC_QBUF, &EnQueueBuf)) { + perror("Enqueue fail"); + break; + } + } + + SLOGI("Stop capture"); + + return NULL; +} + +static int CameraCaptureStart(struct Camera_t* Camera) +{ + SLOGI("Start capture thread"); + if (!Camera->pCallback) { + SLOGW("Capture callback not set, start faild"); + return -1; + } + if (Camera->State != CAMERA_SATTE_OPEN) { + SLOGW("Camera not open, start faild"); + return -1; + } + Camera->State = CAMERA_SATTE_CAP; + + pthread_create(&Camera->CaptureThreadId, NULL, CameraCaptureThread, Camera); + + return 0; +} + +static int CameraCaptureStop(struct Camera_t* Camera) +{ + SLOGI("Stop capture thread"); + Camera->State = CAMERA_SATTE_OPEN; + pthread_join(Camera->CaptureThreadId, NULL); + SLOGI("Capture thread stop"); + return 0; +} + +static void CameraSetCtx(struct Camera_t* Camera, void* ctx) +{ + Camera->ctx_ = ctx; +} + +int CameraOpenFrom(Camera_t* Camera) +{ + int Ret = -1; + if (Camera == NULL) return -1; + /* Check whether the camera is already open or in an error state */ + SLOGI("Open camera %s...", Camera->DevName); + if (Camera->State != CAMERA_STATE_CLOSE) { + SLOGE("Error: camera was open or meet error, now state is: %d", Camera->State); + goto ErrorHandle; + } + + /* Open the camera device */ + if ((Camera->DevFd = open(Camera->DevName, O_RDWR, 0)) == -1) { + SLOGE("Open camera %s faild.", Camera->DevName); + goto ErrorHandle; + } + + SLOGI("Open device successful."); + + /* Set the capture source */ + struct v4l2_input Input; + Input.index = 0; + + if (ioctl(Camera->DevFd, VIDIOC_S_INPUT, &Input)) { + perror("VIDIOC_S_INPUT"); + } + + /* Retrieve camera information */ + struct v4l2_capability Cap; + + Ret = ioctl(Camera->DevFd, VIDIOC_QUERYCAP, &Cap); + + if (Ret) { + perror("Get device info error"); + goto ErrorHandle; + } + + SLOGI("Get device info successful."); + + SLOGI("Driver name: %s", Cap.driver); + SLOGI("Card name: %s", Cap.card); + SLOGI("Bus info: %s", Cap.bus_info); + + struct v4l2_format TvFmt; + struct v4l2_fmtdesc FmtDesc; + struct v4l2_streamparm StreamParam; + + memset(&FmtDesc, 0, sizeof(FmtDesc)); + FmtDesc.index = 0; + FmtDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + /* Enumerate supported frame formats */ + SLOGI("Support format:"); + while (ioctl(Camera->DevFd, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { + FmtDesc.index++; + SLOGI("Pixel format = %c%c%c%c", (FmtDesc.pixelformat & 0xFF), ((FmtDesc.pixelformat >> 8) & 0xFF), + ((FmtDesc.pixelformat >> 16) & 0xFF), (FmtDesc.pixelformat >> 24)); + SLOGI("Description = %s ", FmtDesc.description); + } + + /* Determine if the device is a device that can capture video */ + if (!(Cap.capabilities & V4L2_BUF_TYPE_VIDEO_CAPTURE)) { + SLOGE("The Current device is not a video capture device"); + goto ErrorHandle; + } + + /* Determine if video streaming is supported */ + if (!(Cap.capabilities & V4L2_CAP_STREAMING)) { + SLOGE("The Current device does not support streaming i/o"); + goto ErrorHandle; + } + + /* Get video stream information */ + memset(&StreamParam, 0, sizeof(struct v4l2_streamparm)); + StreamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("Before setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set frame rate denominator and numerator */ + StreamParam.parm.capture.timeperframe.denominator = Camera->capture_fps; + StreamParam.parm.capture.timeperframe.numerator = 1; + + if (ioctl(Camera->DevFd, VIDIOC_S_PARM, &StreamParam) == -1) { + perror("Unable to set fps"); + goto ErrorHandle; + } + + /* Get video stream information */ + Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("After setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set automatic exposure */ + struct v4l2_control Ctrl; + Ctrl.id = V4L2_CID_EXPOSURE_ABSOLUTE; + Ret = ioctl(Camera->DevFd, VIDIOC_G_CTRL, &Ctrl); + + if (Ret) { + perror("Set exposure failed"); + // goto ErrorHandle; + } + + /* Set capture format */ + TvFmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + TvFmt.fmt.pix.width = Camera->Width; + TvFmt.fmt.pix.height = Camera->Height; + TvFmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + TvFmt.fmt.pix.field = V4L2_FIELD_INTERLACED; + + if (ioctl(Camera->DevFd, VIDIOC_S_FMT, &TvFmt) < 0) { + perror("Unable set VIDIOC_S_FMT"); + goto ErrorHandle; + } + + if (ioctl(Camera->DevFd, VIDIOC_G_FMT, &TvFmt) < 0) { + perror("Get VIDIOC_S_FMT error"); + goto ErrorHandle; + } + + SLOGI("After setting frame params:"); + SLOGI("Width: %d, height: %d", TvFmt.fmt.pix.width, TvFmt.fmt.pix.height); + SLOGI("Pixel format = %c%c%c%c", (TvFmt.fmt.pix.pixelformat & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 8) & 0xFF), + ((TvFmt.fmt.pix.pixelformat >> 16) & 0xFF), (TvFmt.fmt.pix.pixelformat >> 24)); + + Camera->Width = TvFmt.fmt.pix.width; + Camera->Height = TvFmt.fmt.pix.height; + + /* Request V4L2 driver to allocate memory */ + struct v4l2_requestbuffers ReqBuffer; + memset(&ReqBuffer, 0, sizeof(ReqBuffer)); + ReqBuffer.count = Camera->BufferCnt; + ReqBuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + ReqBuffer.memory = V4L2_MEMORY_MMAP; + + if (-1 == ioctl(Camera->DevFd, VIDIOC_REQBUFS, &ReqBuffer)) { + perror("Fail to ioctl 'VIDIOC_REQBUFS'"); + goto ErrorHandle; + } + + Camera->BufferCnt = ReqBuffer.count; + SLOGI("Buffer count: %d", Camera->BufferCnt); + + Camera->pCameraBuffer = (CAMERA_Buffer_t*)calloc(Camera->BufferCnt, sizeof(CAMERA_Buffer_t)); + if (Camera->pCameraBuffer == NULL) { + SLOGE("Malloc buffer failed"); + goto ErrorHandle; + } + + /* Map kernel buffer to user space */ + struct v4l2_buffer Buf; + for (int i = 0; i < Camera->BufferCnt; i++) { + memset(&Buf, 0, sizeof(Buf)); + Buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Buf.memory = V4L2_MEMORY_MMAP; + Buf.index = i; + + if (-1 == ioctl(Camera->DevFd, VIDIOC_QUERYBUF, &Buf)) { + perror("Fail to ioctl : VIDIOC_QUERYBUF"); + goto ErrorHandle; + } + + Camera->pCameraBuffer[i].Length = Buf.length; + Camera->pCameraBuffer[i].pStart = + mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, Camera->DevFd, Buf.m.offset); + + if (MAP_FAILED == Camera->pCameraBuffer[i].pStart) { + perror("Fail to mmap"); + goto ErrorHandle; + } + + if (ioctl(Camera->DevFd, VIDIOC_QBUF, &Buf)) { + perror("Unable to queue buffer"); + goto ErrorHandle; + } + } + + /* Start capturing */ + enum v4l2_buf_type BufType; + BufType = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (ioctl(Camera->DevFd, VIDIOC_STREAMON, &BufType) < 0) { + perror("VIDIOC_STREAMON error"); + goto ErrorHandle; + } + Camera->State = CAMERA_SATTE_OPEN; + Camera->CameraCaptureCallbackSet = CameraCaptureCallbackSet; + Camera->CameraCaptureStart = CameraCaptureStart; + Camera->CameraCaptureStop = CameraCaptureStop; + Camera->CameraSetCtx = CameraSetCtx; + Camera->is_alloc = 0; + return 0; + +ErrorHandle: + SLOGE("Camera open meet error, now handle it"); + close(Camera->DevFd); + free(Camera->pCameraBuffer); + return -1; +} + +Camera_t* CameraOpen(const char* pDevName, int width, int height, int fps) +{ + int Ret = -1; + Camera_t* Camera = (Camera_t*)malloc(sizeof(Camera_t)); + if (Camera == NULL) return NULL; + memset(Camera, 0, sizeof(Camera_t)); + /* Check whether the camera is already open or in an error state */ + SLOGI("Open camera %s...", pDevName); + if (Camera->State != CAMERA_STATE_CLOSE) { + SLOGE("Error: camera was open or meet error, now state is: %d", Camera->State); + goto ErrorHandle; + } + + /* Open the camera device */ + if ((Camera->DevFd = open(pDevName, O_RDWR, 0)) == -1) { + SLOGE("Open camera %s faild.", pDevName); + goto ErrorHandle; + } + + SLOGI("Open device successful."); + + /* Set the capture source */ + struct v4l2_input Input; + Input.index = 0; + + if (ioctl(Camera->DevFd, VIDIOC_S_INPUT, &Input)) { + perror("VIDIOC_S_INPUT"); + } + + /* Retrieve camera information */ + struct v4l2_capability Cap; + + Ret = ioctl(Camera->DevFd, VIDIOC_QUERYCAP, &Cap); + + if (Ret) { + perror("Get device info error"); + goto ErrorHandle; + } + + SLOGI("Get device info successful."); + + SLOGI("Driver name: %s", Cap.driver); + SLOGI("Card name: %s", Cap.card); + SLOGI("Bus info: %s", Cap.bus_info); + + DevNameSet(Camera, pDevName); + + struct v4l2_format TvFmt; + struct v4l2_fmtdesc FmtDesc; + struct v4l2_streamparm StreamParam; + + memset(&FmtDesc, 0, sizeof(FmtDesc)); + FmtDesc.index = 0; + FmtDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + /* Enumerate supported frame formats */ + SLOGI("Support format:"); + while (ioctl(Camera->DevFd, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { + FmtDesc.index++; + SLOGI("Pixel format = %c%c%c%c", (FmtDesc.pixelformat & 0xFF), ((FmtDesc.pixelformat >> 8) & 0xFF), + ((FmtDesc.pixelformat >> 16) & 0xFF), (FmtDesc.pixelformat >> 24)); + SLOGI("Description = %s ", FmtDesc.description); + } + + /* Determine if the device is a device that can capture video */ + if (!(Cap.capabilities & V4L2_BUF_TYPE_VIDEO_CAPTURE)) { + SLOGE("The Current device is not a video capture device"); + goto ErrorHandle; + } + + /* Determine if video streaming is supported */ + if (!(Cap.capabilities & V4L2_CAP_STREAMING)) { + SLOGE("The Current device does not support streaming i/o"); + goto ErrorHandle; + } + + /* Get video stream information */ + memset(&StreamParam, 0, sizeof(struct v4l2_streamparm)); + StreamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("Before setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set frame rate denominator and numerator */ + StreamParam.parm.capture.timeperframe.denominator = fps; + StreamParam.parm.capture.timeperframe.numerator = 1; + + if (ioctl(Camera->DevFd, VIDIOC_S_PARM, &StreamParam) == -1) { + perror("Unable to set fps"); + goto ErrorHandle; + } + + /* Get video stream information */ + Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + if (Ret) { + perror("Get stream info failed"); + goto ErrorHandle; + } + + SLOGI("After setting stream params:"); + SLOGI("Capability: %u", StreamParam.parm.capture.capability); + SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); + SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); + SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); + SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); + + /* Set automatic exposure */ + struct v4l2_control Ctrl; + Ctrl.id = V4L2_CID_EXPOSURE_ABSOLUTE; + Ret = ioctl(Camera->DevFd, VIDIOC_G_CTRL, &Ctrl); + + if (Ret) { + perror("Set exposure failed"); + // goto ErrorHandle; + } + + /* Set capture format */ + TvFmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + TvFmt.fmt.pix.width = width; + TvFmt.fmt.pix.height = height; + TvFmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + TvFmt.fmt.pix.field = V4L2_FIELD_INTERLACED; + + if (ioctl(Camera->DevFd, VIDIOC_S_FMT, &TvFmt) < 0) { + perror("Unable set VIDIOC_S_FMT"); + goto ErrorHandle; + } + + if (ioctl(Camera->DevFd, VIDIOC_G_FMT, &TvFmt) < 0) { + perror("Get VIDIOC_S_FMT error"); + goto ErrorHandle; + } + + SLOGI("After setting frame params:"); + SLOGI("Width: %d, height: %d", TvFmt.fmt.pix.width, TvFmt.fmt.pix.height); + SLOGI("Pixel format = %c%c%c%c", (TvFmt.fmt.pix.pixelformat & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 8) & 0xFF), + ((TvFmt.fmt.pix.pixelformat >> 16) & 0xFF), (TvFmt.fmt.pix.pixelformat >> 24)); + + Camera->Width = TvFmt.fmt.pix.width; + Camera->Height = TvFmt.fmt.pix.height; + + /* Request V4L2 driver to allocate memory */ + struct v4l2_requestbuffers ReqBuffer; + memset(&ReqBuffer, 0, sizeof(ReqBuffer)); + ReqBuffer.count = CONFIG_CAPTURE_BUF_CNT; + ReqBuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + ReqBuffer.memory = V4L2_MEMORY_MMAP; + + if (-1 == ioctl(Camera->DevFd, VIDIOC_REQBUFS, &ReqBuffer)) { + perror("Fail to ioctl 'VIDIOC_REQBUFS'"); + goto ErrorHandle; + } + + Camera->BufferCnt = ReqBuffer.count; + SLOGI("Buffer count: %d", Camera->BufferCnt); + + Camera->pCameraBuffer = (CAMERA_Buffer_t*)calloc(Camera->BufferCnt, sizeof(CAMERA_Buffer_t)); + if (Camera->pCameraBuffer == NULL) { + SLOGE("Malloc buffer failed"); + goto ErrorHandle; + } + + /* Map kernel buffer to user space */ + struct v4l2_buffer Buf; + for (int i = 0; i < Camera->BufferCnt; i++) { + memset(&Buf, 0, sizeof(Buf)); + Buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + Buf.memory = V4L2_MEMORY_MMAP; + Buf.index = i; + + if (-1 == ioctl(Camera->DevFd, VIDIOC_QUERYBUF, &Buf)) { + perror("Fail to ioctl : VIDIOC_QUERYBUF"); + goto ErrorHandle; + } + + Camera->pCameraBuffer[i].Length = Buf.length; + Camera->pCameraBuffer[i].pStart = + mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, Camera->DevFd, Buf.m.offset); + + if (MAP_FAILED == Camera->pCameraBuffer[i].pStart) { + perror("Fail to mmap"); + goto ErrorHandle; + } + + if (ioctl(Camera->DevFd, VIDIOC_QBUF, &Buf)) { + perror("Unable to queue buffer"); + goto ErrorHandle; + ; + } + } + + /* Start capturing */ + enum v4l2_buf_type BufType; + BufType = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (ioctl(Camera->DevFd, VIDIOC_STREAMON, &BufType) < 0) { + perror("VIDIOC_STREAMON error"); + goto ErrorHandle; + } + Camera->State = CAMERA_SATTE_OPEN; + Camera->CameraCaptureCallbackSet = CameraCaptureCallbackSet; + Camera->CameraCaptureStart = CameraCaptureStart; + Camera->CameraCaptureStop = CameraCaptureStop; + Camera->is_alloc = 1; + return Camera; + +ErrorHandle: + SLOGE("Camera open meet error, now handle it"); + close(Camera->DevFd); + free(Camera->pCameraBuffer); + free(Camera); + return NULL; +} + +int CameraClose(Camera_t* Camera) +{ + if (Camera == NULL) return -1; + if (Camera->State == CAMERA_SATTE_OPEN || Camera->State == CAMERA_SATTE_CAP) { + SLOGI("Close camera %s", Camera->DevName); + } else { + SLOGW("Skip close camera progress"); + return 0; + } + + for (int i = 0; i < CONFIG_CAPTURE_BUF_CNT; i++) { + munmap(Camera->pCameraBuffer[i].pStart, Camera->pCameraBuffer[i].Length); + Camera->pCameraBuffer[i].Length = 0; + } + + free(Camera->pCameraBuffer); + Camera->pCameraBuffer = NULL; + + int Off = 1; + + if (ioctl(Camera->DevFd, VIDIOC_STREAMOFF, &Off)) { + perror("Stop camera fail"); + return -1; + } + + close(Camera->DevFd); + if (Camera->is_alloc) free(Camera); + + SLOGI("Camera closed"); + + return 0; +} \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/camera.h b/projects/llm_framework/main_camera/src/camera.h new file mode 100644 index 00000000..97066131 --- /dev/null +++ b/projects/llm_framework/main_camera/src/camera.h @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#ifndef CAMERA_H +#define CAMERA_H + +#include +#include +// #define CONFIG_CAPTURE_WIDTH 320 // Display width +// #define CONFIG_CAPTURE_HEIGHT 320 // Display height +#define CONFIG_CAPTURE_FPS 30 // +#define CONFIG_CAPTURE_BUF_CNT 10 // +#define CONFIG_ENCODE_BITRATE 2500 // + +#define CONFIG_DEVNAME_LEN 32 +#define CONFIG_IOCTL_RETRY 5 +#define CONFIG_CAPTURE_DEVICE "/dev/video0" + +#if __cplusplus +extern "C" { +#endif +typedef void (*vCameraFrameGet)(uint8_t* pData, uint32_t Width, uint32_t Height, uint32_t Length, void *ctx); +typedef enum { CAMERA_STATE_CLOSE, CAMERA_SATTE_OPEN, CAMERA_SATTE_CAP, CAMERA_STATE_ERR } CAMERA_STATE_t; + +typedef struct { + void* pStart; + int Length; +} CAMERA_Buffer_t; + +typedef struct Camera_t { + char DevName[32]; + int DevFd; + int BufferCnt; + int capture_fps; + int is_alloc; + uint32_t Width; + uint32_t Height; + CAMERA_Buffer_t* pCameraBuffer; + vCameraFrameGet pCallback; + CAMERA_STATE_t State; /* 0 Not open, 1 Turn on the camera, 2 Start capturing, + 3 Error */ + pthread_t CaptureThreadId; + void *ctx_; + /** + * Set capture frame callback + * Return value: 0 for success, -1 for failure + */ + int (*CameraCaptureCallbackSet)(struct Camera_t*, vCameraFrameGet); + /** + * Start capturing + * Return value: 0 for success, -1 for failure + */ + int (*CameraCaptureStart)(struct Camera_t*); + /** + * Stop capturing + * Return value: 0 for success, -1 for failure + */ + int (*CameraCaptureStop)(struct Camera_t*); + void (*CameraSetCtx)(struct Camera_t*, void *); +} Camera_t; + +/** + * Open the camera + * @pDevName Device node + * Return value: NULL for failure + */ +Camera_t* CameraOpen(const char* pDevName, int width, int height, int fps); + +/** + * Open the camera from config + * @pDevName Device node + * Return value: 0 for success, -1 for failure + */ +int CameraOpenFrom(Camera_t* Camera); + +/** + * Close the camera + * Return value: 0 for success, -1 for failure + */ +int CameraClose(Camera_t* Camera); + +#if __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/main.cpp b/projects/llm_framework/main_camera/src/main.cpp new file mode 100644 index 00000000..b617fe68 --- /dev/null +++ b/projects/llm_framework/main_camera/src/main.cpp @@ -0,0 +1,288 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" +#include "camera.h" +#include +#include + +using namespace StackFlows; +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + main_exit_flage = 1; +} + +typedef std::function task_callback_t; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: + Camera_t *cam; + +public: + std::string response_format_; + std::vector inputs_; + task_callback_t out_callback_; + bool enoutput_; + bool enstream_; + std::atomic_int cap_status_; + std::unique_ptr camera_cap_thread_; + std::atomic_bool camera_clear_flage_; + + std::string devname_; + int frame_width_; + int frame_height_; + + static void on_cap_fream(uint8_t *pData, uint32_t width, uint32_t height, uint32_t Length, void *ctx) + { + llm_task *self = static_cast(ctx); + cv::Mat yuv240(height, width, CV_8UC2, pData); + cv::Mat yuv320(self->frame_height_, self->frame_width_, CV_8UC2); + int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv240.copyTo(yuv320(cv::Rect(offsetX, offsetY, width, height))); + if (self->out_callback_) self->out_callback_(yuv320.data, self->frame_height_ * self->frame_width_ * 2); + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + bool parse_config(const nlohmann::json &config_body) + { + try { + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + devname_ = config_body.at("devname"); + frame_width_ = config_body.at("frame_width"); + frame_height_ = config_body.at("frame_height"); + + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } + } catch (...) { + return true; + } + enstream_ = (response_format_.find("stream") != std::string::npos); + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + try { + printf("CameraOpen %d %d \n", frame_width_, frame_height_); + cam = CameraOpen(devname_.c_str(), frame_width_, frame_height_, 30); + if (cam == NULL) { + printf("Camera open failed \n"); + return -1; + } + cam->ctx_ = static_cast(this); + cam->CameraCaptureCallbackSet(cam, on_cap_fream); + cam->CameraCaptureStart(cam); + } catch (...) { + SLOGE("config file read false"); + return -3; + } + return 0; + } + + void inference(const std::string &msg) + { + // std::cout << msg << std::endl; + if (out_callback_) out_callback_("None", 4); + } + + llm_task(const std::string &workid) + { + cam = NULL; + } + + ~llm_task() + { + if (cam) { + cam->CameraCaptureStop(cam); + CameraClose(cam); + cam = NULL; + } + } +}; + +class llm_camera : public StackFlow { +private: + std::unordered_map> llm_task_; + +public: + llm_camera() : StackFlow("camera") + { + rpc_ctx_->register_rpc_action( + "list_camera", std::bind(&llm_camera::list_camera, this, std::placeholders::_1, std::placeholders::_2)); + } + + std::string list_camera(pzmq *_pzmq, const std::string &rawdata) + { + nlohmann::json req_body; + std::string zmq_url = RPC_PARSE_TO_FIRST(rawdata); + std::string param_json = RPC_PARSE_TO_SECOND(rawdata); + std::vector devices; + glob_t glob_result; + glob("/dev/video*", GLOB_TILDE, NULL, &glob_result); + for (size_t i = 0; i < glob_result.gl_pathc; ++i) { + devices.push_back(std::string(glob_result.gl_pathv[i])); + } + globfree(&glob_result); + send("camera.devices", devices, LLM_NO_ERROR, sample_json_str_get(param_json, "work_id"), zmq_url); + return LLM_NONE; + } + + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const void *data, int size) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + std::string out_data((char *)data, size); + llm_channel->send_raw_to_pub(out_data); + if (llm_task_obj->enoutput_) { + std::string base64_data; + int ret = StackFlows::encode_base64(out_data, base64_data); + std::string out_json_str; + out_json_str += R"({"request_id":")"; + out_json_str += llm_channel->request_id_; + out_json_str += R"(","work_id":")"; + out_json_str += llm_channel->work_id_; + out_json_str += R"(","object":"image.raw.base64","error":{"code":0, "message":""},"data":")"; + out_json_str += base64_data; + out_json_str += R"("})"; + llm_channel->send_raw_to_usr(out_json_str); + } + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + llm_task_obj->set_output(std::bind(&llm_camera::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + llm_task_[work_id_num] = llm_task_obj; + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("camera.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs"] = llm_task_obj->inputs_; + send("camera.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_camera() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main(int argc, char *argv[]) +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_camera llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} diff --git a/projects/llm_framework/main_camera_test/Kconfig b/projects/llm_framework/main_camera_test/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/projects/llm_framework/main_camera_test/SConstruct b/projects/llm_framework/main_camera_test/SConstruct new file mode 100644 index 00000000..b5f524ea --- /dev/null +++ b/projects/llm_framework/main_camera_test/SConstruct @@ -0,0 +1,68 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +# SRCS = append_srcs_dir(ADir('src')) +SRCS = Glob('src/*.c*') +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +# INCLUDE += [ADir('../include'), ADir('../include/cppzmq'), ADir('../include/libzmq')] +LINK_SEARCH_PATH += [ADir('../static_lib')] + +REQUIREMENTS += ['ax_sys', 'ax_interpreter'] + + + + + + + + +INCLUDE += [ADir('../include/opencv4')] + +static_file = [] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] +static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] +STATIC_LIB += static_file * 4 + +# STATIC_FILES += [AFile('camera.json')] + +env['COMPONENTS'].append({'target':'llm_camera_test', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_camera_test/camera.json b/projects/llm_framework/main_camera_test/camera.json new file mode 100644 index 00000000..608b163c --- /dev/null +++ b/projects/llm_framework/main_camera_test/camera.json @@ -0,0 +1,18 @@ +{ + "mode": "None", + "type": "camera", + "capabilities": [ + "play", + "cap" + ], + "input_type": [ + "rpc.camera.wav.base64", + "rpc.camera.pcm.base64" + ], + "output_type": [ + "camera.pcm.stream" + ], + "cap_param": { + "None": "None" + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_camera_test/src/main.cpp b/projects/llm_framework/main_camera_test/src/main.cpp new file mode 100644 index 00000000..80c1dfa9 --- /dev/null +++ b/projects/llm_framework/main_camera_test/src/main.cpp @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + main_exit_flage = 1; +} + + +int main(int argc, char *argv[]) +{ + // 打开默认摄像头 + cv::VideoCapture cap(0); + + // 检查摄像头是否成功打开 + if (!cap.isOpened()) { + std::cerr << "无法打开摄像头" << std::endl; + return -1; + } + + cv::Mat frame; + while (true) { + // 读取摄像头的一帧 + cap >> frame; + + // 检查是否成功读取帧 + if (frame.empty()) { + std::cerr << "无法接收帧 (可能是流的终止?)" << std::endl; + break; + } + std::cout << "get ones frame \n"; + // // 显示捕获的帧 + // cv::imshow("摄像头", frame); + + // // 按下'q'键退出 + // if (cv::waitKey(1) == 'q') { + // break; + // } + } + + // 释放摄像头并关闭所有窗口 + cap.release(); + cv::destroyAllWindows(); + + return 0; +} diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 5b8f9433..85f4b66c 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -203,7 +203,7 @@ class llm_task { out_callback_ = out_callback; } - void sys_pcm_on_data(pzmq *_pzmq, const std::string &raw) + void sys_pcm_on_data(const std::string &raw) { static int count = 0; if (count < delay_audio_frame_) { @@ -337,8 +337,10 @@ class llm_kws : public StackFlow { return; } if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { - llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), - std::placeholders::_1, std::placeholders::_2)); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } } @@ -417,7 +419,7 @@ class llm_kws : public StackFlow { } next_data = &tmp_msg2; } - llm_task_obj->sys_pcm_on_data(nullptr, (*next_data)); + llm_task_obj->sys_pcm_on_data((*next_data)); } int setup(const std::string &work_id, const std::string &object, const std::string &data) override @@ -454,9 +456,11 @@ class llm_kws : public StackFlow { for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { - audio_url_ = unit_call("audio", "cap", "None"); - llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(), - std::placeholders::_1, std::placeholders::_2)); + audio_url_ = unit_call("audio", "cap", "None"); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); llm_task_obj->audio_flage_ = true; } else if (input.find("kws") != std::string::npos) { llm_channel->subscriber_work_id( @@ -500,7 +504,7 @@ class llm_kws : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("kws.taskinfo", req_body, LLM_NO_ERROR, work_id); } } @@ -516,6 +520,8 @@ class llm_kws : public StackFlow { send("None", "None", error_body, work_id); return -1; } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); if (llm_task_[work_id_num]->audio_flage_) { unit_call("audio", "cap_stop", "None"); } diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 8b6c5572..efffc034 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -507,7 +507,7 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index cd6ce1ee..532b035f 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -649,7 +649,7 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("melotts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 5fba679a..8f719212 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -530,6 +530,8 @@ void unit_action_match(int com_id, const std::string &json_str) usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } + if(work_id.empty()) + work_id = "sys"; std::string action; error = doc["action"].get_string(action); if (error) { diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index cfc91b28..c5a7ef3d 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -487,7 +487,7 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("tts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 4a1c1eb7..d08a818d 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -520,7 +520,7 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("vlm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index de9e01b9..d85b9ad4 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -56,6 +56,8 @@ class llm_task { bool enstream_; static int ax_init_flage_; task_callback_t out_callback_; + std::atomic_bool camera_flage_; + std::mutex inference_mtx_; bool parse_config(const nlohmann::json &config_body) { @@ -140,16 +142,39 @@ class llm_task { out_callback_ = out_callback; } - bool inference(const std::string &msg) + bool inference_decode(const std::string &msg) { - try { - cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); - if (src.empty()) return true; + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); + if (src.empty()) return true; + return inference(src); + } + + bool inference_raw_yuv(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 2) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); + cv::Mat bgr; + cv::cvtColor(camera_data, bgr, cv::COLOR_YUV2BGR_YUYV); + return inference(bgr); + } + bool inference(cv::Mat &src) + { + try { + int ret = -1; std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); - int ret = -1; - yolo_->SetInput(image.data(), 0); + yolo_->SetInput((void *)image.data(), 0); if (0 != yolo_->RunSync()) { SLOGE("Run yolo model failed!\n"); throw std::string("yolo_ RunSync error"); @@ -328,13 +353,31 @@ class llm_yolo : public StackFlow { return; } next_data = &tmp_msg2; - if (llm_task_obj->inference(*next_data)) { + + if (llm_task_obj->inference_decode(*next_data)) { error_body["code"] = -11; error_body["message"] = "Model run failed."; send("None", "None", error_body, unit_name_); } } + void task_camera_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + SLOGE("Model run failed."); + return; + } + try { + llm_task_obj->inference_raw_yuv(data); + } catch (...) { + SLOGE("data format error"); + } + } + int setup(const std::string &work_id, const std::string &object, const std::string &data) override { nlohmann::json error_body; @@ -373,8 +416,17 @@ class llm_yolo : public StackFlow { "", std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); - } else if (input.find("sys") != std::string::npos) { - // TODO:... + } else { + std::string input_url_name = input + ".out_port"; + std::string input_url = unit_call("sys", "sql_select", input_url_name); + if (!input_url.empty()) { + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; + llm_channel->subscriber( + input_url, [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { + this->task_camera_data(_llm_task_obj, _llm_channel, raw); + }); + } } llm_task_[work_id_num] = llm_task_obj; SLOGI("load_mode success"); @@ -411,8 +463,18 @@ class llm_yolo : public StackFlow { std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); - } else if (data.find("sys") != std::string::npos) { - // TODO:... + } else if (data.find("camera") != std::string::npos){ + std::string input_url_name = data + ".out_port"; + std::string input_url = unit_call("sys", "sql_select", input_url_name); + if (!input_url.empty()) { + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; + llm_channel->subscriber( + input_url, [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { + this->task_camera_data(_llm_task_obj, _llm_channel, raw); + }); + } + llm_task_obj->inputs_.push_back(data); } if (ret) { error_body["code"] = -20; @@ -471,7 +533,7 @@ class llm_yolo : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs_"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); } } From bc703320c5f44510f765b7342bcf806f7259b16b Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Mon, 2 Dec 2024 15:35:15 +0800 Subject: [PATCH 29/44] [update] - update camera. - StackFlow add get_config_file_paths. - StackFlow output add \n. - sys.hwinfo add eth info. --- .../StackFlow/stackflow/StackFlow.h | 11 +- .../StackFlow/stackflow/StackFlowUtil.cpp | 15 + .../StackFlow/stackflow/StackFlowUtil.h | 2 + projects/llm_framework/main_asr/src/main.cpp | 6 +- .../llm_framework/main_audio/src/main.cpp | 4 +- .../llm_framework/main_camera/src/camera.c | 389 +++++------------- .../llm_framework/main_camera/src/camera.h | 61 ++- .../llm_framework/main_camera/src/main.cpp | 55 ++- .../llm_framework/main_camera_test/Kconfig | 0 .../llm_framework/main_camera_test/SConstruct | 68 --- .../main_camera_test/camera.json | 18 - .../main_camera_test/src/main.cpp | 59 --- projects/llm_framework/main_kws/src/main.cpp | 4 +- projects/llm_framework/main_llm/src/main.cpp | 4 +- .../llm_framework/main_melotts/src/main.cpp | 4 +- .../llm_framework/main_sys/src/event_loop.cpp | 22 + .../llm_framework/main_sys/src/zmq_bus.cpp | 3 +- projects/llm_framework/main_tts/src/main.cpp | 4 +- projects/llm_framework/main_vlm/src/main.cpp | 4 +- projects/llm_framework/main_yolo/src/main.cpp | 4 +- projects/llm_framework/tools/llm_pack.py | 2 +- 21 files changed, 219 insertions(+), 520 deletions(-) delete mode 100644 projects/llm_framework/main_camera_test/Kconfig delete mode 100644 projects/llm_framework/main_camera_test/SConstruct delete mode 100644 projects/llm_framework/main_camera_test/camera.json delete mode 100644 projects/llm_framework/main_camera_test/src/main.cpp diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h index 30ce62f5..2098458c 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.h +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -133,6 +133,7 @@ class llm_channel_obj { } else out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; if (outuart) return output_to_uart(out); else @@ -155,6 +156,7 @@ class llm_channel_obj { out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; send_raw_to_pub(out); if (enoutput_) return send_raw_to_usr(out); return 0; @@ -176,6 +178,7 @@ class llm_channel_obj { out_body["error"] = error_msg; std::string out = out_body.dump(); + out += "\n"; send_raw_to_pub(out); if (enoutput_) return send_raw_to_usr(out); return 0; @@ -342,10 +345,14 @@ class StackFlow { out_body["error"] = error_msg; if (zmq_url.empty()) { pzmq _zmq(out_zmq_url_, ZMQ_PUSH); - return _zmq.send_data(out_body.dump()); + std::string out = out_body.dump(); + out += "\n"; + return _zmq.send_data(out); } else { pzmq _zmq(zmq_url, ZMQ_PUSH); - return _zmq.send_data(out_body.dump()); + std::string out = out_body.dump(); + out += "\n"; + return _zmq.send_data(out); } } diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 756889ec..89a31409 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -263,4 +263,19 @@ std::string StackFlows::unit_call(const std::string &unit_name, const std::strin pzmq _call(unit_name); _call.call_rpc_action(unit_action, data, [&value](pzmq *_pzmq, const std::string &raw) { value = raw; }); return value; +} + +std::list StackFlows::get_config_file_paths(const std::string &base_model_path, + const std::string &mode_name) +{ + std::shared_ptr base_path((std::string *)(&base_model_path), [](std::string *Npt) {}); + if (base_path->empty()) base_path = std::make_shared("/opt/m5stack/data/"); + std::list config_file_paths; + config_file_paths.push_back(std::string("./") + mode_name + ".json"); + config_file_paths.push_back(std::string("./asr_mode_") + mode_name + ".json"); + config_file_paths.push_back((*base_path) + "../share/" + mode_name + ".json"); + config_file_paths.push_back((*base_path) + "../share/asr_mode_" + mode_name + ".json"); + config_file_paths.push_back((*base_path) + mode_name + ".json"); + config_file_paths.push_back((*base_path) + "asr_mode_" + mode_name + ".json"); + return config_file_paths; } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h index ffbe00f6..d4713f33 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.h +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -7,6 +7,7 @@ #include #include #include +#include #define WORK_ID_NONE -100 #define RPC_PUSH_PARAM(_obj, _data1, _data2) \ @@ -31,4 +32,5 @@ bool decode_stream(const std::string &in, std::string &out, std::unordered_map get_config_file_paths(const std::string &base_model_path, const std::string &mode_name); }; // namespace StackFlows diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 9c4d39e6..f4c8f9d8 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -92,9 +92,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -644,7 +642,7 @@ class llm_asr : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("asr.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 5dda246e..cb38547d 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -139,9 +139,7 @@ class llm_audio : public StackFlow { nlohmann::json config_body; nlohmann::json file_body; nlohmann::json error_body; - std::list config_file_paths; - config_file_paths.push_back("./audio.json"); - config_file_paths.push_back("/opt/m5stack/share/audio.json"); + std::list config_file_paths = get_config_file_paths("/opt/m5stack/data", "audio"); try { config_body = nlohmann::json::parse(data); for (auto file_name : config_file_paths) { diff --git a/projects/llm_framework/main_camera/src/camera.c b/projects/llm_framework/main_camera/src/camera.c index 0150857c..c9efbf71 100644 --- a/projects/llm_framework/main_camera/src/camera.c +++ b/projects/llm_framework/main_camera/src/camera.c @@ -17,42 +17,26 @@ #include #include -/** - * Name of storage device - */ -static int DevNameSet(struct Camera_t* Camera, const char* pDevName) +static int camera_capture_callback_set(struct camera_t* camera, vcamera_frame_get pcallback) { - int CopyLen = 0; - CopyLen = strlen(pDevName); - if (CopyLen > CONFIG_DEVNAME_LEN - 1) { - SLOGE("Error: device name length over limit: %d", CopyLen); - return -1; - } - memset(Camera->DevName, 0, CONFIG_DEVNAME_LEN); - memcpy(Camera->DevName, pDevName, CopyLen); - return 0; -} - -static int CameraCaptureCallbackSet(struct Camera_t* Camera, vCameraFrameGet pCallback) -{ - if (Camera->State == CAMERA_SATTE_CAP) { + if (camera->state_ == CAMERA_SATTE_CAP) { SLOGW("Set capture callback failed"); return -1; } - Camera->pCallback = pCallback; + camera->pcallback_ = pcallback; return 0; } -static void* CameraCaptureThread(void* pParam) +static void* camera_capture_thread(void* param) { int Ret = -1; - Camera_t* Camera = (Camera_t*)pParam; + camera_t* camera = (camera_t*)param; struct v4l2_buffer EnQueueBuf; struct v4l2_buffer DeQueueBuf; SLOGI("Start capture"); - while (Camera->State == CAMERA_SATTE_CAP) { + while (camera->state_ == CAMERA_SATTE_CAP) { memset(&DeQueueBuf, 0, sizeof(struct v4l2_buffer)); DeQueueBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; DeQueueBuf.memory = V4L2_MEMORY_MMAP; @@ -62,18 +46,18 @@ static void* CameraCaptureThread(void* pParam) EnQueueBuf.memory = V4L2_MEMORY_MMAP; // Retrieve data from the queue - Ret = ioctl(Camera->DevFd, VIDIOC_DQBUF, &DeQueueBuf); + Ret = ioctl(camera->dev_fd_, VIDIOC_DQBUF, &DeQueueBuf); if (Ret) { perror("Dequeue fail"); break; } - Camera->pCallback(Camera->pCameraBuffer[DeQueueBuf.index].pStart, Camera->Width, Camera->Height, - Camera->pCameraBuffer[DeQueueBuf.index].Length, Camera->ctx_); + camera->pcallback_(camera->pcamera_buffer_[DeQueueBuf.index].pstart, camera->width_, camera->height_, + camera->pcamera_buffer_[DeQueueBuf.index].length, camera->ctx_); // Re-queue the data EnQueueBuf.index = DeQueueBuf.index; - if (ioctl(Camera->DevFd, VIDIOC_QBUF, &EnQueueBuf)) { + if (ioctl(camera->dev_fd_, VIDIOC_QBUF, &EnQueueBuf)) { perror("Enqueue fail"); break; } @@ -84,52 +68,52 @@ static void* CameraCaptureThread(void* pParam) return NULL; } -static int CameraCaptureStart(struct Camera_t* Camera) +static int camera_capture_start(struct camera_t* camera) { SLOGI("Start capture thread"); - if (!Camera->pCallback) { + if (!camera->pcallback_) { SLOGW("Capture callback not set, start faild"); return -1; } - if (Camera->State != CAMERA_SATTE_OPEN) { + if (camera->state_ != CAMERA_SATTE_OPEN) { SLOGW("Camera not open, start faild"); return -1; } - Camera->State = CAMERA_SATTE_CAP; + camera->state_ = CAMERA_SATTE_CAP; - pthread_create(&Camera->CaptureThreadId, NULL, CameraCaptureThread, Camera); + pthread_create(&camera->capture_thread_id_, NULL, camera_capture_thread, camera); return 0; } -static int CameraCaptureStop(struct Camera_t* Camera) +static int camera_capture_stop(struct camera_t* camera) { SLOGI("Stop capture thread"); - Camera->State = CAMERA_SATTE_OPEN; - pthread_join(Camera->CaptureThreadId, NULL); + camera->state_ = CAMERA_SATTE_OPEN; + pthread_join(camera->capture_thread_id_, NULL); SLOGI("Capture thread stop"); return 0; } -static void CameraSetCtx(struct Camera_t* Camera, void* ctx) +static void camera_set_ctx(struct camera_t* camera, void* ctx) { - Camera->ctx_ = ctx; + camera->ctx_ = ctx; } -int CameraOpenFrom(Camera_t* Camera) +int camera_open_from(camera_t* camera) { int Ret = -1; - if (Camera == NULL) return -1; + if (camera == NULL) return -1; /* Check whether the camera is already open or in an error state */ - SLOGI("Open camera %s...", Camera->DevName); - if (Camera->State != CAMERA_STATE_CLOSE) { - SLOGE("Error: camera was open or meet error, now state is: %d", Camera->State); + SLOGI("Open camera %s...", camera->dev_name_); + if (camera->state_ != CAMERA_STATE_CLOSE) { + SLOGE("Error: camera was open or meet error, now state is: %d", camera->state_); goto ErrorHandle; } /* Open the camera device */ - if ((Camera->DevFd = open(Camera->DevName, O_RDWR, 0)) == -1) { - SLOGE("Open camera %s faild.", Camera->DevName); + if ((camera->dev_fd_ = open(camera->dev_name_, O_RDWR, 0)) == -1) { + SLOGE("Open camera %s faild.", camera->dev_name_); goto ErrorHandle; } @@ -139,14 +123,14 @@ int CameraOpenFrom(Camera_t* Camera) struct v4l2_input Input; Input.index = 0; - if (ioctl(Camera->DevFd, VIDIOC_S_INPUT, &Input)) { + if (ioctl(camera->dev_fd_, VIDIOC_S_INPUT, &Input)) { perror("VIDIOC_S_INPUT"); } /* Retrieve camera information */ struct v4l2_capability Cap; - Ret = ioctl(Camera->DevFd, VIDIOC_QUERYCAP, &Cap); + Ret = ioctl(camera->dev_fd_, VIDIOC_QUERYCAP, &Cap); if (Ret) { perror("Get device info error"); @@ -169,7 +153,7 @@ int CameraOpenFrom(Camera_t* Camera) /* Enumerate supported frame formats */ SLOGI("Support format:"); - while (ioctl(Camera->DevFd, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { + while (ioctl(camera->dev_fd_, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { FmtDesc.index++; SLOGI("Pixel format = %c%c%c%c", (FmtDesc.pixelformat & 0xFF), ((FmtDesc.pixelformat >> 8) & 0xFF), ((FmtDesc.pixelformat >> 16) & 0xFF), (FmtDesc.pixelformat >> 24)); @@ -191,7 +175,7 @@ int CameraOpenFrom(Camera_t* Camera) /* Get video stream information */ memset(&StreamParam, 0, sizeof(struct v4l2_streamparm)); StreamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + Ret = ioctl(camera->dev_fd_, VIDIOC_G_PARM, &StreamParam); if (Ret) { perror("Get stream info failed"); goto ErrorHandle; @@ -205,16 +189,16 @@ int CameraOpenFrom(Camera_t* Camera) SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); /* Set frame rate denominator and numerator */ - StreamParam.parm.capture.timeperframe.denominator = Camera->capture_fps; + StreamParam.parm.capture.timeperframe.denominator = camera->capture_fps_; StreamParam.parm.capture.timeperframe.numerator = 1; - if (ioctl(Camera->DevFd, VIDIOC_S_PARM, &StreamParam) == -1) { + if (ioctl(camera->dev_fd_, VIDIOC_S_PARM, &StreamParam) == -1) { perror("Unable to set fps"); goto ErrorHandle; } /* Get video stream information */ - Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); + Ret = ioctl(camera->dev_fd_, VIDIOC_G_PARM, &StreamParam); if (Ret) { perror("Get stream info failed"); goto ErrorHandle; @@ -230,7 +214,7 @@ int CameraOpenFrom(Camera_t* Camera) /* Set automatic exposure */ struct v4l2_control Ctrl; Ctrl.id = V4L2_CID_EXPOSURE_ABSOLUTE; - Ret = ioctl(Camera->DevFd, VIDIOC_G_CTRL, &Ctrl); + Ret = ioctl(camera->dev_fd_, VIDIOC_G_CTRL, &Ctrl); if (Ret) { perror("Set exposure failed"); @@ -239,17 +223,17 @@ int CameraOpenFrom(Camera_t* Camera) /* Set capture format */ TvFmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - TvFmt.fmt.pix.width = Camera->Width; - TvFmt.fmt.pix.height = Camera->Height; + TvFmt.fmt.pix.width = camera->width_; + TvFmt.fmt.pix.height = camera->height_; TvFmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; TvFmt.fmt.pix.field = V4L2_FIELD_INTERLACED; - if (ioctl(Camera->DevFd, VIDIOC_S_FMT, &TvFmt) < 0) { + if (ioctl(camera->dev_fd_, VIDIOC_S_FMT, &TvFmt) < 0) { perror("Unable set VIDIOC_S_FMT"); goto ErrorHandle; } - if (ioctl(Camera->DevFd, VIDIOC_G_FMT, &TvFmt) < 0) { + if (ioctl(camera->dev_fd_, VIDIOC_G_FMT, &TvFmt) < 0) { perror("Get VIDIOC_S_FMT error"); goto ErrorHandle; } @@ -259,53 +243,53 @@ int CameraOpenFrom(Camera_t* Camera) SLOGI("Pixel format = %c%c%c%c", (TvFmt.fmt.pix.pixelformat & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 8) & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 16) & 0xFF), (TvFmt.fmt.pix.pixelformat >> 24)); - Camera->Width = TvFmt.fmt.pix.width; - Camera->Height = TvFmt.fmt.pix.height; + camera->width_ = TvFmt.fmt.pix.width; + camera->height_ = TvFmt.fmt.pix.height; /* Request V4L2 driver to allocate memory */ struct v4l2_requestbuffers ReqBuffer; memset(&ReqBuffer, 0, sizeof(ReqBuffer)); - ReqBuffer.count = Camera->BufferCnt; + ReqBuffer.count = camera->buffer_cnt_; ReqBuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ReqBuffer.memory = V4L2_MEMORY_MMAP; - if (-1 == ioctl(Camera->DevFd, VIDIOC_REQBUFS, &ReqBuffer)) { + if (-1 == ioctl(camera->dev_fd_, VIDIOC_REQBUFS, &ReqBuffer)) { perror("Fail to ioctl 'VIDIOC_REQBUFS'"); goto ErrorHandle; } - Camera->BufferCnt = ReqBuffer.count; - SLOGI("Buffer count: %d", Camera->BufferCnt); + camera->buffer_cnt_ = ReqBuffer.count; + SLOGI("Buffer count: %d", camera->buffer_cnt_); - Camera->pCameraBuffer = (CAMERA_Buffer_t*)calloc(Camera->BufferCnt, sizeof(CAMERA_Buffer_t)); - if (Camera->pCameraBuffer == NULL) { + camera->pcamera_buffer_ = (CAMERA_Buffer_t*)calloc(camera->buffer_cnt_, sizeof(CAMERA_Buffer_t)); + if (camera->pcamera_buffer_ == NULL) { SLOGE("Malloc buffer failed"); goto ErrorHandle; } /* Map kernel buffer to user space */ struct v4l2_buffer Buf; - for (int i = 0; i < Camera->BufferCnt; i++) { + for (int i = 0; i < camera->buffer_cnt_; i++) { memset(&Buf, 0, sizeof(Buf)); Buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; Buf.memory = V4L2_MEMORY_MMAP; Buf.index = i; - if (-1 == ioctl(Camera->DevFd, VIDIOC_QUERYBUF, &Buf)) { + if (-1 == ioctl(camera->dev_fd_, VIDIOC_QUERYBUF, &Buf)) { perror("Fail to ioctl : VIDIOC_QUERYBUF"); goto ErrorHandle; } - Camera->pCameraBuffer[i].Length = Buf.length; - Camera->pCameraBuffer[i].pStart = - mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, Camera->DevFd, Buf.m.offset); + camera->pcamera_buffer_[i].length = Buf.length; + camera->pcamera_buffer_[i].pstart = + mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, camera->dev_fd_, Buf.m.offset); - if (MAP_FAILED == Camera->pCameraBuffer[i].pStart) { + if (MAP_FAILED == camera->pcamera_buffer_[i].pstart) { perror("Fail to mmap"); goto ErrorHandle; } - if (ioctl(Camera->DevFd, VIDIOC_QBUF, &Buf)) { + if (ioctl(camera->dev_fd_, VIDIOC_QBUF, &Buf)) { perror("Unable to queue buffer"); goto ErrorHandle; } @@ -314,276 +298,87 @@ int CameraOpenFrom(Camera_t* Camera) /* Start capturing */ enum v4l2_buf_type BufType; BufType = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(Camera->DevFd, VIDIOC_STREAMON, &BufType) < 0) { + if (ioctl(camera->dev_fd_, VIDIOC_STREAMON, &BufType) < 0) { perror("VIDIOC_STREAMON error"); goto ErrorHandle; } - Camera->State = CAMERA_SATTE_OPEN; - Camera->CameraCaptureCallbackSet = CameraCaptureCallbackSet; - Camera->CameraCaptureStart = CameraCaptureStart; - Camera->CameraCaptureStop = CameraCaptureStop; - Camera->CameraSetCtx = CameraSetCtx; - Camera->is_alloc = 0; + camera->state_ = CAMERA_SATTE_OPEN; + camera->camera_capture_callback_set = camera_capture_callback_set; + camera->camera_capture_start = camera_capture_start; + camera->camera_capture_stop = camera_capture_stop; + camera->camera_set_ctx = camera_set_ctx; + camera->is_alloc_ = 0; return 0; ErrorHandle: SLOGE("Camera open meet error, now handle it"); - close(Camera->DevFd); - free(Camera->pCameraBuffer); + close(camera->dev_fd_); + free(camera->pcamera_buffer_); return -1; } -Camera_t* CameraOpen(const char* pDevName, int width, int height, int fps) +camera_t* camera_open(const char* pdev_name, int width, int height, int fps) { int Ret = -1; - Camera_t* Camera = (Camera_t*)malloc(sizeof(Camera_t)); - if (Camera == NULL) return NULL; - memset(Camera, 0, sizeof(Camera_t)); - /* Check whether the camera is already open or in an error state */ - SLOGI("Open camera %s...", pDevName); - if (Camera->State != CAMERA_STATE_CLOSE) { - SLOGE("Error: camera was open or meet error, now state is: %d", Camera->State); - goto ErrorHandle; - } - - /* Open the camera device */ - if ((Camera->DevFd = open(pDevName, O_RDWR, 0)) == -1) { - SLOGE("Open camera %s faild.", pDevName); - goto ErrorHandle; - } - - SLOGI("Open device successful."); - - /* Set the capture source */ - struct v4l2_input Input; - Input.index = 0; + camera_t* camera = (camera_t*)malloc(sizeof(camera_t)); + if (camera == NULL) return NULL; + memset(camera, 0, sizeof(camera_t)); + camera->buffer_cnt_ = CONFIG_CAPTURE_BUF_CNT; - if (ioctl(Camera->DevFd, VIDIOC_S_INPUT, &Input)) { - perror("VIDIOC_S_INPUT"); - } - - /* Retrieve camera information */ - struct v4l2_capability Cap; - - Ret = ioctl(Camera->DevFd, VIDIOC_QUERYCAP, &Cap); - - if (Ret) { - perror("Get device info error"); - goto ErrorHandle; - } - - SLOGI("Get device info successful."); - - SLOGI("Driver name: %s", Cap.driver); - SLOGI("Card name: %s", Cap.card); - SLOGI("Bus info: %s", Cap.bus_info); - - DevNameSet(Camera, pDevName); - - struct v4l2_format TvFmt; - struct v4l2_fmtdesc FmtDesc; - struct v4l2_streamparm StreamParam; - - memset(&FmtDesc, 0, sizeof(FmtDesc)); - FmtDesc.index = 0; - FmtDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - - /* Enumerate supported frame formats */ - SLOGI("Support format:"); - while (ioctl(Camera->DevFd, VIDIOC_ENUM_FMT, &FmtDesc) == 0) { - FmtDesc.index++; - SLOGI("Pixel format = %c%c%c%c", (FmtDesc.pixelformat & 0xFF), ((FmtDesc.pixelformat >> 8) & 0xFF), - ((FmtDesc.pixelformat >> 16) & 0xFF), (FmtDesc.pixelformat >> 24)); - SLOGI("Description = %s ", FmtDesc.description); - } - - /* Determine if the device is a device that can capture video */ - if (!(Cap.capabilities & V4L2_BUF_TYPE_VIDEO_CAPTURE)) { - SLOGE("The Current device is not a video capture device"); - goto ErrorHandle; - } - - /* Determine if video streaming is supported */ - if (!(Cap.capabilities & V4L2_CAP_STREAMING)) { - SLOGE("The Current device does not support streaming i/o"); - goto ErrorHandle; - } - - /* Get video stream information */ - memset(&StreamParam, 0, sizeof(struct v4l2_streamparm)); - StreamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); - if (Ret) { - perror("Get stream info failed"); - goto ErrorHandle; - } - - SLOGI("Before setting stream params:"); - SLOGI("Capability: %u", StreamParam.parm.capture.capability); - SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); - SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); - SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); - SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); - - /* Set frame rate denominator and numerator */ - StreamParam.parm.capture.timeperframe.denominator = fps; - StreamParam.parm.capture.timeperframe.numerator = 1; - - if (ioctl(Camera->DevFd, VIDIOC_S_PARM, &StreamParam) == -1) { - perror("Unable to set fps"); - goto ErrorHandle; - } - - /* Get video stream information */ - Ret = ioctl(Camera->DevFd, VIDIOC_G_PARM, &StreamParam); - if (Ret) { - perror("Get stream info failed"); + int CopyLen = strlen(pdev_name); + if (CopyLen > CONFIG_DEVNAME_LEN - 1) { + SLOGE("Error: device name length over limit: %d", CopyLen); goto ErrorHandle; } + memset(camera->dev_name_, 0, CONFIG_DEVNAME_LEN); + memcpy(camera->dev_name_, pdev_name, CopyLen); - SLOGI("After setting stream params:"); - SLOGI("Capability: %u", StreamParam.parm.capture.capability); - SLOGI("Capturemode: %u", StreamParam.parm.capture.capturemode); - SLOGI("Extendedmode: %u", StreamParam.parm.capture.extendedmode); - SLOGI("Timeperframe denominator: %u", StreamParam.parm.capture.timeperframe.denominator); - SLOGI("Timeperframe numerator: %u", StreamParam.parm.capture.timeperframe.numerator); - - /* Set automatic exposure */ - struct v4l2_control Ctrl; - Ctrl.id = V4L2_CID_EXPOSURE_ABSOLUTE; - Ret = ioctl(Camera->DevFd, VIDIOC_G_CTRL, &Ctrl); + camera->width_ = width; + camera->height_ = height; + camera->capture_fps_ = fps; + Ret = camera_open_from(camera); if (Ret) { - perror("Set exposure failed"); - // goto ErrorHandle; - } - - /* Set capture format */ - TvFmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - TvFmt.fmt.pix.width = width; - TvFmt.fmt.pix.height = height; - TvFmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; - TvFmt.fmt.pix.field = V4L2_FIELD_INTERLACED; - - if (ioctl(Camera->DevFd, VIDIOC_S_FMT, &TvFmt) < 0) { - perror("Unable set VIDIOC_S_FMT"); - goto ErrorHandle; - } - - if (ioctl(Camera->DevFd, VIDIOC_G_FMT, &TvFmt) < 0) { - perror("Get VIDIOC_S_FMT error"); - goto ErrorHandle; - } - - SLOGI("After setting frame params:"); - SLOGI("Width: %d, height: %d", TvFmt.fmt.pix.width, TvFmt.fmt.pix.height); - SLOGI("Pixel format = %c%c%c%c", (TvFmt.fmt.pix.pixelformat & 0xFF), ((TvFmt.fmt.pix.pixelformat >> 8) & 0xFF), - ((TvFmt.fmt.pix.pixelformat >> 16) & 0xFF), (TvFmt.fmt.pix.pixelformat >> 24)); - - Camera->Width = TvFmt.fmt.pix.width; - Camera->Height = TvFmt.fmt.pix.height; - - /* Request V4L2 driver to allocate memory */ - struct v4l2_requestbuffers ReqBuffer; - memset(&ReqBuffer, 0, sizeof(ReqBuffer)); - ReqBuffer.count = CONFIG_CAPTURE_BUF_CNT; - ReqBuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - ReqBuffer.memory = V4L2_MEMORY_MMAP; - - if (-1 == ioctl(Camera->DevFd, VIDIOC_REQBUFS, &ReqBuffer)) { - perror("Fail to ioctl 'VIDIOC_REQBUFS'"); - goto ErrorHandle; - } - - Camera->BufferCnt = ReqBuffer.count; - SLOGI("Buffer count: %d", Camera->BufferCnt); - - Camera->pCameraBuffer = (CAMERA_Buffer_t*)calloc(Camera->BufferCnt, sizeof(CAMERA_Buffer_t)); - if (Camera->pCameraBuffer == NULL) { - SLOGE("Malloc buffer failed"); - goto ErrorHandle; - } - - /* Map kernel buffer to user space */ - struct v4l2_buffer Buf; - for (int i = 0; i < Camera->BufferCnt; i++) { - memset(&Buf, 0, sizeof(Buf)); - Buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - Buf.memory = V4L2_MEMORY_MMAP; - Buf.index = i; - - if (-1 == ioctl(Camera->DevFd, VIDIOC_QUERYBUF, &Buf)) { - perror("Fail to ioctl : VIDIOC_QUERYBUF"); - goto ErrorHandle; - } - - Camera->pCameraBuffer[i].Length = Buf.length; - Camera->pCameraBuffer[i].pStart = - mmap(NULL, Buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, Camera->DevFd, Buf.m.offset); - - if (MAP_FAILED == Camera->pCameraBuffer[i].pStart) { - perror("Fail to mmap"); - goto ErrorHandle; - } - - if (ioctl(Camera->DevFd, VIDIOC_QBUF, &Buf)) { - perror("Unable to queue buffer"); - goto ErrorHandle; - ; - } - } - - /* Start capturing */ - enum v4l2_buf_type BufType; - BufType = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(Camera->DevFd, VIDIOC_STREAMON, &BufType) < 0) { - perror("VIDIOC_STREAMON error"); goto ErrorHandle; } - Camera->State = CAMERA_SATTE_OPEN; - Camera->CameraCaptureCallbackSet = CameraCaptureCallbackSet; - Camera->CameraCaptureStart = CameraCaptureStart; - Camera->CameraCaptureStop = CameraCaptureStop; - Camera->is_alloc = 1; - return Camera; + camera->is_alloc_ = 1; + return camera; ErrorHandle: SLOGE("Camera open meet error, now handle it"); - close(Camera->DevFd); - free(Camera->pCameraBuffer); - free(Camera); + free(camera); return NULL; } -int CameraClose(Camera_t* Camera) +int camera_close(camera_t* camera) { - if (Camera == NULL) return -1; - if (Camera->State == CAMERA_SATTE_OPEN || Camera->State == CAMERA_SATTE_CAP) { - SLOGI("Close camera %s", Camera->DevName); + if (camera == NULL) return -1; + if (camera->state_ == CAMERA_SATTE_OPEN || camera->state_ == CAMERA_SATTE_CAP) { + SLOGI("Close camera %s", camera->dev_name_); } else { SLOGW("Skip close camera progress"); return 0; } for (int i = 0; i < CONFIG_CAPTURE_BUF_CNT; i++) { - munmap(Camera->pCameraBuffer[i].pStart, Camera->pCameraBuffer[i].Length); - Camera->pCameraBuffer[i].Length = 0; + munmap(camera->pcamera_buffer_[i].pstart, camera->pcamera_buffer_[i].length); + camera->pcamera_buffer_[i].length = 0; } - free(Camera->pCameraBuffer); - Camera->pCameraBuffer = NULL; + free(camera->pcamera_buffer_); + camera->pcamera_buffer_ = NULL; int Off = 1; - if (ioctl(Camera->DevFd, VIDIOC_STREAMOFF, &Off)) { + if (ioctl(camera->dev_fd_, VIDIOC_STREAMOFF, &Off)) { perror("Stop camera fail"); return -1; } - close(Camera->DevFd); - if (Camera->is_alloc) free(Camera); + close(camera->dev_fd_); + if (camera->is_alloc_) free(camera); - SLOGI("Camera closed"); + SLOGI("camera closed"); return 0; } \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/camera.h b/projects/llm_framework/main_camera/src/camera.h index 97066131..71df9803 100644 --- a/projects/llm_framework/main_camera/src/camera.h +++ b/projects/llm_framework/main_camera/src/camera.h @@ -8,78 +8,71 @@ #include #include -// #define CONFIG_CAPTURE_WIDTH 320 // Display width -// #define CONFIG_CAPTURE_HEIGHT 320 // Display height -#define CONFIG_CAPTURE_FPS 30 // -#define CONFIG_CAPTURE_BUF_CNT 10 // -#define CONFIG_ENCODE_BITRATE 2500 // - -#define CONFIG_DEVNAME_LEN 32 -#define CONFIG_IOCTL_RETRY 5 -#define CONFIG_CAPTURE_DEVICE "/dev/video0" +#define CONFIG_CAPTURE_BUF_CNT 10 +#define CONFIG_DEVNAME_LEN 32 #if __cplusplus extern "C" { #endif -typedef void (*vCameraFrameGet)(uint8_t* pData, uint32_t Width, uint32_t Height, uint32_t Length, void *ctx); +typedef void (*vcamera_frame_get)(uint8_t* pdata, uint32_t width, uint32_t height, uint32_t length, void* ctx); typedef enum { CAMERA_STATE_CLOSE, CAMERA_SATTE_OPEN, CAMERA_SATTE_CAP, CAMERA_STATE_ERR } CAMERA_STATE_t; typedef struct { - void* pStart; - int Length; + void* pstart; + int length; } CAMERA_Buffer_t; -typedef struct Camera_t { - char DevName[32]; - int DevFd; - int BufferCnt; - int capture_fps; - int is_alloc; - uint32_t Width; - uint32_t Height; - CAMERA_Buffer_t* pCameraBuffer; - vCameraFrameGet pCallback; - CAMERA_STATE_t State; /* 0 Not open, 1 Turn on the camera, 2 Start capturing, +typedef struct camera_t { + char dev_name_[CONFIG_DEVNAME_LEN]; + int dev_fd_; + int buffer_cnt_; + int capture_fps_; + int is_alloc_; + uint32_t width_; + uint32_t height_; + CAMERA_Buffer_t* pcamera_buffer_; + vcamera_frame_get pcallback_; + CAMERA_STATE_t state_; /* 0 Not open, 1 Turn on the camera, 2 Start capturing, 3 Error */ - pthread_t CaptureThreadId; - void *ctx_; + pthread_t capture_thread_id_; + void* ctx_; /** * Set capture frame callback * Return value: 0 for success, -1 for failure */ - int (*CameraCaptureCallbackSet)(struct Camera_t*, vCameraFrameGet); + int (*camera_capture_callback_set)(struct camera_t*, vcamera_frame_get); /** * Start capturing * Return value: 0 for success, -1 for failure */ - int (*CameraCaptureStart)(struct Camera_t*); + int (*camera_capture_start)(struct camera_t*); /** * Stop capturing * Return value: 0 for success, -1 for failure */ - int (*CameraCaptureStop)(struct Camera_t*); - void (*CameraSetCtx)(struct Camera_t*, void *); -} Camera_t; + int (*camera_capture_stop)(struct camera_t*); + void (*camera_set_ctx)(struct camera_t*, void*); +} camera_t; /** * Open the camera - * @pDevName Device node + * @pdev_name Device node * Return value: NULL for failure */ -Camera_t* CameraOpen(const char* pDevName, int width, int height, int fps); +camera_t* camera_open(const char* pdev_name, int width, int height, int fps); /** * Open the camera from config * @pDevName Device node * Return value: 0 for success, -1 for failure */ -int CameraOpenFrom(Camera_t* Camera); +int camera_open_from(camera_t* camera); /** * Close the camera * Return value: 0 for success, -1 for failure */ -int CameraClose(Camera_t* Camera); +int camera_close(camera_t* camera); #if __cplusplus } diff --git a/projects/llm_framework/main_camera/src/main.cpp b/projects/llm_framework/main_camera/src/main.cpp index b617fe68..2884e272 100644 --- a/projects/llm_framework/main_camera/src/main.cpp +++ b/projects/llm_framework/main_camera/src/main.cpp @@ -33,7 +33,7 @@ typedef std::function task_callback_t; class llm_task { private: - Camera_t *cam; + camera_t *cam; public: std::string response_format_; @@ -48,16 +48,41 @@ class llm_task { std::string devname_; int frame_width_; int frame_height_; + cv::Mat yuv_dist_; static void on_cap_fream(uint8_t *pData, uint32_t width, uint32_t height, uint32_t Length, void *ctx) { llm_task *self = static_cast(ctx); - cv::Mat yuv240(height, width, CV_8UC2, pData); - cv::Mat yuv320(self->frame_height_, self->frame_width_, CV_8UC2); - int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; - int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; - yuv240.copyTo(yuv320(cv::Rect(offsetX, offsetY, width, height))); - if (self->out_callback_) self->out_callback_(yuv320.data, self->frame_height_ * self->frame_width_ * 2); + if ((self->frame_height_ == height) && (self->frame_width_ == width)) { + if (self->out_callback_) self->out_callback_(pData, Length); + } else { + cv::Mat yuv_src(height, width, CV_8UC2, pData); + if ((self->frame_height_ >= height) && (self->frame_width_ >= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src.copyTo(self->yuv_dist_(cv::Rect(offsetX, offsetY, width, height))); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else if ((self->frame_height_ <= height) && (self->frame_width_ <= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (width - self->frame_width_) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (height - self->frame_height_) / 2; + yuv_src(cv::Rect(offsetX, offsetY, self->frame_width_, self->frame_height_)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else if ((self->frame_height_ >= height) && (self->frame_width_ <= width)) { + int offsetX = (self->frame_width_ == width) ? 0 : (width - self->frame_width_) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src(cv::Rect(offsetX, offsetY, self->frame_width_, height)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } else { + int offsetX = (self->frame_width_ == width) ? 0 : (self->frame_width_ - width) / 2; + int offsetY = (self->frame_height_ == height) ? 0 : (self->frame_height_ - height) / 2; + yuv_src(cv::Rect(offsetX, offsetY, width, self->frame_height_)).copyTo(self->yuv_dist_); + if (self->out_callback_) + self->out_callback_(self->yuv_dist_.data, self->frame_height_ * self->frame_width_ * 2); + } + } } void set_output(task_callback_t out_callback) @@ -87,6 +112,7 @@ class llm_task { return true; } enstream_ = (response_format_.find("stream") != std::string::npos); + yuv_dist_ = cv::Mat(frame_height_, frame_width_, CV_8UC2); return false; } @@ -96,15 +122,14 @@ class llm_task { return -1; } try { - printf("CameraOpen %d %d \n", frame_width_, frame_height_); - cam = CameraOpen(devname_.c_str(), frame_width_, frame_height_, 30); + cam = camera_open(devname_.c_str(), frame_width_, frame_height_, 30); if (cam == NULL) { printf("Camera open failed \n"); return -1; } cam->ctx_ = static_cast(this); - cam->CameraCaptureCallbackSet(cam, on_cap_fream); - cam->CameraCaptureStart(cam); + cam->camera_capture_callback_set(cam, on_cap_fream); + cam->camera_capture_start(cam); } catch (...) { SLOGE("config file read false"); return -3; @@ -126,8 +151,8 @@ class llm_task { ~llm_task() { if (cam) { - cam->CameraCaptureStop(cam); - CameraClose(cam); + cam->camera_capture_stop(cam); + camera_close(cam); cam = NULL; } } @@ -178,9 +203,9 @@ class llm_camera : public StackFlow { out_json_str += llm_channel->request_id_; out_json_str += R"(","work_id":")"; out_json_str += llm_channel->work_id_; - out_json_str += R"(","object":"image.raw.base64","error":{"code":0, "message":""},"data":")"; + out_json_str += R"(","object":"image.yuvraw.base64","error":{"code":0, "message":""},"data":")"; out_json_str += base64_data; - out_json_str += R"("})"; + out_json_str += R"("}\n)"; llm_channel->send_raw_to_usr(out_json_str); } } diff --git a/projects/llm_framework/main_camera_test/Kconfig b/projects/llm_framework/main_camera_test/Kconfig deleted file mode 100644 index e69de29b..00000000 diff --git a/projects/llm_framework/main_camera_test/SConstruct b/projects/llm_framework/main_camera_test/SConstruct deleted file mode 100644 index b5f524ea..00000000 --- a/projects/llm_framework/main_camera_test/SConstruct +++ /dev/null @@ -1,68 +0,0 @@ -import os - -Import('env') -with open(env['PROJECT_TOOL_S']) as f: - exec(f.read()) - -# SRCS = append_srcs_dir(ADir('src')) -SRCS = Glob('src/*.c*') -INCLUDE = [ADir('include'), ADir('.')] -PRIVATE_INCLUDE = [] -REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] -STATIC_LIB = [] -DYNAMIC_LIB = [] -DEFINITIONS = [] -DEFINITIONS_PRIVATE = [] -LDFLAGS = [] -LINK_SEARCH_PATH = [] -STATIC_FILES = [] - -DEFINITIONS += ['-std=c++17'] -LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] -# INCLUDE += [ADir('../include'), ADir('../include/cppzmq'), ADir('../include/libzmq')] -LINK_SEARCH_PATH += [ADir('../static_lib')] - -REQUIREMENTS += ['ax_sys', 'ax_interpreter'] - - - - - - - - -INCLUDE += [ADir('../include/opencv4')] - -static_file = [] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libittnotify.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibjpeg-turbo.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibopenjp2.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibpng.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibtiff.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/liblibwebp.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_core.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_highgui.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgcodecs.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_imgproc.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libopencv_videoio.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal.a')] -static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] -STATIC_LIB += static_file * 4 - -# STATIC_FILES += [AFile('camera.json')] - -env['COMPONENTS'].append({'target':'llm_camera_test', - 'SRCS':SRCS, - 'INCLUDE':INCLUDE, - 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, - 'REQUIREMENTS':REQUIREMENTS, - 'STATIC_LIB':STATIC_LIB, - 'DYNAMIC_LIB':DYNAMIC_LIB, - 'DEFINITIONS':DEFINITIONS, - 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, - 'LDFLAGS':LDFLAGS, - 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, - 'STATIC_FILES':STATIC_FILES, - 'REGISTER':'project' - }) diff --git a/projects/llm_framework/main_camera_test/camera.json b/projects/llm_framework/main_camera_test/camera.json deleted file mode 100644 index 608b163c..00000000 --- a/projects/llm_framework/main_camera_test/camera.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "mode": "None", - "type": "camera", - "capabilities": [ - "play", - "cap" - ], - "input_type": [ - "rpc.camera.wav.base64", - "rpc.camera.pcm.base64" - ], - "output_type": [ - "camera.pcm.stream" - ], - "cap_param": { - "None": "None" - } -} \ No newline at end of file diff --git a/projects/llm_framework/main_camera_test/src/main.cpp b/projects/llm_framework/main_camera_test/src/main.cpp deleted file mode 100644 index 80c1dfa9..00000000 --- a/projects/llm_framework/main_camera_test/src/main.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD - * - * SPDX-License-Identifier: MIT - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include "../../../../SDK/components/utilities/include/sample_log.h" - -int main_exit_flage = 0; -static void __sigint(int iSigNo) -{ - main_exit_flage = 1; -} - - -int main(int argc, char *argv[]) -{ - // 打开默认摄像头 - cv::VideoCapture cap(0); - - // 检查摄像头是否成功打开 - if (!cap.isOpened()) { - std::cerr << "无法打开摄像头" << std::endl; - return -1; - } - - cv::Mat frame; - while (true) { - // 读取摄像头的一帧 - cap >> frame; - - // 检查是否成功读取帧 - if (frame.empty()) { - std::cerr << "无法接收帧 (可能是流的终止?)" << std::endl; - break; - } - std::cout << "get ones frame \n"; - // // 显示捕获的帧 - // cv::imshow("摄像头", frame); - - // // 按下'q'键退出 - // if (cv::waitKey(1) == 'q') { - // break; - // } - } - - // 释放摄像头并关闭所有窗口 - cap.release(); - cv::destroyAllWindows(); - - return 0; -} diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 85f4b66c..17713512 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -98,9 +98,7 @@ class llm_task { } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index efffc034..4540280b 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -85,9 +85,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 532b035f..266fd4ce 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -127,9 +127,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index 8f719212..a65650fe 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -35,6 +35,7 @@ #include "zmq_bus.h" #include "remote_action.h" #include +#include "hv/ifconfig.h" void usr_print_error(const std::string &request_id, const std::string &work_id, const std::string &error_msg, int zmq_out) @@ -156,6 +157,27 @@ int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) get_memory_info(&one, &two); float use = ((one * 1.0f) - (two * 1.0f)) / (one * 1.0f); data_body["mem"] = (int)(use * 100); + std::vector ifcs; + ifconfig(ifcs); + std::vector jifcs; + for (auto& item : ifcs) + { + nlohmann::json eth_info; + eth_info["name"] = item.name; + eth_info["ip"] = item.ip; + char eth_ip_buff[128] = {0}; + sprintf(eth_ip_buff, "/sys/class/net/%s/speed", item.name); + FILE *file = fopen(eth_ip_buff, "r"); + memset(eth_ip_buff, 0, sizeof(eth_ip_buff)); + if(file != NULL) + { + fread(eth_ip_buff, 1, sizeof(eth_ip_buff), file); + fclose(file); + } + eth_info["speed"] = eth_ip_buff; + jifcs.push_back(eth_info); + } + data_body["eth_info"] = jifcs; out_body["data"] = data_body; std::string out = out_body.dump(); zmq_com_send(com_id, out); diff --git a/projects/llm_framework/main_sys/src/zmq_bus.cpp b/projects/llm_framework/main_sys/src/zmq_bus.cpp index e60e5a13..a9fb7b85 100644 --- a/projects/llm_framework/main_sys/src/zmq_bus.cpp +++ b/projects/llm_framework/main_sys/src/zmq_bus.cpp @@ -154,7 +154,8 @@ void zmq_com_send(int com_id, const std::string &out_str) char zmq_push_url[128]; sprintf(zmq_push_url, zmq_c_format.c_str(), com_id); pzmq _zmq(zmq_push_url, ZMQ_PUSH); - _zmq.send_data(out_str); + std::string out = out_str + "\n"; + _zmq.send_data(out); } void zmq_bus_work() diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index c5a7ef3d..a15b4808 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -95,9 +95,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index d08a818d..825aab2d 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -89,9 +89,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index fa2e33e4..c1b69bec 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -89,9 +89,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths; - config_file_paths.push_back(std::string("./") + model_ + ".json"); - config_file_paths.push_back(base_model_path_ + "../share/" + model_ + ".json"); + std::list config_file_paths = get_config_file_paths(base_model_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index d998f610..46d3b28f 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -239,7 +239,7 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): os.system('rm ./*.deb m5stack_* -rf') exit(0) - version = '1.2' + version = '1.3' data_version = '0.2' src_folder = '../dist' revision = 'm5stack1' From fb8feab54b683e32c260cff0197af5e3332a72f9 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Mon, 2 Dec 2024 17:10:26 +0800 Subject: [PATCH 30/44] [update] StackFlow add default serial_zmq_url --- .gitignore | 1 + .../StackFlow/stackflow/StackFlow.cpp | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index c6f9a448..9ec37dd5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .vscode/settings.json +projects/core135_llm_product_test_ui \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp index 046ed9ff..236cc7ab 100644 --- a/ext_components/StackFlow/stackflow/StackFlow.cpp +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -188,8 +188,19 @@ StackFlow::StackFlow::StackFlow(const std::string &unit_name) status_.store(0); exit_flage_.store(false); - even_loop_thread_ = std::make_unique(std::bind(&StackFlow::even_loop, this)); - event_queue_.enqueue(EVENT_SYS_INIT, "", ""); + even_loop_thread_ = std::make_unique(std::bind(&StackFlow::even_loop, this)); + llm_channel_obj::uart_push_url = std::string("ipc:///tmp/llm/5556.sock"); + status_.store(1); + repeat_event(1000, [this]() { + std::string serial_zmq_url = this->sys_sql_select("serial_zmq_url"); + if (!serial_zmq_url.empty()) { + SLOGI("serial_zmq_url:%s", serial_zmq_url.c_str()); + llm_channel_obj::uart_push_url = serial_zmq_url; + return 0; + } else { + return 1; + } + }); } StackFlow::~StackFlow() @@ -222,17 +233,7 @@ void StackFlow::_none_event(const std::string &data1, const std::string &data2) void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) { - // serial_zmq_url - std::string serial_zmq_url; - serial_zmq_url = sys_sql_select("serial_zmq_url"); - if (!serial_zmq_url.empty()) { - SLOGI("serial_zmq_url:%s", serial_zmq_url.c_str()); - llm_channel_obj::uart_push_url = serial_zmq_url; - status_.store(1); - } else { - sleep(1); - event_queue_.enqueue(EVENT_SYS_INIT, "", ""); - } + // todo:... } std::string StackFlow::_rpc_setup(pzmq *_pzmq, const std::string &data) From 71fe7f2f0325515ddc685e87f2fcdd7b57b5e44d Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 2 Dec 2024 18:30:02 +0800 Subject: [PATCH 31/44] [update] fix melotts english generate. --- .../main_melotts/src/runner/Lexicon.hpp | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp index 67d8c1ab..8b6255d2 100644 --- a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp +++ b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp @@ -92,9 +92,45 @@ class Lexicon { return words; } + bool is_english(std::string s) { + if (s.size() == 1) + return (s[0] >= 'A' && s[0] <= 'Z') || (s[0] >= 'a' && s[0] <= 'z'); + else + return false; + } + + std::vector merge_english(const std::vector& splitted_text) { + std::vector words; + int i = 0; + while (i < splitted_text.size()) { + std::string s; + if (is_english(splitted_text[i])) { + while (i < splitted_text.size()) { + if (!is_english(splitted_text[i])) { + break; + } + s += splitted_text[i]; + i++; + } + // to lowercase + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c){ return std::tolower(c); }); + words.push_back(s); + if (i >= splitted_text.size()) + break; + } + else { + words.push_back(splitted_text[i]); + i++; + } + } + return words; + } + void convert(const std::string& text, std::vector& phones, std::vector& tones) { auto splitted_text = splitEachChar(text); - for (auto c : splitted_text) { + auto zh_mix_en = merge_english(splitted_text); + for (auto c : zh_mix_en) { std::string s{c}; if (s == "，") s = ","; From ce0d88a6af5f4895083c66ed3258318fb529d97a Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 3 Dec 2024 12:08:32 +0800 Subject: [PATCH 32/44] [update] fix config path. --- ext_components/StackFlow/stackflow/StackFlowUtil.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 89a31409..310edb0b 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -273,8 +273,8 @@ std::list StackFlows::get_config_file_paths(const std::string &base std::list config_file_paths; config_file_paths.push_back(std::string("./") + mode_name + ".json"); config_file_paths.push_back(std::string("./asr_mode_") + mode_name + ".json"); - config_file_paths.push_back((*base_path) + "../share/" + mode_name + ".json"); - config_file_paths.push_back((*base_path) + "../share/asr_mode_" + mode_name + ".json"); + config_file_paths.push_back((*base_path) + "/../share/" + mode_name + ".json"); + config_file_paths.push_back((*base_path) + "/../share/asr_mode_" + mode_name + ".json"); config_file_paths.push_back((*base_path) + mode_name + ".json"); config_file_paths.push_back((*base_path) + "asr_mode_" + mode_name + ".json"); return config_file_paths; From c2bf248f9b33f6524a3e2fcacc7f66a0bac59930 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 3 Dec 2024 15:09:07 +0800 Subject: [PATCH 33/44] [update] pack deb --- .../StackFlow/stackflow/StackFlowUtil.cpp | 36 +- .../StackFlow/stackflow/StackFlowUtil.h | 2 +- projects/llm_framework/main_asr/SConstruct | 3 +- ...n-streaming-zipformer-20M-2023-02-17.json} | 0 ...treaming-zipformer-zh-14M-2023-02-23.json} | 0 projects/llm_framework/main_asr/src/main.cpp | 18 +- projects/llm_framework/main_audio/SConstruct | 1 + .../llm_framework/main_audio/src/main.cpp | 4 +- projects/llm_framework/main_camera/SConstruct | 1 + projects/llm_framework/main_kws/SConstruct | 3 +- ...zipformer-gigaspeech-3.3M-2024-01-01.json} | 0 ...ipformer-wenetspeech-3.3M-2024-01-01.json} | 0 projects/llm_framework/main_kws/src/main.cpp | 18 +- projects/llm_framework/main_llm/SConstruct | 7 +- ...n => mode_llama3.2-1B-prefill-ax630c.json} | 0 ...=> mode_openbuddy-llama3.2-1b-ax630c.json} | 0 ...son => mode_qwen2.5-0.5B-prefill-20e.json} | 0 ...on => mode_qwen2.5-coder-0.5B-ax630c.json} | 0 projects/llm_framework/main_llm/src/main.cpp | 18 +- .../llm_framework/main_melotts/SConstruct | 2 +- ...tts_zh-cn.json => mode_melotts-zh-cn.json} | 0 .../llm_framework/main_melotts/src/main.cpp | 24 +- projects/llm_framework/main_skel/SConstruct | 2 +- projects/llm_framework/main_skel/src/main.c | 1120 ++++++++--------- .../llm_framework/main_sys/src/event_loop.cpp | 6 +- projects/llm_framework/main_sys/src/main.cpp | 1 + .../Kconfig | 0 .../SConstruct | 0 .../_tokenizer.py | 0 .../src/main.cpp | 0 projects/llm_framework/main_tts/SConstruct | 4 +- ... => mode_single-speaker-english-fast.json} | 0 ...ast.json => mode_single-speaker-fast.json} | 0 projects/llm_framework/main_tts/src/main.cpp | 28 +- projects/llm_framework/main_vlm/SConstruct | 5 +- ...30c.json => mode_internvl2-1b-ax630c.json} | 0 projects/llm_framework/main_vlm/src/main.cpp | 18 +- projects/llm_framework/main_yolo/SConstruct | 2 +- ...lo11n-pose.json => mode_yolo11n-pose.json} | 0 ...yolo11n-seg.json => mode_yolo11n-seg.json} | 0 .../{yolo11n.json => mode_yolo11n.json} | 0 projects/llm_framework/main_yolo/src/main.cpp | 18 +- projects/llm_framework/tools/llm_pack.py | 26 +- 43 files changed, 639 insertions(+), 728 deletions(-) rename projects/llm_framework/main_asr/{sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json => mode_sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json} (100%) rename projects/llm_framework/main_asr/{sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json => mode_sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json} (100%) rename projects/llm_framework/main_kws/{sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json => mode_sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json} (100%) rename projects/llm_framework/main_kws/{sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json => mode_sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json} (100%) rename projects/llm_framework/main_llm/{llama3.2-1B-prefill-ax630c.json => mode_llama3.2-1B-prefill-ax630c.json} (100%) rename projects/llm_framework/main_llm/{openbuddy-llama3.2-1b-ax630c.json => mode_openbuddy-llama3.2-1b-ax630c.json} (100%) rename projects/llm_framework/main_llm/{qwen2.5-0.5B-prefill-20e.json => mode_qwen2.5-0.5B-prefill-20e.json} (100%) rename projects/llm_framework/main_llm/{qwen2.5-coder-0.5B-ax630c.json => mode_qwen2.5-coder-0.5B-ax630c.json} (100%) rename projects/llm_framework/main_melotts/{melotts_zh-cn.json => mode_melotts-zh-cn.json} (100%) rename projects/llm_framework/{main_llm_tokenizer => main_tokenizer}/Kconfig (100%) rename projects/llm_framework/{main_llm_tokenizer => main_tokenizer}/SConstruct (100%) rename projects/llm_framework/{main_llm_tokenizer => main_tokenizer}/_tokenizer.py (100%) rename projects/llm_framework/{main_llm_tokenizer => main_tokenizer}/src/main.cpp (100%) rename projects/llm_framework/main_tts/{single_speaker_english_fast.json => mode_single-speaker-english-fast.json} (100%) rename projects/llm_framework/main_tts/{single_speaker_fast.json => mode_single-speaker-fast.json} (100%) rename projects/llm_framework/main_vlm/{internvl2-1b-ax630c.json => mode_internvl2-1b-ax630c.json} (100%) rename projects/llm_framework/main_yolo/{yolo11n-pose.json => mode_yolo11n-pose.json} (100%) rename projects/llm_framework/main_yolo/{yolo11n-seg.json => mode_yolo11n-seg.json} (100%) rename projects/llm_framework/main_yolo/{yolo11n.json => mode_yolo11n.json} (100%) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 89a31409..e7bbb44a 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -265,17 +265,37 @@ std::string StackFlows::unit_call(const std::string &unit_name, const std::strin return value; } -std::list StackFlows::get_config_file_paths(const std::string &base_model_path, +std::list StackFlows::get_config_file_paths(std::string &base_model_path, + std::string &base_model_config_path, const std::string &mode_name) { - std::shared_ptr base_path((std::string *)(&base_model_path), [](std::string *Npt) {}); - if (base_path->empty()) base_path = std::make_shared("/opt/m5stack/data/"); + if (base_model_path.empty()) { + base_model_path = sample_unescapeString(unit_call("sys", "sql_select", "config_base_mode_path")); + if (base_model_path.empty()) { + base_model_path = "/opt/m5stack/data/"; + } + } + if (base_model_config_path.empty()) { + base_model_config_path = sample_unescapeString(unit_call("sys", "sql_select", "config_base_mode_config_path")); + if (base_model_config_path.empty()) { + base_model_config_path = "/opt/m5stack/etc/"; + } + } + std::string config_model_d = sample_unescapeString(unit_call("sys", "sql_select", "config_model_d")); + if (config_model_d.empty()) config_model_d = "/opt/m5stack/data/models/"; + std::list config_file_paths; config_file_paths.push_back(std::string("./") + mode_name + ".json"); - config_file_paths.push_back(std::string("./asr_mode_") + mode_name + ".json"); - config_file_paths.push_back((*base_path) + "../share/" + mode_name + ".json"); - config_file_paths.push_back((*base_path) + "../share/asr_mode_" + mode_name + ".json"); - config_file_paths.push_back((*base_path) + mode_name + ".json"); - config_file_paths.push_back((*base_path) + "asr_mode_" + mode_name + ".json"); + config_file_paths.push_back(std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + mode_name + std::string("/") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + mode_name + std::string("/") + std::string("./mode_") + mode_name + + ".json"); + config_file_paths.push_back(config_model_d + mode_name + ".json"); + config_file_paths.push_back(config_model_d + std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_config_path + mode_name + ".json"); + config_file_paths.push_back(base_model_config_path + std::string("./mode_") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + std::string("../share/") + mode_name + ".json"); + config_file_paths.push_back(base_model_path + std::string("../share/") + std::string("./mode_") + mode_name + + ".json"); return config_file_paths; } \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h index d4713f33..d1f6f8a7 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.h +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -32,5 +32,5 @@ bool decode_stream(const std::string &in, std::string &out, std::unordered_map get_config_file_paths(const std::string &base_model_path, const std::string &mode_name); +std::list get_config_file_paths(std::string &base_model_path, std::string &base_model_config_path, const std::string &mode_name); }; // namespace StackFlows diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index 9a84dddd..c947a9c5 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -24,8 +24,7 @@ INCLUDE += [ADir('../include/sherpa'), ADir('../include/sherpa/sherpa-ncnn')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')] REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] -STATIC_FILES += [AFile('sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json'), AFile('sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json')] - +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_asr', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json b/projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json similarity index 100% rename from projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json rename to projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-20M-2023-02-17.json diff --git a/projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json b/projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json similarity index 100% rename from projects/llm_framework/main_asr/sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json rename to projects/llm_framework/main_asr/mode_sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.json diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index f4c8f9d8..319bcf7f 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -92,7 +92,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -265,27 +265,11 @@ class llm_asr : public StackFlow { int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_asr::_load_config success"); - return 0; - } - } public: llm_asr() : StackFlow("asr") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_asr::_load_config, this)); event_queue_.appendListener( EVENT_TASK_PAUSE, std::bind(&llm_asr::_task_pause, this, std::placeholders::_1, std::placeholders::_2)); } diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 8e6721e6..afbad26d 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -27,6 +27,7 @@ REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_audio', 'ax_audio_3a', 'ax_fdk' REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] STATIC_FILES += [AFile('audio.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_audio', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index cb38547d..36e6c2cc 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -139,7 +139,9 @@ class llm_audio : public StackFlow { nlohmann::json config_body; nlohmann::json file_body; nlohmann::json error_body; - std::list config_file_paths = get_config_file_paths("/opt/m5stack/data", "audio"); + std::string base_model_path; + std::string base_model_config_path; + std::list config_file_paths = get_config_file_paths(base_model_path, base_model_config_path, "audio"); try { config_body = nlohmann::json::parse(data); for (auto file_name : config_file_paths) { diff --git a/projects/llm_framework/main_camera/SConstruct b/projects/llm_framework/main_camera/SConstruct index 28bc9381..8de268d2 100644 --- a/projects/llm_framework/main_camera/SConstruct +++ b/projects/llm_framework/main_camera/SConstruct @@ -43,6 +43,7 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] STATIC_LIB += static_file * 4 STATIC_FILES += [AFile('camera.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_camera', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 95d3dafa..62910304 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -31,8 +31,7 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] -STATIC_FILES += [AFile('sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json'), - AFile('sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_kws', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json b/projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json similarity index 100% rename from projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json rename to projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.json diff --git a/projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json b/projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json similarity index 100% rename from projects/llm_framework/main_kws/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json rename to projects/llm_framework/main_kws/mode_sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.json diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 17713512..c4c8f207 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -98,7 +98,7 @@ class llm_task { } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -263,27 +263,11 @@ class llm_kws : public StackFlow { int task_count_; std::string audio_url_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_kws::_load_config success"); - return 0; - } - } public: llm_kws() : StackFlow("kws") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_kws::_load_config, this)); } void play_awake_wav(const std::string &wav_file) diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index c95c1458..eca59a3e 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -34,14 +34,11 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib/module-llm/libsentencepiece.a'), AFile('../static_lib/module-llm/libsentencepiece_train.a')] STATIC_LIB += static_file * 4 -STATIC_FILES += [AFile('qwen2.5-0.5B-prefill-20e.json'), - AFile('llama3.2-1B-prefill-ax630c.json'), - AFile('openbuddy-llama3.2-1b-ax630c.json'), - AFile('qwen2.5-coder-0.5B-ax630c.json'), - AFile('llama3.2-1B-prefill-ax630c_tokenizer.py'), +STATIC_FILES += [AFile('llama3.2-1B-prefill-ax630c_tokenizer.py'), AFile('openbuddy-llama3.2-1b-ax630c_tokenizer.py'), AFile('qwen2.5-coder-0.5B-ax630c_tokenizer.py') ] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c.json b/projects/llm_framework/main_llm/mode_llama3.2-1B-prefill-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c.json rename to projects/llm_framework/main_llm/mode_llama3.2-1B-prefill-ax630c.json diff --git a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c.json b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1b-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c.json rename to projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1b-ax630c.json diff --git a/projects/llm_framework/main_llm/qwen2.5-0.5B-prefill-20e.json b/projects/llm_framework/main_llm/mode_qwen2.5-0.5B-prefill-20e.json similarity index 100% rename from projects/llm_framework/main_llm/qwen2.5-0.5B-prefill-20e.json rename to projects/llm_framework/main_llm/mode_qwen2.5-0.5B-prefill-20e.json diff --git a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c.json b/projects/llm_framework/main_llm/mode_qwen2.5-coder-0.5B-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c.json rename to projects/llm_framework/main_llm/mode_qwen2.5-coder-0.5B-ax630c.json diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 4540280b..e4f478d8 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -85,7 +85,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -230,27 +230,11 @@ class llm_llm : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_llm::_load_config success"); - return 0; - } - } public: llm_llm() : StackFlow("llm") { task_count_ = 2; - repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index 6124c3e7..e8c09a6b 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -26,7 +26,7 @@ REQUIREMENTS += ['onnxruntime', 'samplerate'] INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] -STATIC_FILES += [AFile('melotts_zh-cn.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_melotts', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_melotts/melotts_zh-cn.json b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json similarity index 100% rename from projects/llm_framework/main_melotts/melotts_zh-cn.json rename to projects/llm_framework/main_melotts/mode_melotts-zh-cn.json diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 266fd4ce..584d229e 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -127,7 +127,11 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + // Compatible operation + if (model_ == "melotts_zh-cn") + config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, "melotts-zh-cn"); + try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -347,27 +351,11 @@ class llm_tts : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_melotts::_load_config success"); - return 0; - } - } public: llm_tts() : StackFlow("melotts") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, @@ -647,7 +635,7 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("melotts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_skel/SConstruct b/projects/llm_framework/main_skel/SConstruct index b6aeb01a..f40e42d3 100644 --- a/projects/llm_framework/main_skel/SConstruct +++ b/projects/llm_framework/main_skel/SConstruct @@ -29,7 +29,7 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 - +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_skel', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_skel/src/main.c b/projects/llm_framework/main_skel/src/main.c index b1e9364a..144d3a46 100644 --- a/projects/llm_framework/main_skel/src/main.c +++ b/projects/llm_framework/main_skel/src/main.c @@ -45,24 +45,26 @@ #define AX_MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif -#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" +#define SKEL_SAMPLE_OUTPUT_BODY_PATH "body" #define SKEL_SAMPLE_OUTPUT_VEHICLE_PATH "vehicle" -#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" -#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" -#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" -#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" +#define SKEL_SAMPLE_OUTPUT_CYCLE_PATH "cycle" +#define SKEL_SAMPLE_OUTPUT_FACE_PATH "face" +#define SKEL_SAMPLE_OUTPUT_PLATE_PATH "plate" +#define SKEL_SAMPLE_OUTPUT_LOG_FILE "output.txt" #define SAMPLE_SKEL_SHIFT_LEFT_ALIGN(a) (1 << (a)) -#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) -#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) +#define SAMPLE_SKEL_VDEC_WIDTH_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(8) +#define SAMPLE_SKEL_HEIGHT_ALIGN SAMPLE_SKEL_SHIFT_LEFT_ALIGN(6) -#define RUN_COMMAND(format, ...) do { \ - char cmd[512]; \ - snprintf(cmd, 512, format, ##__VA_ARGS__); \ - system(cmd); \ - } while(0); +#define RUN_COMMAND(format, ...) \ + do { \ + char cmd[512]; \ + snprintf(cmd, 512, format, ##__VA_ARGS__); \ + system(cmd); \ + } while (0); -AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) { +AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) +{ if (file) { va_list args; char szLog[1024] = {0}; @@ -82,8 +84,8 @@ AX_VOID LogSaveToFile(FILE *file, const char *fmt, ...) { } } #define OUTPUT_LOG_SAVE(fmt, ...) LogSaveToFile(fpResultFile, fmt "\n", ##__VA_ARGS__) -#define BLACK_VIDEO_FRAME_COUNT 60 -#define SKEL_SAMPLE_OBJECT_SIZE 512 +#define BLACK_VIDEO_FRAME_COUNT 60 +#define SKEL_SAMPLE_OBJECT_SIZE 512 struct skeleton { int connection[2]; @@ -112,7 +114,8 @@ typedef struct _AI_Detection_SkelResult_t { AI_Detection_Box_t tBox; } AI_Detection_SkelResult_t; -static AX_U64 get_tick_count(AX_VOID) { +static AX_U64 get_tick_count(AX_VOID) +{ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); @@ -124,7 +127,8 @@ static AX_U64 get_tick_count(AX_VOID) { // return (ts.tv_sec * 1000000 + ts.tv_nsec / 1000); // } -static AX_VOID ShowUsage(AX_VOID) { +static AX_VOID ShowUsage(AX_VOID) +{ // printf("usage: ./%s ...\n", SAMPLE_SKEL_NAME); printf("options:\n"); printf("-i, \tInput File(yuv)\n"); @@ -138,35 +142,41 @@ static AX_VOID ShowUsage(AX_VOID) { printf("-H, \tHuman track size limit((unsigned int), default=0)\n"); printf("-V, \tVehicle track size limit((unsigned int), default=0)\n"); printf("-C, \tCylcle track size limit((unsigned int), default=0)\n"); - printf("-d, \tSkel detect type((unsigned int), default=2)\n" - "\t\t0: detect only\n" - "\t\t1: detect + track\n" - "\t\t2: detect + track + push\n"); - printf("-u, \tSkel push strategy((unsigned int), default=3)\n" - "\t\t1: fast push strategy\n" - "\t\t2: push strategy\n" - "\t\t3: best push strategy\n"); - printf("-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" - "\t\t0: VNPU Disable\n" - "\t\t1: STD-VNPU Default\n" - "\t\t2: STD-VNPU1\n" - "\t\t3: STD-VNPU2\n"); - printf("-p, \tSkel PPL((unsigned int), default=1)\n" - "\t\t1: AX_SKEL_PPL_HVCP\n" - "\t\t2: AX_SKEL_PPL_FACE\n"); - printf("-v, \tLog level((unsigned int), default=5)\n" - "\t\t0: LOG_EMERGENCY_LEVEL\n" - "\t\t1: LOG_ALERT_LEVEL\n" - "\t\t2: LOG_CRITICAL_LEVEL\n" - "\t\t3: LOG_ERROR_LEVEL\n" - "\t\t4: LOG_WARN_LEVEL\n" - "\t\t5: LOG_NOTICE_LEVEL\n" - "\t\t6: LOG_INFO_LEVEL\n" - "\t\t7: LOG_DEBUG_LEVEL\n"); + printf( + "-d, \tSkel detect type((unsigned int), default=2)\n" + "\t\t0: detect only\n" + "\t\t1: detect + track\n" + "\t\t2: detect + track + push\n"); + printf( + "-u, \tSkel push strategy((unsigned int), default=3)\n" + "\t\t1: fast push strategy\n" + "\t\t2: push strategy\n" + "\t\t3: best push strategy\n"); + printf( + "-N, \tSkel NPU type((unsigned int), default=0(VNPU Disable)\n" + "\t\t0: VNPU Disable\n" + "\t\t1: STD-VNPU Default\n" + "\t\t2: STD-VNPU1\n" + "\t\t3: STD-VNPU2\n"); + printf( + "-p, \tSkel PPL((unsigned int), default=1)\n" + "\t\t1: AX_SKEL_PPL_HVCP\n" + "\t\t2: AX_SKEL_PPL_FACE\n"); + printf( + "-v, \tLog level((unsigned int), default=5)\n" + "\t\t0: LOG_EMERGENCY_LEVEL\n" + "\t\t1: LOG_ALERT_LEVEL\n" + "\t\t2: LOG_CRITICAL_LEVEL\n" + "\t\t3: LOG_ERROR_LEVEL\n" + "\t\t4: LOG_WARN_LEVEL\n" + "\t\t5: LOG_NOTICE_LEVEL\n" + "\t\t6: LOG_INFO_LEVEL\n" + "\t\t7: LOG_DEBUG_LEVEL\n"); printf("-h, \tprint this message\n"); } -AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { +AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) +{ if (pstConfig->nSize > 0 && pstConfig->pstItems) { for (size_t i = 0; i < pstConfig->nSize; i++) { if (pstConfig->pstItems[i].pstrType && pstConfig->pstItems[i].pstrValue) { @@ -177,7 +187,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "vehicle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -187,7 +198,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "cycle_max_target_count", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -197,7 +209,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %d", pstConfig->pstItems[i].pstrType, (AX_U8)pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "body_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -207,7 +220,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "face_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -217,7 +231,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "vehicle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -227,7 +242,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "cycle_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -237,7 +253,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "plate_confidence", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -247,7 +264,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "crop_encoder_qpLevel", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * @@ -257,7 +275,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s: %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "body_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * @@ -267,7 +286,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "face_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * @@ -277,7 +297,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "vehicle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * @@ -287,7 +308,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "cycle_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * @@ -297,7 +319,8 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "plate_min_size", value_type: AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T * @@ -307,166 +330,201 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { (AX_SKEL_OBJECT_SIZE_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s %dx%d", pstConfig->pstItems[i].pstrType, pstConf->nWidth, pstConf->nHeight); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } - // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * + // cmd: "detect_roi_polygon", value_type: AX_SKEL_ROI_POLYGON_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "detect_roi_polygon") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T)) { - AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, pstConf->nPointNum); + AX_SKEL_ROI_POLYGON_CONFIG_T *pstConf = + (AX_SKEL_ROI_POLYGON_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%d]: nPointNum[%d]", pstConfig->pstItems[i].pstrType, pstConf->bEnable, + pstConf->nPointNum); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_strategy", value_type: AX_SKEL_PUSH_STRATEGY_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_strategy") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_STRATEGY_T)) { AX_SKEL_PUSH_STRATEGY_T *pstConf = (AX_SKEL_PUSH_STRATEGY_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, pstConf->ePushMode, pstConf->nIntervalTimes, - pstConf->nPushCounts, pstConf->bPushSameFrame); + ALOGI("SKEL get %s [mode:%d, times:%d, count:%d, same:%d]", pstConfig->pstItems[i].pstrType, + pstConf->ePushMode, pstConf->nIntervalTimes, pstConf->nPushCounts, + pstConf->bPushSameFrame); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "body_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "body_crop_encoder") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { - AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, - pstConf->fScaleLeft, pstConf->fScaleRight, - pstConf->fScaleTop, pstConf->fScaleBottom); + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "vehicle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "vehicle_crop_encoder") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { - AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, - pstConf->fScaleLeft, pstConf->fScaleRight, - pstConf->fScaleTop, pstConf->fScaleBottom); + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "cycle_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "cycle_crop_encoder") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { - AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, - pstConf->fScaleLeft, pstConf->fScaleRight, - pstConf->fScaleTop, pstConf->fScaleBottom); + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "face_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "face_crop_encoder") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { - AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, - pstConf->fScaleLeft, pstConf->fScaleRight, - pstConf->fScaleTop, pstConf->fScaleBottom); + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "plate_crop_encoder", value_type: AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "plate_crop_encoder") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T)) { - AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, - pstConf->fScaleLeft, pstConf->fScaleRight, - pstConf->fScaleTop, pstConf->fScaleBottom); + AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [%f, %f, %f, %f]", pstConfig->pstItems[i].pstrType, pstConf->fScaleLeft, + pstConf->fScaleRight, pstConf->fScaleTop, pstConf->fScaleBottom); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_panorama", value_type: AX_SKEL_PUSH_PANORAMA_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_panorama") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T)) { - AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_PUSH_PANORAMA_CONFIG_T *pstConf = + (AX_SKEL_PUSH_PANORAMA_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s [Enable: %d, Quality: %d]", pstConfig->pstItems[i].pstrType, - pstConf->bEnable, pstConf->nQuality); + pstConf->bEnable, pstConf->nQuality); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_quality_body", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_body") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { - AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_quality_vehicle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_vehicle") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { - AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_quality_cycle", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_cycle") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { - AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_quality_face", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_face") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { - AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s [W: %d, H: %d, P: %f, Y: %f, R: %f, B: %f]", pstConfig->pstItems[i].pstrType, - pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, - pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, - pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); + pstConf->stFaceAttrFilterConfig.nWidth, pstConf->stFaceAttrFilterConfig.nHeight, + pstConf->stFaceAttrFilterConfig.stPoseblur.fPitch, + pstConf->stFaceAttrFilterConfig.stPoseblur.fYaw, + pstConf->stFaceAttrFilterConfig.stPoseblur.fRoll, + pstConf->stFaceAttrFilterConfig.stPoseblur.fBlur); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_quality_plate", value_type: AX_SKEL_ATTR_FILTER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_quality_plate") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T)) { - AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; - ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, pstConf->stCommonAttrFilterConfig.fQuality); + AX_SKEL_ATTR_FILTER_CONFIG_T *pstConf = + (AX_SKEL_ATTR_FILTER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + ALOGI("SKEL get %s [Q: %f]", pstConfig->pstItems[i].pstrType, + pstConf->stCommonAttrFilterConfig.fQuality); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_bind_enable", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_bind_enable") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { - AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s : %f", pstConfig->pstItems[i].pstrType, pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } - // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * + // cmd: "track_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * else if (strcmp(pstConfig->pstItems[i].pstrType, "track_enable") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { - AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_enable", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_enable") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { - AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL get %s : %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "target_config", value_type: AX_SKEL_TARGET_CONFIG_T * @@ -474,17 +532,18 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; if (pstConf->pstItems) { - for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { if (pstConf->pstItems[j].pstrObjectCategory) { - ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, + pstConf->pstItems[j].pstrObjectCategory); } } - } - else { + } else { ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); } } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_target_config", value_type: AX_SKEL_TARGET_CONFIG_S * @@ -492,54 +551,59 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_TARGET_CONFIG_T)) { AX_SKEL_TARGET_CONFIG_T *pstConf = (AX_SKEL_TARGET_CONFIG_T *)pstConfig->pstItems[i].pstrValue; if (pstConf->pstItems) { - for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { if (pstConf->pstItems[j].pstrObjectCategory) { - ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, pstConf->pstItems[j].pstrObjectCategory); + ALOGI("SKEL set %s: [%d]%s", pstConfig->pstItems[i].pstrType, j, + pstConf->pstItems[j].pstrObjectCategory); } } - } - else { + } else { ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); } } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "analyzer_attr_config", value_type: AX_SKEL_ANALYZER_CONFIG_T * else if (strcmp(pstConfig->pstItems[i].pstrType, "analyzer_attr_config") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_ANALYZER_CONFIG_T)) { - AX_SKEL_ANALYZER_CONFIG_T *pstConf = (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_ANALYZER_CONFIG_T *pstConf = + (AX_SKEL_ANALYZER_CONFIG_T *)pstConfig->pstItems[i].pstrValue; if (pstConf->peItems) { - for (AX_U32 j = 0; j < pstConf->nSize; j ++) { + for (AX_U32 j = 0; j < pstConf->nSize; j++) { ALOGI("SKEL set %s: [%d]%d", pstConfig->pstItems[i].pstrType, j, pstConf->peItems[j]); } - } - else { + } else { ALOGE("SKEL cmd: %s invalid param", pstConfig->pstItems[i].pstrType); } } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "venc_attr_config", value_type: AX_SKEL_COMMON_THRESHOLD_CONFIG_S * else if (strcmp(pstConfig->pstItems[i].pstrType, "venc_attr_config") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T)) { - AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T *pstConf = + (AX_SKEL_COMMON_THRESHOLD_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, (AX_U32)pstConf->fValue); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } } // cmd: "push_attr_always", value_type: AX_SKEL_COMMON_ENABLE_CONFIG_S * else if (strcmp(pstConfig->pstItems[i].pstrType, "push_attr_always") == 0) { if (pstConfig->pstItems[i].nValueSize == sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T)) { - AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; + AX_SKEL_COMMON_ENABLE_CONFIG_T *pstConf = + (AX_SKEL_COMMON_ENABLE_CONFIG_T *)pstConfig->pstItems[i].pstrValue; ALOGI("SKEL set %s: %d", pstConfig->pstItems[i].pstrType, pstConf->bEnable); } else { - ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, pstConfig->pstItems[i].nValueSize); + ALOGE("SKEL %s size(%d) no match", pstConfig->pstItems[i].pstrType, + pstConfig->pstItems[i].nValueSize); } - } - else { + } else { ALOGE("SKEL cmd: %s not support", pstConfig->pstItems[i].pstrType); } } @@ -549,7 +613,7 @@ AX_S32 ParseConfigParam(const AX_SKEL_CONFIG_T *pstConfig) { return 0; } -FILE* LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) +FILE *LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) { /* Reading input file */ FILE *f_in = fopen(pFile, "rb"); @@ -567,7 +631,7 @@ FILE* LoadFile(const AX_CHAR *pFile, AX_U64 *pLen) return f_in; } -int ReLoadFile(FILE* pFile) +int ReLoadFile(FILE *pFile) { if (pFile) { rewind(pFile); @@ -577,17 +641,9 @@ int ReLoadFile(FILE* pFile) return -1; } -int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, - AX_VOID *pSrcVirAddr, - AX_U32 nSrcSize, - AX_U32 nSrcStride, - AX_U32 nSrcHeight, - AX_U64 nDstPhyAddr, - AX_VOID *pDstVirAddr, - AX_U32 nDstSize, - AX_U32 nDstStride, - AX_U32 nDstHeight - ) { +int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, AX_VOID *pSrcVirAddr, AX_U32 nSrcSize, AX_U32 nSrcStride, AX_U32 nSrcHeight, + AX_U64 nDstPhyAddr, AX_VOID *pDstVirAddr, AX_U32 nDstSize, AX_U32 nDstStride, AX_U32 nDstHeight) +{ AX_U64 nStartTime = get_tick_count(); // src check @@ -595,7 +651,7 @@ int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, return -1; } - if (nSrcSize != nSrcStride * nSrcHeight * 3 /2) { + if (nSrcSize != nSrcStride * nSrcHeight * 3 / 2) { return -1; } @@ -604,7 +660,7 @@ int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, return -1; } - if (nDstSize != nDstStride * nDstHeight * 3 /2) { + if (nDstSize != nDstStride * nDstHeight * 3 / 2) { return -1; } @@ -622,7 +678,7 @@ int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, AX_VOID *src = pSrcVirAddr; AX_VOID *dst = pDstVirAddr; - for(AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { + for (AX_U32 i = 0; i < nSrcHeight * 3 / 2; i++) { memcpy(dst, src, nSrcStride); src += nSrcStride; dst += nDstStride; @@ -630,7 +686,8 @@ int NV12ToStrideNV12(AX_U64 nSrcPhyAddr, AX_U64 nProcessElasped = get_tick_count() - nStartTime; - ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, nProcessElasped); + ALOGD("NV12ToStrideNV12 %dx%d=>%dx%d elapse: %lld ms", nSrcStride, nSrcHeight, nDstStride, nDstHeight, + nProcessElasped); return 0; } @@ -643,8 +700,7 @@ int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) if (nPhyAddr != 0 && pVirAddr) { memset((AX_U8 *)pVirAddr, 0x00, nSize); nReadSize = fread((AX_U8 *)pVirAddr, 1, nSize, pFile); - } - else { + } else { fseeko(pFile, nSize, SEEK_CUR); } } @@ -652,21 +708,21 @@ int LoadFileToMem(FILE *pFile, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_S32 nSize) return nReadSize; } -int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) +int DecodeJpeg(FILE *InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID *OneYUVDataVir, AX_U32 nWidth, AX_U32 nHeight) { SAMPLE_INPUT_FILE_INFO_T stStreamInfo; SAMPLE_STREAM_BUF_T stStreamBuf; - size_t nReadLen = 0; - AX_U64 streamPhyAddr = 0; + size_t nReadLen = 0; + AX_U64 streamPhyAddr = 0; AX_VOID *pStreamVirAddr = NULL; AX_VDEC_DEC_ONE_FRM_T decOneFrmParam; - size_t nFileSize = 0; + size_t nFileSize = 0; AX_U32 heightAlign = 0; - AX_U32 frmStride = 0; - frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; - heightAlign = ALIGN_UP(nHeight, 16); - AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; - + AX_U32 frmStride = 0; + frmStride = AX_COMM_ALIGN(nWidth * 8, AX_VDEC_WIDTH_ALIGN * 8) / 8; + heightAlign = ALIGN_UP(nHeight, 16); + AX_U32 nFrameSize = heightAlign * frmStride * 3 / 2; + memset(&stStreamInfo, 0, sizeof(SAMPLE_INPUT_FILE_INFO_T)); memset(&stStreamBuf, 0, sizeof(SAMPLE_STREAM_BUF_T)); memset(&decOneFrmParam, 0, sizeof(AX_VDEC_DEC_ONE_FRM_T)); @@ -675,19 +731,18 @@ int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataV fseek(InputFileHandle, 0, SEEK_END); nFileSize = (size_t)ftello(InputFileHandle); - stStreamInfo.fInput = InputFileHandle; + stStreamInfo.fInput = InputFileHandle; stStreamInfo.sFileSize = nFileSize; stStreamBuf.uBufSize = ((AX_U32)nFileSize) > STREAM_BUFFER_MAX_SIZE ? STREAM_BUFFER_MAX_SIZE : nFileSize; - int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, - stStreamBuf.uBufSize, 0x100, (AX_S8 *)"vdec_input_stream"); + int ret = AX_SYS_MemAlloc(&streamPhyAddr, (AX_VOID **)&pStreamVirAddr, stStreamBuf.uBufSize, 0x100, + (AX_S8 *)"vdec_input_stream"); if (ret != AX_SUCCESS) { - ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", - stStreamBuf.uBufSize, ret); + ALOGE("AX_SYS_MemAlloc FAILED! uBufSize:0x%x ret:0x%x\n", stStreamBuf.uBufSize, ret); return -1; } - stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; + stStreamBuf.tBufAddr.pVirAddr = pStreamVirAddr; stStreamBuf.tBufAddr.u64PhyAddr = streamPhyAddr; ret = StreamParserReadFrameJpeg(&stStreamInfo, &stStreamBuf, &nReadLen); @@ -704,8 +759,8 @@ int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataV return ret; } - decOneFrmParam.stStream.pu8Addr = (AX_U8*)stStreamBuf.tBufAddr.pVirAddr; - decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; + decOneFrmParam.stStream.pu8Addr = (AX_U8 *)stStreamBuf.tBufAddr.pVirAddr; + decOneFrmParam.stStream.u64PhyAddr = stStreamBuf.tBufAddr.u64PhyAddr; decOneFrmParam.stStream.u32StreamPackLen = (AX_U32)nReadLen; decOneFrmParam.stFrame.u64VirAddr[0] = (AX_ULONG)OneYUVDataVir; @@ -716,15 +771,14 @@ int DecodeJpeg(FILE* InputFileHandle, AX_U64 OneYUVDataPhy, AX_VOID* OneYUVDataV ret = AX_VDEC_JpegDecodeOneFrame(&decOneFrmParam); if (ret != AX_SUCCESS) { - ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", - ret, SampleVdecRetStr(ret)); + ALOGE("AX_VDEC_JpegDecodeOneFrame FAILED! ret:0x%x %s\n", ret, SampleVdecRetStr(ret)); AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); return ret; } AX_SYS_MemFree(streamPhyAddr, pStreamVirAddr); - return 0; + return 0; } int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr, AX_U32 *pLen) @@ -742,7 +796,7 @@ int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr rewind(f_in); if (pPhyAddr && ppVirAddr) { - AX_U64 nPhyAddr = 0; + AX_U64 nPhyAddr = 0; AX_VOID *pVirAddr = NULL; // pVirAddr = (AX_VOID *)malloc(nFileSize); @@ -756,7 +810,7 @@ int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr fread((AX_U8 *)pVirAddr, nFileSize, 1, f_in); - *pPhyAddr = nPhyAddr; + *pPhyAddr = nPhyAddr; *ppVirAddr = pVirAddr; } @@ -769,14 +823,16 @@ int LoadFileToMemExt(const AX_CHAR *pFile, AX_U64 *pPhyAddr, AX_VOID **ppVirAddr return 0; } -int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, AX_VOID *pVirAddr, AX_U32 nLen) { +int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, AX_U32 nHeight, AX_U64 nPhyAddr, + AX_VOID *pVirAddr, AX_U32 nLen) +{ AX_S32 s32Ret = AX_SUCCESS; AX_U32 frameSize = nLen; AX_JPEG_ENCODE_ONCE_PARAMS_T stJpegEncodeOnceParam; memset(&stJpegEncodeOnceParam, 0, sizeof(stJpegEncodeOnceParam)); - AX_U64 outPhyAddr = 0; + AX_U64 outPhyAddr = 0; AX_VOID *outVirAddr = NULL; s32Ret = AX_SYS_MemAlloc(&outPhyAddr, &outVirAddr, frameSize, 128, (AX_S8 *)"SKEL-VENC"); @@ -787,24 +843,26 @@ int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, // output stJpegEncodeOnceParam.u32OutBufSize = frameSize; - stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; - stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; + stJpegEncodeOnceParam.ulPhyAddr = outPhyAddr; + stJpegEncodeOnceParam.pu8Addr = (AX_U8 *)outVirAddr; // input stJpegEncodeOnceParam.stJpegParam.u32Qfactor = 90; - stJpegEncodeOnceParam.u32Width = nWidth; - stJpegEncodeOnceParam.u32Height = nHeight; - stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; - stJpegEncodeOnceParam.u32PicStride[0] = nStride; - stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; - stJpegEncodeOnceParam.u32PicStride[2] = 0; + stJpegEncodeOnceParam.u32Width = nWidth; + stJpegEncodeOnceParam.u32Height = nHeight; + stJpegEncodeOnceParam.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stJpegEncodeOnceParam.u32PicStride[0] = nStride; + stJpegEncodeOnceParam.u32PicStride[1] = stJpegEncodeOnceParam.u32PicStride[0]; + stJpegEncodeOnceParam.u32PicStride[2] = 0; stJpegEncodeOnceParam.u64PhyAddr[0] = nPhyAddr; - stJpegEncodeOnceParam.u64PhyAddr[1] = stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64PhyAddr[1] = + stJpegEncodeOnceParam.u64PhyAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; stJpegEncodeOnceParam.u64PhyAddr[2] = 0; stJpegEncodeOnceParam.u64VirAddr[0] = (AX_ULONG)pVirAddr; - stJpegEncodeOnceParam.u64VirAddr[1] = stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; + stJpegEncodeOnceParam.u64VirAddr[1] = + stJpegEncodeOnceParam.u64VirAddr[0] + stJpegEncodeOnceParam.u32PicStride[0] * stJpegEncodeOnceParam.u32Height; stJpegEncodeOnceParam.u64VirAddr[2] = 0; s32Ret = AX_VENC_JpegEncodeOneFrame(&stJpegEncodeOnceParam); @@ -832,7 +890,8 @@ int EncodeOneFrameToJpeg(const AX_CHAR *dstFile, AX_U32 nStride, AX_U32 nWidth, return s32Ret; } -AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) { +AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameSeqNum) +{ if (nFrameSeqNum < 1) { nFrameSeqNum = 1; } @@ -852,54 +911,59 @@ AX_BOOL FrameSkipCtrl(AX_S32 nSrcFrameRate, AX_S32 nDstFrameRate, AX_S32 nFrameS } } -AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { +AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) +{ + while (argc == 1) { + sleep(10); + } + AX_S32 nRet = 0; AX_S32 c; - AX_S32 isExit = 0; - AX_BOOL bJpgFile = AX_FALSE; - FILE *InputFileHandle = NULL; - const AX_CHAR *InputFile = NULL; + AX_S32 isExit = 0; + AX_BOOL bJpgFile = AX_FALSE; + FILE *InputFileHandle = NULL; + const AX_CHAR *InputFile = NULL; const AX_CHAR *InputResolution = NULL; - AX_S32 InputFileFrameCnt = 1; - const AX_CHAR *ReWritePath = NULL; - const AX_CHAR *SaveResultPath = NULL; - const AX_CHAR *ModelsPath = NULL; - FILE *fpResultFile = NULL; - AX_S32 nRepeatTimes = 1; - AX_S32 nPPL = AX_SKEL_PPL_HVCP; - AX_S32 nDetectType = 2; - AX_S32 nPushStrategy = 3; - AX_S32 nNpuType = 0; - AX_S32 nInterval = 0; - AX_U32 nStride = 1920; - AX_U32 nWidth = 1920; - AX_U32 nHeight = 1080; - AX_U32 nJencBufSize = 0; - AX_U32 nFrameDepth = 1; - AX_U32 nOneSize = 0; - AX_U64 OneYUVDataPhy = 0; - AX_VOID *OneYUVDataVir = NULL; - AX_U32 nFrameSize = nStride * nHeight * 3 / 2; - AX_U32 nCacheListDepth = 1; - AX_S32 nSrcFrameRate = 25; - AX_S32 nDstFrameRate = 25; - AX_VOID *YUVDataVir = NULL; - AX_U64 YUVDataPhy = 0; - AX_VOID *YUVDataWrVir = NULL; - AX_U64 YUVDataWrPhy = 0; - AX_SKEL_HANDLE pHandle = NULL; - AX_SKEL_FRAME_T stFrame = {0}; - AX_SKEL_RESULT_T *pstResult = NULL; - AX_U64 nFileSize = 0; - AX_U64 nStartTime = 0; - AX_U64 nInitElasped = 0; - AX_U64 nCreateElasped = 0; - AX_U64 nProcessElasped = 0; - AX_U64 nResultElasped = 0; - AX_U64 nResultElaspedMin = (AX_U64)-1; - AX_U64 nResultElaspedMax = 0; - AX_U64 nResultElaspedTotal = 0; - AX_F32 fConfidence = 0.0; + AX_S32 InputFileFrameCnt = 1; + const AX_CHAR *ReWritePath = NULL; + const AX_CHAR *SaveResultPath = NULL; + const AX_CHAR *ModelsPath = NULL; + FILE *fpResultFile = NULL; + AX_S32 nRepeatTimes = 1; + AX_S32 nPPL = AX_SKEL_PPL_HVCP; + AX_S32 nDetectType = 2; + AX_S32 nPushStrategy = 3; + AX_S32 nNpuType = 0; + AX_S32 nInterval = 0; + AX_U32 nStride = 1920; + AX_U32 nWidth = 1920; + AX_U32 nHeight = 1080; + AX_U32 nJencBufSize = 0; + AX_U32 nFrameDepth = 1; + AX_U32 nOneSize = 0; + AX_U64 OneYUVDataPhy = 0; + AX_VOID *OneYUVDataVir = NULL; + AX_U32 nFrameSize = nStride * nHeight * 3 / 2; + AX_U32 nCacheListDepth = 1; + AX_S32 nSrcFrameRate = 25; + AX_S32 nDstFrameRate = 25; + AX_VOID *YUVDataVir = NULL; + AX_U64 YUVDataPhy = 0; + AX_VOID *YUVDataWrVir = NULL; + AX_U64 YUVDataWrPhy = 0; + AX_SKEL_HANDLE pHandle = NULL; + AX_SKEL_FRAME_T stFrame = {0}; + AX_SKEL_RESULT_T *pstResult = NULL; + AX_U64 nFileSize = 0; + AX_U64 nStartTime = 0; + AX_U64 nInitElasped = 0; + AX_U64 nCreateElasped = 0; + AX_U64 nProcessElasped = 0; + AX_U64 nResultElasped = 0; + AX_U64 nResultElaspedMin = (AX_U64)-1; + AX_U64 nResultElaspedMax = 0; + AX_U64 nResultElaspedTotal = 0; + AX_F32 fConfidence = 0.0; // AX_U32 nHumantracksize = 0; // AX_U32 nVehicletracksize = 0; // AX_U32 nCycletracksize = 0; @@ -975,12 +1039,9 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { } } - if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) - || (log_level < 0 || log_level >= SKEL_LOG_MAX) - || (fConfidence < 0 || fConfidence > 1) - || (nDetectType < 0 || nDetectType > 2) - || (nPushStrategy < 1 || nPushStrategy > 3) - || (nNpuType > 3)) { + if (isExit || !InputFile || !InputResolution || (nPPL < AX_SKEL_PPL_HVCP) || (nPPL > AX_SKEL_PPL_MAX) || + (log_level < 0 || log_level >= SKEL_LOG_MAX) || (fConfidence < 0 || fConfidence > 1) || + (nDetectType < 0 || nDetectType > 2) || (nPushStrategy < 1 || nPushStrategy > 3) || (nNpuType > 3)) { ShowUsage(); exit(0); } @@ -1034,7 +1095,7 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { exit(0); } - nWidth = atoi(InputResolution); + nWidth = atoi(InputResolution); nStride = nWidth; nHeight = atoi(temp_p + 1); } @@ -1051,10 +1112,10 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { AX_VENC_MOD_ATTR_T stVencModAttr; memset(&stVencModAttr, 0x00, sizeof(stVencModAttr)); - stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; + stVencModAttr.enVencType = AX_VENC_MULTI_ENCODER; stVencModAttr.stModThdAttr.u32TotalThreadNum = 1; - stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; - nRet = AX_VENC_Init(&stVencModAttr); + stVencModAttr.stModThdAttr.bExplicitSched = AX_FALSE; + nRet = AX_VENC_Init(&stVencModAttr); if (AX_SUCCESS != nRet) { ALOGE("AX_VENC_Init FAILED! ret:0x%x\n", nRet); goto EXIT0; @@ -1076,18 +1137,15 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { #if defined(CHIP_AX650) if (nNpuType == 0) { npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; - } - else if (nNpuType >= 1 && nNpuType <= 4) { + } else if (nNpuType >= 1 && nNpuType <= 4) { npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_STD; - } - else if (nNpuType >= 5 && nNpuType <= 7) { + } else if (nNpuType >= 5 && nNpuType <= 7) { npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_BIG_LITTLE; } #elif defined(CHIP_AX620E) if (nNpuType == 0) { npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE; - } - else { + } else { npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_ENABLE; } #else @@ -1100,27 +1158,23 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { goto EXIT0; } - nOneSize = nWidth * nHeight * 3 / 2; + nOneSize = nWidth * nHeight * 3 / 2; nFrameSize = nStride * nHeight * 3 / 2; InputFileHandle = LoadFile(InputFile, &nFileSize); if (bJpgFile) { InputFileFrameCnt = 1; - } - else { - if (!InputFileHandle - || (nFileSize % nOneSize) != 0) { + } else { + if (!InputFileHandle || (nFileSize % nOneSize) != 0) { ALOGE("%s file is not %dx%d", InputFile, nWidth, nHeight); goto EXIT1; } InputFileFrameCnt = nFileSize / nOneSize; } - - if (nWidth%2 == 1 - || nHeight%2 == 1) { + if (nWidth % 2 == 1 || nHeight % 2 == 1) { ALOGE("wxh(%dx%d) should be even", nWidth, nHeight); goto EXIT1; } @@ -1150,8 +1204,7 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { if (ModelsPath) { stInitParam.pStrModelDeploymentPath = ModelsPath; - } - else { + } else { stInitParam.pStrModelDeploymentPath = "/opt/etc/skelModels"; } @@ -1214,62 +1267,62 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { AX_SKEL_HANDLE_PARAM_T stHandleParam = {0}; - stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; - stHandleParam.nFrameDepth = nFrameDepth; + stHandleParam.ePPL = (AX_SKEL_PPL_E)nPPL; + stHandleParam.nFrameDepth = nFrameDepth; stHandleParam.nFrameCacheDepth = nCacheListDepth; - stHandleParam.nIoDepth = 0; - stHandleParam.nWidth = nWidth; - stHandleParam.nHeight = nHeight; + stHandleParam.nIoDepth = 0; + stHandleParam.nWidth = nWidth; + stHandleParam.nHeight = nHeight; // config settings (if need) - AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_T stConfig = {0}; AX_SKEL_CONFIG_ITEM_T stItems[16] = {0}; - AX_U8 itemIndex = 0; - stConfig.nSize = 0; - stConfig.pstItems = &stItems[0]; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; // venc_attr_config // default will be system definition: w*h*3/8 AX_SKEL_COMMON_THRESHOLD_CONFIG_T stVencAttrConfigThreshold = {0}; if (nJencBufSize > 0) { - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; - stVencAttrConfigThreshold.fValue = (AX_F32) nJencBufSize; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"venc_attr_config"; + stVencAttrConfigThreshold.fValue = (AX_F32)nJencBufSize; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stVencAttrConfigThreshold; stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); itemIndex++; } AX_SKEL_COMMON_THRESHOLD_CONFIG_T stTrackEnableThreshold = {0}; - AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; + AX_SKEL_COMMON_THRESHOLD_CONFIG_T stPushEnableThreshold = {0}; // detect only (disable track + disable push) if (nDetectType == 0) { // track_disable - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; - stTrackEnableThreshold.fValue = 0; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + stTrackEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnableThreshold; stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); itemIndex++; // push_disable - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; - stPushEnableThreshold.fValue = 0; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); itemIndex++; } // detect + track (disable push) else if (nDetectType == 1) { // push_disable - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; - stPushEnableThreshold.fValue = 0; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + stPushEnableThreshold.fValue = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnableThreshold; stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_THRESHOLD_CONFIG_T); itemIndex++; } if (itemIndex > 0) { - stConfig.nSize = itemIndex; + stConfig.nSize = itemIndex; stHandleParam.stConfig = stConfig; } @@ -1280,8 +1333,7 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { // STD-NPU else if (nNpuType == 2) { stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_1; - } - else if (nNpuType == 3) { + } else if (nNpuType == 3) { stHandleParam.nNpuType = (AX_S32)AX_SKEL_STD_VNPU_2; } @@ -1303,11 +1355,11 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { // set config { - AX_SKEL_CONFIG_T stConfig = {0}; + AX_SKEL_CONFIG_T stConfig = {0}; AX_SKEL_CONFIG_ITEM_T stItems[64] = {0}; - AX_U8 itemIndex = 0; - stConfig.nSize = 0; - stConfig.pstItems = &stItems[0]; + AX_U8 itemIndex = 0; + stConfig.nSize = 0; + stConfig.pstItems = &stItems[0]; #if 0 // body_max_target_count stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"body_max_target_count"; @@ -1434,27 +1486,27 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { // detect_roi_polygon AX_SKEL_ROI_POLYGON_CONFIG_T stDetectRoi = {0}; - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; - stDetectRoi.bEnable = AX_FALSE; - stDetectRoi.nPointNum = 4; - AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; - stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; - stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"detect_roi_polygon"; + stDetectRoi.bEnable = AX_FALSE; + stDetectRoi.nPointNum = 4; + AX_SKEL_POINT_T stPoint[4] = {{0, 0}, {nWidth, 0}, {nWidth, nHeight}, {0, nHeight}}; + stDetectRoi.pstPoint = (AX_SKEL_POINT_T *)stPoint; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stDetectRoi; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ROI_POLYGON_CONFIG_T); itemIndex++; // push_strategy - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; - AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; - stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; - stPushStrategy.nIntervalTimes = 2000; - stPushStrategy.nPushCounts = 1; - stPushStrategy.bPushSameFrame = AX_TRUE; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_strategy"; + AX_SKEL_PUSH_STRATEGY_T stPushStrategy = {0}; + stPushStrategy.ePushMode = (AX_SKEL_PUSH_MODE_E)nPushStrategy; + stPushStrategy.nIntervalTimes = 2000; + stPushStrategy.nPushCounts = 1; + stPushStrategy.bPushSameFrame = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushStrategy; stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_STRATEGY_T); itemIndex++; - //4 crop_encoder + // 4 crop_encoder #if 0 // body_crop_encoder AX_SKEL_CROP_ENCODER_THRESHOLD_CONFIG_T stBodyCropEncoderThreshold = {0}; @@ -1512,12 +1564,12 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { itemIndex++; #endif // push_panorama - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_panorama"; AX_SKEL_PUSH_PANORAMA_CONFIG_T stPushPanoramaConfig = {0}; - stPushPanoramaConfig.bEnable = AX_FALSE; - stPushPanoramaConfig.nQuality = 0; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; - stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); + stPushPanoramaConfig.bEnable = AX_FALSE; + stPushPanoramaConfig.nQuality = 0; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushPanoramaConfig; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_PUSH_PANORAMA_CONFIG_T); itemIndex++; #if 0 // push_quality_body @@ -1565,23 +1617,23 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_ATTR_FILTER_CONFIG_T); itemIndex++; #endif -//#if 0 - // track_enable - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; + // #if 0 + // track_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"track_enable"; AX_SKEL_COMMON_ENABLE_CONFIG_T stTrackEnable = {0}; - stTrackEnable.bEnable = AX_TRUE; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; - stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + stTrackEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stTrackEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); itemIndex++; - //push_enable - stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; + // push_enable + stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"push_enable"; AX_SKEL_COMMON_ENABLE_CONFIG_T stPushEnable = {0}; - stPushEnable.bEnable = AX_TRUE; - stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; - stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); + stPushEnable.bEnable = AX_TRUE; + stConfig.pstItems[itemIndex].pstrValue = (AX_VOID *)&stPushEnable; + stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); itemIndex++; -//#endif +// #endif #if 0 //target_config stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"target_config"; @@ -1616,10 +1668,10 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { stConfig.pstItems[itemIndex].nValueSize = sizeof(AX_SKEL_COMMON_ENABLE_CONFIG_T); itemIndex++; #endif - //analyzer_attr_config - // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; - // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; - // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; + // analyzer_attr_config + // stConfig.pstItems[itemIndex].pstrType = (AX_CHAR *)"analyzer_attr_config"; + // AX_SKEL_ANALYZER_CONFIG_T stAnalyze = {0}; + // //AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_NONE; // stAnalyze.nSize = 1; // AX_SKEL_ANALYZER_ATTR_E analyze_attribute = AX_SKEL_ANALYZER_ATTR_FACE_ATTRIBUTE; @@ -1686,14 +1738,14 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { OUTPUT_LOG_SAVE("#####SKEL Process times: %d", nRepeat + 1); STAT_OBJECT_NUM_T tObjectTrackNum = {0}; - STAT_OBJECT_NUM_T tObjectPushNum = {0}; + STAT_OBJECT_NUM_T tObjectPushNum = {0}; for (AX_U32 nFrameCnt = 0; nFrameCnt < InputFileFrameCnt; nFrameCnt++) { AX_BLK blkId = AX_INVALID_BLOCKID; if (InputFileFrameCnt != 1) { if (FrameSkipCtrl(nSrcFrameRate, nDstFrameRate, nSkelFrameId)) { - nSkelFrameId ++; + nSkelFrameId++; LoadFileToMem(InputFileHandle, 0, NULL, nOneSize); @@ -1701,10 +1753,8 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { usleep(nInterval); } continue; - } - else { + } else { if (!FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId)) { - ALOGE("FrameMgrGet fail"); goto EXIT1; @@ -1713,28 +1763,16 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { if (nOneSize != nFrameSize) { LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); - NV12ToStrideNV12(OneYUVDataPhy, - OneYUVDataVir, - nOneSize, - nWidth, - nHeight, - YUVDataPhy, - YUVDataVir, - nFrameSize, - nStride, - nHeight - ); - } - else { + NV12ToStrideNV12(OneYUVDataPhy, OneYUVDataVir, nOneSize, nWidth, nHeight, YUVDataPhy, + YUVDataVir, nFrameSize, nStride, nHeight); + } else { LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nOneSize); } } } else { FrameMgrGet(&YUVDataPhy, &YUVDataVir, nFrameSize, nSkelFrameId, &blkId); - if (nRet != 0 - || YUVDataPhy == 0 - || YUVDataVir == NULL) { + if (nRet != 0 || YUVDataPhy == 0 || YUVDataVir == NULL) { ALOGE("Load input file fail"); goto EXIT1; } @@ -1745,52 +1783,42 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { ALOGE("DecodeJpeg failed!"); goto EXIT3; } - } - else { + } else { if (nOneSize != nFrameSize) { - nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, (AX_S8 *)"SKEL_TEST"); + nRet = AX_SYS_MemAlloc(&OneYUVDataPhy, (AX_VOID **)&OneYUVDataVir, nOneSize, 256, + (AX_S8 *)"SKEL_TEST"); nRet = LoadFileToMem(InputFileHandle, OneYUVDataPhy, OneYUVDataVir, nOneSize); - NV12ToStrideNV12(OneYUVDataPhy, - OneYUVDataVir, - nOneSize, - nWidth, - nHeight, - YUVDataPhy, - YUVDataVir, - nFrameSize, - nStride, - nHeight - ); - } - else { + NV12ToStrideNV12(OneYUVDataPhy, OneYUVDataVir, nOneSize, nWidth, nHeight, YUVDataPhy, + YUVDataVir, nFrameSize, nStride, nHeight); + } else { nRet = LoadFileToMem(InputFileHandle, YUVDataPhy, YUVDataVir, nFrameSize); } } } - stFrame.stFrame.u32Width = nWidth; - stFrame.stFrame.u32Height = nHeight; - stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; - stFrame.stFrame.u32FrameSize = nFrameSize; + stFrame.stFrame.u32Width = nWidth; + stFrame.stFrame.u32Height = nHeight; + stFrame.stFrame.enImgFormat = AX_FORMAT_YUV420_SEMIPLANAR; + stFrame.stFrame.u32FrameSize = nFrameSize; stFrame.stFrame.u32PicStride[0] = nStride; stFrame.stFrame.u32PicStride[1] = nStride; stFrame.stFrame.u32PicStride[2] = nStride; - stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; - stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; - stFrame.stFrame.u64PhyAddr[2] = 0; - stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); - stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); - stFrame.stFrame.u64VirAddr[2] = 0; - stFrame.stFrame.u32BlkId[0] = blkId; - stFrame.stFrame.u32BlkId[1] = 0; - stFrame.stFrame.u32BlkId[2] = 0; + stFrame.stFrame.u64PhyAddr[0] = YUVDataPhy; + stFrame.stFrame.u64PhyAddr[1] = YUVDataPhy + nStride * nHeight; + stFrame.stFrame.u64PhyAddr[2] = 0; + stFrame.stFrame.u64VirAddr[0] = (AX_ULONG)(YUVDataVir); + stFrame.stFrame.u64VirAddr[1] = (AX_ULONG)(YUVDataVir + nStride * nHeight); + stFrame.stFrame.u64VirAddr[2] = 0; + stFrame.stFrame.u32BlkId[0] = blkId; + stFrame.stFrame.u32BlkId[1] = 0; + stFrame.stFrame.u32BlkId[2] = 0; ALOGN("*****SKEL Frame(%d) Process", nFrameCnt + 1); - //OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); + // OUTPUT_LOG_SAVE("*****SKEL Frame(%d) Process Start", nFrameCnt + 1); - stFrame.nFrameId = nSkelFrameId ++; - nStartTime = get_tick_count(); + stFrame.nFrameId = nSkelFrameId++; + nStartTime = get_tick_count(); while (1) { nRet = AX_SKEL_SendFrame(pHandle, &stFrame, 0); @@ -1846,7 +1874,7 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { ALOGN("\tObject Num: %d", pstResult->nObjectSize); - AX_U32 nSkelSize = 0; + AX_U32 nSkelSize = 0; AI_Detection_SkelResult_t Skels[SKEL_SAMPLE_OBJECT_SIZE] = {0}; for (AX_U32 i = 0; i < pstResult->nObjectSize; i++) { AX_SKEL_OBJECT_ITEM_T *pstItems = &pstResult->pstObjectItems[i]; @@ -1855,59 +1883,53 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { ALOGI("\t\tTrackId: %lld, TrackState: %d", pstItems->nTrackId, pstItems->eTrackState); ALOGN("\t\tRect[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, pstItems->pstrObjectCategory, - pstItems->stRect.fX, - pstItems->stRect.fY, pstItems->stRect.fW, - pstItems->stRect.fH, pstItems->fConfidence); + pstItems->stRect.fX, pstItems->stRect.fY, pstItems->stRect.fW, pstItems->stRect.fH, + pstItems->fConfidence); StatTrackMgr(pstItems, &tObjectTrackNum); StatPushMgr(pstItems, &tObjectPushNum); // get detect box only new or update state - if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW - || pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) - && nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { + if ((pstItems->eTrackState == AX_SKEL_TRACK_STATUS_NEW || + pstItems->eTrackState == AX_SKEL_TRACK_STATUS_UPDATE) && + nSkelSize < SKEL_SAMPLE_OBJECT_SIZE) { Skels[nSkelSize].pstrObjectCategory = pstItems->pstrObjectCategory; - Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; - Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; - Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; - Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; + Skels[nSkelSize].tBox.fX = pstItems->stRect.fX; + Skels[nSkelSize].tBox.fY = pstItems->stRect.fY; + Skels[nSkelSize].tBox.fW = pstItems->stRect.fW; + Skels[nSkelSize].tBox.fH = pstItems->stRect.fH; - OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", - i, pstItems->pstrObjectCategory, - pstItems->stRect.fX, pstItems->stRect.fY, - pstItems->stRect.fW, pstItems->stRect.fH, - pstItems->fConfidence); + OUTPUT_LOG_SAVE("\t\tObject[%d] %s: [%f, %f, %f, %f], Confidence: %f", i, + pstItems->pstrObjectCategory, pstItems->stRect.fX, pstItems->stRect.fY, + pstItems->stRect.fW, pstItems->stRect.fH, pstItems->fConfidence); ALOGN("\t\t[%d]Point Set Size: %d", i, pstItems->nPointSetSize); // point Skels[nSkelSize].nPointNum = AX_MIN(DETECT_SKEL_POINT_COUNT, pstItems->nPointSetSize); for (AX_U32 j = 0; j < Skels[i].nPointNum; j++) { - ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, pstItems->pstPointSet[j].pstrObjectCategory, - pstItems->pstPointSet[j].stPoint.fX, - pstItems->pstPointSet[j].stPoint.fY, - pstItems->pstPointSet[j].fConfidence); - Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; - Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; - - OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", - j, + ALOGI("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f", j, + pstItems->pstPointSet[j].pstrObjectCategory, pstItems->pstPointSet[j].stPoint.fX, + pstItems->pstPointSet[j].stPoint.fY, pstItems->pstPointSet[j].fConfidence); + Skels[nSkelSize].tPoint[j].fX = pstItems->pstPointSet[j].stPoint.fX; + Skels[nSkelSize].tPoint[j].fY = pstItems->pstPointSet[j].stPoint.fY; + + OUTPUT_LOG_SAVE("\t\t\tPoint[%d] %s: [%f, %f] Confidence: %f\n", j, pstItems->pstPointSet[j].pstrObjectCategory, - pstItems->pstPointSet[j].stPoint.fX, - pstItems->pstPointSet[j].stPoint.fY, + pstItems->pstPointSet[j].stPoint.fX, pstItems->pstPointSet[j].stPoint.fY, pstItems->pstPointSet[j].fConfidence); } - //face attr - if(nPPL == AX_SKEL_PPL_FACE){ + // face attr + if (nPPL == AX_SKEL_PPL_FACE) { ALOGN("\t\tFace Attribute:"); - ALOGI("\t\t\tfYaw %f,",pstItems->stFaceAttr.fYaw); - ALOGI("\t\t\tfPitch %f,",pstItems->stFaceAttr.fPitch); - ALOGI("\t\t\tfRoll %f,",pstItems->stFaceAttr.fRoll); - ALOGI("\t\t\tfMask %f,",pstItems->stFaceAttr.fMask); - ALOGI("\t\t\tage %d",pstItems->stFaceAttr.nAge); - ALOGI("\t\t\tgender %d",pstItems->stFaceAttr.nGender); + ALOGI("\t\t\tfYaw %f,", pstItems->stFaceAttr.fYaw); + ALOGI("\t\t\tfPitch %f,", pstItems->stFaceAttr.fPitch); + ALOGI("\t\t\tfRoll %f,", pstItems->stFaceAttr.fRoll); + ALOGI("\t\t\tfMask %f,", pstItems->stFaceAttr.fMask); + ALOGI("\t\t\tage %d", pstItems->stFaceAttr.nAge); + ALOGI("\t\t\tgender %d", pstItems->stFaceAttr.nGender); } - nSkelSize ++; + nSkelSize++; } if (AX_SKEL_TRACK_STATUS_SELECT == pstItems->eTrackState) { @@ -1915,125 +1937,93 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { memset(&AttrInfo, 0x00, sizeof(AttrInfo)); AttrParser(pstItems, &AttrInfo); - if (pstItems->bCropFrame - && pstItems->stCropFrame.pFrameData - && 0 < pstItems->stCropFrame.nFrameDataSize - && SaveResultPath) { + if (pstItems->bCropFrame && pstItems->stCropFrame.pFrameData && + 0 < pstItems->stCropFrame.nFrameDataSize && SaveResultPath) { // save attribute - AX_CHAR arrDat[256] = {0}; + AX_CHAR arrDat[256] = {0}; AX_CHAR strFile[512] = {0}; if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { sprintf(arrDat, "frame%lld_crop_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d", - pstItems->stCropFrame.nFrameId, - pstItems->pstrObjectCategory, - pstItems->nTrackId, - AttrInfo.tFaceInfo.szGender, - AttrInfo.tFaceInfo.nAge, - AttrInfo.tFaceInfo.szMask, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); OUTPUT_LOG_SAVE("\t\t[FACE ATTR] %s", arrDat); - } - else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { sprintf(arrDat, "frame%lld_crop_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d", - pstItems->stCropFrame.nFrameId, - pstItems->pstrObjectCategory, - pstItems->nTrackId, - AttrInfo.tPlateInfo.bValid, - AttrInfo.tPlateInfo.szNum, - AttrInfo.tPlateInfo.szColor, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); OUTPUT_LOG_SAVE("\t\t[PLATE ATTR] %s", arrDat); - } - else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { - sprintf(arrDat, "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", - pstItems->stCropFrame.nFrameId, - pstItems->pstrObjectCategory, - pstItems->nTrackId, - AttrInfo.tPlateInfo.bValid, - AttrInfo.tPlateInfo.szNum, - AttrInfo.tPlateInfo.szColor, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, + "frame%lld_crop_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d", + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, + AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); OUTPUT_LOG_SAVE("\t\t[VEHICLE ATTR] %s", arrDat); - } - else { - sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", - pstItems->stCropFrame.nFrameId, - pstItems->pstrObjectCategory, - pstItems->nTrackId, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, - pstItems->stCropFrame.nFrameHeight); + } else { + sprintf(arrDat, "frame%lld_crop_%s[%lld]_Score[%f]_%dx%d", pstItems->stCropFrame.nFrameId, + pstItems->pstrObjectCategory, pstItems->nTrackId, pstItems->fConfidence, + pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); } - sprintf(strFile, "%s/%s/%s.jpg", - SaveResultPath, - pstItems->pstrObjectCategory, - arrDat); + sprintf(strFile, "%s/%s/%s.jpg", SaveResultPath, pstItems->pstrObjectCategory, arrDat); FILE *fp_w = fopen(strFile, "wb"); if (fp_w) { ALOGI("Write crop jpg to file: %s", strFile); - fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, fp_w); + fwrite((AX_U8 *)pstItems->stCropFrame.pFrameData, 1, pstItems->stCropFrame.nFrameDataSize, + fp_w); fclose(fp_w); } - } // panora frame - if (pstItems->bPanoraFrame - && pstItems->stPanoraFrame.pFrameData - && 0 < pstItems->stPanoraFrame.nFrameDataSize - && SaveResultPath) { - AX_CHAR arrDat[256] = {0}; + if (pstItems->bPanoraFrame && pstItems->stPanoraFrame.pFrameData && + 0 < pstItems->stPanoraFrame.nFrameDataSize && SaveResultPath) { + AX_CHAR arrDat[256] = {0}; AX_CHAR strFile[512] = {0}; if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_FACE) { sprintf(arrDat, "frame%lld_panora_%s[%lld]_Gender[%s]_Age[%d]_Mask[%s]_Score[%f]_%dx%d.jpg", pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, AttrInfo.tFaceInfo.szGender, AttrInfo.tFaceInfo.nAge, AttrInfo.tFaceInfo.szMask, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); - } - else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_PLATE) { sprintf(arrDat, "frame%lld_panora_%s[%lld]_Valid[%d]_Num[%s]_Color[%s]_Score[%f]_%dx%d.jpg", pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); - } - else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { - sprintf(arrDat, "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else if (AttrInfo.bExist && AttrInfo.eType == ATTR_TYPE_VEHICLE) { + sprintf(arrDat, + "frame%lld_panora_%s[%lld]_[plate_Valid[%d]_Num[%s]_Color[%s]]_Score[%f]_%dx%d.jpg", pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, AttrInfo.tPlateInfo.bValid, AttrInfo.tPlateInfo.szNum, AttrInfo.tPlateInfo.szColor, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); - } - else { + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); + } else { sprintf(arrDat, "frame%lld_panora_%s[%lld]_Score[%f]_%dx%d.jpg", pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, pstItems->nTrackId, - pstItems->fConfidence, - pstItems->stCropFrame.nFrameWidth, pstItems->stCropFrame.nFrameHeight); + pstItems->fConfidence, pstItems->stCropFrame.nFrameWidth, + pstItems->stCropFrame.nFrameHeight); } - sprintf(strFile, "%s/%s/%s.jpg", - SaveResultPath, - pstItems->pstrObjectCategory, - arrDat); + sprintf(strFile, "%s/%s/%s.jpg", SaveResultPath, pstItems->pstrObjectCategory, arrDat); FILE *fp_w = fopen(strFile, "wb"); if (fp_w) { ALOGI("Write panora jpg to file: %s", strFile); - fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, pstItems->stPanoraFrame.nFrameDataSize, fp_w); + fwrite((AX_U8 *)pstItems->stPanoraFrame.pFrameData, 1, + pstItems->stPanoraFrame.nFrameDataSize, fp_w); fclose(fp_w); } } @@ -2041,42 +2031,37 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { // feature ALOGI("\t\tFeature Size: %d", pstItems->nFeatureSize); - if (pstItems->pstFeatureItem - && 0 < pstItems->nFeatureSize - && SaveResultPath) { + if (pstItems->pstFeatureItem && 0 < pstItems->nFeatureSize && SaveResultPath) { AX_CHAR strFile[256] = {0}; - sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", - SaveResultPath, pstItems->pstrObjectCategory, - pstItems->stCropFrame.nFrameId, - pstItems->pstrObjectCategory, i); + sprintf(strFile, "%s/%s/frame%lld_feature_%s_%d.db", SaveResultPath, pstItems->pstrObjectCategory, + pstItems->stCropFrame.nFrameId, pstItems->pstrObjectCategory, i); FILE *fp_w = fopen(strFile, "wb"); if (fp_w) { - ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", - strFile, - pstItems->pstFeatureItem[0].nValueSize); - fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, pstItems->pstFeatureItem[0].nValueSize, fp_w); + ALOGI("\t\t\tWrite feature to file: %s, nValueSize: %d", strFile, + pstItems->pstFeatureItem[0].nValueSize); + fwrite((AX_U8 *)pstItems->pstFeatureItem[0].pstrValue, 1, + pstItems->pstFeatureItem[0].nValueSize, fp_w); fclose(fp_w); } } // meta ALOGI("\t\tMeta Size: %d", pstItems->nMetaInfoSize); - if (pstItems->pstMetaInfo - && 0 < pstItems->nMetaInfoSize) { + if (pstItems->pstMetaInfo && 0 < pstItems->nMetaInfoSize) { for (AX_U32 i = 0; i < pstItems->nMetaInfoSize; i++) { - ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, pstItems->pstMetaInfo[i].pstrValue); + ALOGI("\t\tMetaInfo[%d] %s: \n%s", i, pstItems->pstMetaInfo[i].pstrType, + pstItems->pstMetaInfo[i].pstrValue); } } // binds ALOGI("\t\tBind Size: %d", pstItems->nObjectBindSize); - if (pstItems->pstObjectBind - && 0 < pstItems->nObjectBindSize) { + if (pstItems->pstObjectBind && 0 < pstItems->nObjectBindSize) { for (AX_U32 i = 0; i < pstItems->nObjectBindSize; i++) { ALOGI("\t\t[%s] [TrackId] %lld bind to ObjectBind[%d]: %s [TrackId] %lld", - pstItems->pstrObjectCategory, pstItems->nTrackId, i, - pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); + pstItems->pstrObjectCategory, pstItems->nTrackId, i, + pstItems->pstObjectBind[i].pstrObjectCategoryBind, pstItems->pstObjectBind[i].nTrackId); } } } @@ -2092,41 +2077,41 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { memcpy(YUVDataWrVir, YUVDataVir, nFrameSize); YUV_IMAGE_T YUVImage = {0}; - YUVImage.pImage = YUVDataWrVir; - YUVImage.nWidth = nWidth; - YUVImage.nHeight = nHeight; - YUVImage.stride = nStride; - YUVImage.nSize = nFrameSize; - YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; + YUVImage.pImage = YUVDataWrVir; + YUVImage.nWidth = nWidth; + YUVImage.nHeight = nHeight; + YUVImage.stride = nStride; + YUVImage.nSize = nFrameSize; + YUVImage.eType = AX_FORMAT_YUV420_SEMIPLANAR; for (size_t i = 0; i < nSkelSize; i++) { // draw rect AX_CHAR ObjectCategory = Skels[i].pstrObjectCategory[0]; - AX_S16 x0 = Skels[i].tBox.fX; - AX_S16 y0 = Skels[i].tBox.fY; - AX_U16 w = Skels[i].tBox.fW; - AX_U16 h = Skels[i].tBox.fH; - YUV_COLOR RectColor = YUV_WHITE; - - switch(ObjectCategory){ - case 'b': //body - RectColor = YUV_WHITE; - break; - case 'v': //vehicle - RectColor = YUV_PURPLE; - break; - case 'c': //cycle - RectColor = YUV_GREEN; - break; - case 'f': //face - RectColor = YUV_YELLOW; - break; - case 'p': //plate - RectColor = YUV_RED; - break; - default : - RectColor = YUV_WHITE; - break; + AX_S16 x0 = Skels[i].tBox.fX; + AX_S16 y0 = Skels[i].tBox.fY; + AX_U16 w = Skels[i].tBox.fW; + AX_U16 h = Skels[i].tBox.fH; + YUV_COLOR RectColor = YUV_WHITE; + + switch (ObjectCategory) { + case 'b': // body + RectColor = YUV_WHITE; + break; + case 'v': // vehicle + RectColor = YUV_PURPLE; + break; + case 'c': // cycle + RectColor = YUV_GREEN; + break; + case 'f': // face + RectColor = YUV_YELLOW; + break; + case 'p': // plate + RectColor = YUV_RED; + break; + default: + RectColor = YUV_WHITE; + break; } DrawRect(&YUVImage, x0, y0, w, h, RectColor); @@ -2145,13 +2130,13 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { AX_CHAR arrWrFile[512] = {0}; sprintf(arrWrFile, "%s_result_%d_%d.jpg", InputFile, nFrameCnt + 1, nRepeat + 1); - AX_CHAR *fileName = NULL; - AX_U32 fileNameLen = strlen(arrWrFile); + AX_CHAR *fileName = NULL; + AX_U32 fileNameLen = strlen(arrWrFile); AX_CHAR fullFileName[256] = {0}; - AX_S32 i = fileNameLen - 1; + AX_S32 i = fileNameLen - 1; if (fileNameLen > 0) { - for (; i >= 0; i --) { + for (; i >= 0; i--) { if (arrWrFile[i] == '/') { break; } @@ -2160,16 +2145,16 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { if (i < 0) { i = 0; - } - else if (i < fileNameLen - 1) { - i ++; + } else if (i < fileNameLen - 1) { + i++; } fileName = (AX_CHAR *)&arrWrFile[i]; sprintf(fullFileName, "%s/%s", ReWritePath, fileName); - EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, nFrameSize); + EncodeOneFrameToJpeg((AX_CHAR *)fullFileName, nStride, nWidth, nHeight, YUVDataWrPhy, YUVDataWrVir, + nFrameSize); } if (pstResult) { @@ -2184,32 +2169,20 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { } ALOGN("SKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", - tObjectTrackNum.nBodyNum, - tObjectTrackNum.nVehicleNum, - tObjectTrackNum.nCycleNum, - tObjectTrackNum.nFaceNum, - tObjectTrackNum.nPlateNum); + tObjectTrackNum.nBodyNum, tObjectTrackNum.nVehicleNum, tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, tObjectTrackNum.nPlateNum); OUTPUT_LOG_SAVE("\nSKEL Process Objects Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", - tObjectTrackNum.nBodyNum, - tObjectTrackNum.nVehicleNum, - tObjectTrackNum.nCycleNum, - tObjectTrackNum.nFaceNum, - tObjectTrackNum.nPlateNum); + tObjectTrackNum.nBodyNum, tObjectTrackNum.nVehicleNum, tObjectTrackNum.nCycleNum, + tObjectTrackNum.nFaceNum, tObjectTrackNum.nPlateNum); ALOGN("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]", - tObjectPushNum.nBodyNum, - tObjectPushNum.nVehicleNum, - tObjectPushNum.nCycleNum, - tObjectPushNum.nFaceNum, - tObjectPushNum.nPlateNum); + tObjectPushNum.nBodyNum, tObjectPushNum.nVehicleNum, tObjectPushNum.nCycleNum, tObjectPushNum.nFaceNum, + tObjectPushNum.nPlateNum); OUTPUT_LOG_SAVE("SKEL Process Push Statistics: Body[%d], Vehicle[%d], Cycle[%d], Face[%d], Plate[%d]\n", - tObjectPushNum.nBodyNum, - tObjectPushNum.nVehicleNum, - tObjectPushNum.nCycleNum, - tObjectPushNum.nFaceNum, - tObjectPushNum.nPlateNum); + tObjectPushNum.nBodyNum, tObjectPushNum.nVehicleNum, tObjectPushNum.nCycleNum, + tObjectPushNum.nFaceNum, tObjectPushNum.nPlateNum); if (InputFileHandle) { ReLoadFile(InputFileHandle); @@ -2220,16 +2193,11 @@ AX_S32 main(AX_S32 argc, AX_CHAR *argv[]) { // ALOGE("AX_POOL_DestroyPool failed! Error Code:0x%X\n", nRet); // } - ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", - nRepeatTimes, - nResultElaspedMin, - (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, - nResultElaspedMax); + ALOGN("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", nRepeatTimes, + nResultElaspedMin, (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, nResultElaspedMax); - OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", - nRepeatTimes, - nResultElaspedMin, - (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, + OUTPUT_LOG_SAVE("SKEL Process Elapsed Info: Repeats: %d, (min: %lld ms, avr: %lld ms, max: %lld ms)", nRepeatTimes, + nResultElaspedMin, (nSkelFrameId > 1) ? (nResultElaspedTotal / (nSkelFrameId - 1)) : 0, nResultElaspedMax); EXIT3: diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index a65650fe..e7f129d1 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -171,7 +171,9 @@ int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) memset(eth_ip_buff, 0, sizeof(eth_ip_buff)); if(file != NULL) { - fread(eth_ip_buff, 1, sizeof(eth_ip_buff), file); + int size = fread(eth_ip_buff, 1, sizeof(eth_ip_buff), file); + if(size > 0) + eth_ip_buff[size-1] = '\0'; fclose(file); } eth_info["speed"] = eth_ip_buff; @@ -490,7 +492,7 @@ int sys_reset(int com_id, const nlohmann::json &json_obj) int sys_version(int com_id, const nlohmann::json &json_obj) { - usr_out(json_obj["request_id"], json_obj["work_id"], std::string("v1.2"), com_id); + usr_out(json_obj["request_id"], json_obj["work_id"], std::string("v1.3"), com_id); int out = 0; return out; } diff --git a/projects/llm_framework/main_sys/src/main.cpp b/projects/llm_framework/main_sys/src/main.cpp index 84765bf4..7afbef6e 100644 --- a/projects/llm_framework/main_sys/src/main.cpp +++ b/projects/llm_framework/main_sys/src/main.cpp @@ -52,6 +52,7 @@ void get_run_config() key_sql["config_zmq_s_format"] = std::string("ipc:///tmp/llm/%i.sock"); key_sql["config_zmq_c_format"] = std::string("ipc:///tmp/llm/%i.sock"); key_sql["config_lsmod_dir"] = std::string("/opt/m5stack/data/models/"); + key_sql["config_model_d"] = std::string("/opt/m5stack/data/models/"); key_sql["config_base_mode_path"] = std::string("/opt/m5stack/data/"); key_sql["config_base_mode_config_path"] = std::string("/opt/m5stack/etc/"); key_sql["config_tcp_server"] = 10001; diff --git a/projects/llm_framework/main_llm_tokenizer/Kconfig b/projects/llm_framework/main_tokenizer/Kconfig similarity index 100% rename from projects/llm_framework/main_llm_tokenizer/Kconfig rename to projects/llm_framework/main_tokenizer/Kconfig diff --git a/projects/llm_framework/main_llm_tokenizer/SConstruct b/projects/llm_framework/main_tokenizer/SConstruct similarity index 100% rename from projects/llm_framework/main_llm_tokenizer/SConstruct rename to projects/llm_framework/main_tokenizer/SConstruct diff --git a/projects/llm_framework/main_llm_tokenizer/_tokenizer.py b/projects/llm_framework/main_tokenizer/_tokenizer.py similarity index 100% rename from projects/llm_framework/main_llm_tokenizer/_tokenizer.py rename to projects/llm_framework/main_tokenizer/_tokenizer.py diff --git a/projects/llm_framework/main_llm_tokenizer/src/main.cpp b/projects/llm_framework/main_tokenizer/src/main.cpp similarity index 100% rename from projects/llm_framework/main_llm_tokenizer/src/main.cpp rename to projects/llm_framework/main_tokenizer/src/main.cpp diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index 0ce3b6bb..c21e0aa8 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -25,9 +25,7 @@ REQUIREMENTS += ['tts'] INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), ADir('src/runner/include'), ADir('src/runner/src/header')] -STATIC_FILES += [AFile('single_speaker_english_fast.json'), - AFile('single_speaker_fast.json') - ] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_tts', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_tts/single_speaker_english_fast.json b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json similarity index 100% rename from projects/llm_framework/main_tts/single_speaker_english_fast.json rename to projects/llm_framework/main_tts/mode_single-speaker-english-fast.json diff --git a/projects/llm_framework/main_tts/single_speaker_fast.json b/projects/llm_framework/main_tts/mode_single-speaker-fast.json similarity index 100% rename from projects/llm_framework/main_tts/single_speaker_fast.json rename to projects/llm_framework/main_tts/mode_single-speaker-fast.json diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index a15b4808..d2962a4f 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -95,7 +95,15 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + // Compatible operation + if (model_ == "single_speaker_english_fast") + config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, "single-speaker-english-fast"); + else if (model_ == "single_speaker_fast") + config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, "single-speaker-fast"); + try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -169,27 +177,11 @@ class llm_tts : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_tts::_load_config success"); - return 0; - } - } public: llm_tts() : StackFlow("tts") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_tts::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, @@ -485,7 +477,7 @@ class llm_tts : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("tts.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index b1f51c8a..2f399933 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -49,9 +49,8 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] STATIC_LIB += static_file * 4 -STATIC_FILES += [AFile('internvl2-1b-ax630c.json'), - AFile('internvl2-1b-ax630c_tokenizer.py') - ] +STATIC_FILES += [AFile('internvl2-1b-ax630c_tokenizer.py')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_vlm', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_vlm/internvl2-1b-ax630c.json b/projects/llm_framework/main_vlm/mode_internvl2-1b-ax630c.json similarity index 100% rename from projects/llm_framework/main_vlm/internvl2-1b-ax630c.json rename to projects/llm_framework/main_vlm/mode_internvl2-1b-ax630c.json diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 825aab2d..ca014d83 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -89,7 +89,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -240,27 +240,11 @@ class llm_llm : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_llm::_load_config success"); - return 0; - } - } public: llm_llm() : StackFlow("vlm") { task_count_ = 2; - repeat_event(1000, std::bind(&llm_llm::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 02f8a7d2..e5ea4cd4 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -27,7 +27,7 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') STATIC_LIB += static_file * 2 -STATIC_FILES += [AFile('yolo11n.json'), AFile('yolo11n-seg.json'), AFile('yolo11n-pose.json')] +STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_yolo', diff --git a/projects/llm_framework/main_yolo/yolo11n-pose.json b/projects/llm_framework/main_yolo/mode_yolo11n-pose.json similarity index 100% rename from projects/llm_framework/main_yolo/yolo11n-pose.json rename to projects/llm_framework/main_yolo/mode_yolo11n-pose.json diff --git a/projects/llm_framework/main_yolo/yolo11n-seg.json b/projects/llm_framework/main_yolo/mode_yolo11n-seg.json similarity index 100% rename from projects/llm_framework/main_yolo/yolo11n-seg.json rename to projects/llm_framework/main_yolo/mode_yolo11n-seg.json diff --git a/projects/llm_framework/main_yolo/yolo11n.json b/projects/llm_framework/main_yolo/mode_yolo11n.json similarity index 100% rename from projects/llm_framework/main_yolo/yolo11n.json rename to projects/llm_framework/main_yolo/mode_yolo11n.json diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index c1b69bec..9bdb59e5 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -89,7 +89,7 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, model_); + std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -251,27 +251,11 @@ class llm_yolo : public StackFlow { private: int task_count_; std::unordered_map> llm_task_; - int _load_config() - { - if (base_model_path_.empty()) { - base_model_path_ = sys_sql_select("config_base_mode_path"); - } - if (base_model_config_path_.empty()) { - base_model_config_path_ = sys_sql_select("config_base_mode_config_path"); - } - if (base_model_path_.empty() || base_model_config_path_.empty()) { - return -1; - } else { - SLOGI("llm_yolo::_load_config success"); - return 0; - } - } public: llm_yolo() : StackFlow("yolo") { task_count_ = 1; - repeat_event(1000, std::bind(&llm_yolo::_load_config, this)); } void task_output(const std::weak_ptr llm_task_obj_weak, diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 46d3b28f..1637eb39 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -31,7 +31,7 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): elif item.startswith('lib'): os.makedirs(os.path.join(deb_folder, 'opt/m5stack/lib'), exist_ok = True) shutil.copy2(os.path.join(src_folder, item), os.path.join(deb_folder, 'opt/m5stack/lib', item)) - else: + elif not item.startswith('mode_'): os.makedirs(os.path.join(deb_folder, 'opt/m5stack/share'), exist_ok = True) shutil.copy2(os.path.join(src_folder, item), os.path.join(deb_folder, 'opt/m5stack/share', item)) # os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data'), exist_ok = True) @@ -98,6 +98,12 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): f.write(f'[ -f "/lib/systemd/system/llm-llm.service" ] && systemctl start llm-llm.service\n') f.write(f'[ -f "/lib/systemd/system/llm-tts.service" ] && systemctl enable llm-tts.service\n') f.write(f'[ -f "/lib/systemd/system/llm-tts.service" ] && systemctl start llm-tts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl enable llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl start llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl enable llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl start llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl enable llm-melotts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl start llm-melotts.service\n') f.write(f'exit 0\n') with open(os.path.join(deb_folder, 'DEBIAN/prerm'),'w') as f: f.write(f'#!/bin/sh\n') @@ -113,6 +119,12 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): f.write(f'[ -f "/lib/systemd/system/llm-asr.service" ] && systemctl disable llm-asr.service\n') f.write(f'[ -f "/lib/systemd/system/llm-sys.service" ] && systemctl stop llm-sys.service\n') f.write(f'[ -f "/lib/systemd/system/llm-sys.service" ] && systemctl disable llm-sys.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl stop llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-camera.service" ] && systemctl disable llm-camera.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl stop llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-yolo.service" ] && systemctl disable llm-yolo.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl stop llm-melotts.service\n') + f.write(f'[ -f "/lib/systemd/system/llm-melotts.service" ] && systemctl disable llm-melotts.service\n') f.write(f'exit 0\n') os.chmod(os.path.join(deb_folder, 'DEBIAN/postinst'), 0o755) os.chmod(os.path.join(deb_folder, 'DEBIAN/prerm'), 0o755) @@ -145,6 +157,12 @@ def create_data_deb(package_name, version, src_folder, revision = 'm5stack1'): if os.path.exists(zip_file_extrpath): shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/data')) + os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data/models'), exist_ok = True) + if os.path.exists(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:]))): + shutil.copy2(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), os.path.join(deb_folder, 'opt/m5stack/data/models', 'mode_{}.json'.format(package_name[4:]))) + else: + print(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), " miss") + os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: f.write(f'Package: {package_name}\n') @@ -258,6 +276,12 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_bin_deb('llm-llm', version, src_folder, revision) create_bin_deb('llm-tts', version, src_folder, revision) create_bin_deb('llm-melotts', version, src_folder, revision) + create_bin_deb('llm-camera', version, src_folder, revision) + create_bin_deb('llm-vlm', version, src_folder, revision) + create_bin_deb('llm-yolo', version, src_folder, revision) + create_bin_deb('llm-skel', version, src_folder, revision) + # create_bin_deb('llm-tokenizer', version, src_folder, revision) + if (create_data): create_data_deb('llm-audio-en-us', data_version, src_folder, revision) create_data_deb('llm-audio-zh-cn', data_version, src_folder, revision) From 9f8115cde81e49040099c5022ddb32406a54b06b Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 3 Dec 2024 16:45:47 +0800 Subject: [PATCH 34/44] [fix] tts input msg escape --- projects/llm_framework/main_melotts/src/main.cpp | 6 ++++-- projects/llm_framework/main_tts/src/main.cpp | 3 ++- projects/llm_framework/tools/llm_pack.py | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 584d229e..66405a8e 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -127,7 +127,8 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); // Compatible operation if (model_ == "melotts_zh-cn") config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, "melotts-zh-cn"); @@ -427,7 +428,8 @@ class llm_tts : public StackFlow { } next_data = &tmp_msg2; } - std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar((*next_data)); + std::string user_msg = sample_unescapeString(*next_data); + std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar(user_msg); for (auto cutf8 : tmp_data) { if (is_breakpoint(cutf8)) { llm_task_obj->tts_string_stream_buff += cutf8; diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index d2962a4f..055412e2 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -268,7 +268,8 @@ class llm_tts : public StackFlow { } next_data = &tmp_msg2; } - std::vector tmp_data = splitEachChar((*next_data)); + std::string user_msg = sample_unescapeString(*next_data); + std::vector tmp_data = splitEachChar(user_msg); for (auto cutf8 : tmp_data) { if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") { diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 1637eb39..925e6e1c 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -157,11 +157,13 @@ def create_data_deb(package_name, version, src_folder, revision = 'm5stack1'): if os.path.exists(zip_file_extrpath): shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/data')) + RED = "\033[31m" + RESET = "\033[0m" os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data/models'), exist_ok = True) if os.path.exists(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:]))): shutil.copy2(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), os.path.join(deb_folder, 'opt/m5stack/data/models', 'mode_{}.json'.format(package_name[4:]))) else: - print(os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), " miss") + print(RED, os.path.join(src_folder,'mode_{}.json'.format(package_name[4:])), " miss", RESET) os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: From 69c9de4c5c647a069a1819067903bbf9ab720dd4 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 3 Dec 2024 17:17:24 +0800 Subject: [PATCH 35/44] [update] add yolo package. --- projects/llm_framework/tools/llm_pack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 925e6e1c..5728b381 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -297,3 +297,6 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_data_deb('llm-single-speaker-english-fast', data_version, src_folder, revision) create_data_deb('llm-single-speaker-fast', data_version, src_folder, revision) create_data_deb('llm-melotts-zh-cn', data_version, src_folder, revision) + create_data_deb('llm-yolo11n', data_version, src_folder, revision) + create_data_deb('llm-yolo11n-pose', data_version, src_folder, revision) + create_data_deb('llm-yolo11n-seg', data_version, src_folder, revision) From 6cb8075e95f3e326c257fb20d88fd36a17af47ba Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 3 Dec 2024 17:21:44 +0800 Subject: [PATCH 36/44] [update] yolo --- projects/llm_framework/main_yolo/src/main.cpp | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 9bdb59e5..380d6c42 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -161,17 +161,43 @@ class llm_task { throw std::string("img size error"); } cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); - cv::Mat bgr; - cv::cvtColor(camera_data, bgr, cv::COLOR_YUV2BGR_YUYV); - return inference(bgr); + cv::Mat rgb; + cv::cvtColor(camera_data, rgb, cv::COLOR_YUV2RGB_YUYV); + return inference(rgb, false); } - bool inference(cv::Mat &src) + bool inference_raw_rgb(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + return inference(camera_data, false); + } + + bool inference_raw_bgr(const std::string &msg) + { + if (inference_mtx_.try_lock()) + std::lock_guard guard(inference_mtx_, std::adopt_lock); + else + return true; + if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { + throw std::string("img size error"); + } + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + return inference(camera_data); + } + + bool inference(cv::Mat &src, bool bgr2rgb = true) { try { int ret = -1; std::vector image(mode_config_.img_w * mode_config_.img_h * 3, 0); - common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, true); + common::get_input_data_letterbox(src, image, mode_config_.img_w, mode_config_.img_h, bgr2rgb); yolo_->SetInput((void *)image.data(), 0); if (0 != yolo_->RunSync()) { SLOGE("Run yolo model failed!\n"); From c91d0f92a7ea7d23e5a62579a3a1224ab966a0ee Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Tue, 3 Dec 2024 18:50:10 +0800 Subject: [PATCH 37/44] [update] down play audio --- projects/llm_framework/main_audio/audio.json | 2 +- projects/llm_framework/main_yolo/src/EngineWrapper.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/llm_framework/main_audio/audio.json b/projects/llm_framework/main_audio/audio.json index de7fb6dc..eab11c98 100644 --- a/projects/llm_framework/main_audio/audio.json +++ b/projects/llm_framework/main_audio/audio.json @@ -15,7 +15,7 @@ "play_param": { "card": 0, "device": 1, - "volume": 1.0, + "volume": 0.5, "channel": 1, "rate": 16000, "bit": 16, diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp index 592d74e4..e07a044e 100644 --- a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -340,7 +340,7 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c auto mask_proto_ptr = (float*)io_data->pOutputs[6].pVirAddr; detection::get_out_bbox_mask(proposals, objects, mask_proto_ptr, 32, 4, nms_threshold, input_h, input_w, mat.rows, mat.cols); - detection::draw_objects_mask(mat, objects, CLASS_NAMES, COCO_COLORS, "yolo11_seg_out"); + // detection::draw_objects_mask(mat, objects, CLASS_NAMES, COCO_COLORS, "yolo11_seg_out"); } else if (model_type == "pose") { float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, (float*)io_data->pOutputs[1].pVirAddr, (float*)io_data->pOutputs[2].pVirAddr}; @@ -355,7 +355,7 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c input_h, input_w, 17, cls_num); } detection::get_out_bbox_kps(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); - detection::draw_keypoints(mat, objects, KPS_COLORS, LIMB_COLORS, SKELETON, "yolo11_pose_out"); + // detection::draw_keypoints(mat, objects, KPS_COLORS, LIMB_COLORS, SKELETON, "yolo11_pose_out"); } else if (model_type == "obb") { std::vector strides = {8, 16, 32}; std::vector grid_strides; @@ -364,7 +364,7 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c detection::obb::generate_proposals_yolov8_obb_native(grid_strides, feat_ptr, prob_threshold, proposals, input_w, input_h, cls_num); detection::obb::get_out_obb_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); - detection::obb::draw_objects_obb(mat, objects, OBB_CLASS_NAMES, "yolo11_obb_out", 1); + // detection::obb::draw_objects_obb(mat, objects, OBB_CLASS_NAMES, "yolo11_obb_out", 1); } } From a399667aea31c8034b2426888384e6a5e913a3c8 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 3 Dec 2024 19:06:05 +0800 Subject: [PATCH 38/44] [update] add qwen2.5-coder, llama3.2-1B, openbuddy-llama3.2. rename json name.Disable yolo draw. --- projects/llm_framework/main_llm/SConstruct | 2 +- ...-1b-ax630c.json => mode_openbuddy-llama3.2-1B-ax630c.json} | 0 ...tokenizer.py => openbuddy-llama3.2-1B-ax630c_tokenizer.py} | 0 projects/llm_framework/main_yolo/src/main.cpp | 4 ++-- projects/llm_framework/tools/llm_pack.py | 3 +++ 5 files changed, 6 insertions(+), 3 deletions(-) rename projects/llm_framework/main_llm/{mode_openbuddy-llama3.2-1b-ax630c.json => mode_openbuddy-llama3.2-1B-ax630c.json} (100%) rename projects/llm_framework/main_llm/{openbuddy-llama3.2-1b-ax630c_tokenizer.py => openbuddy-llama3.2-1B-ax630c_tokenizer.py} (100%) diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index eca59a3e..d20973e7 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -35,7 +35,7 @@ static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib STATIC_LIB += static_file * 4 STATIC_FILES += [AFile('llama3.2-1B-prefill-ax630c_tokenizer.py'), - AFile('openbuddy-llama3.2-1b-ax630c_tokenizer.py'), + AFile('openbuddy-llama3.2-1B-ax630c_tokenizer.py'), AFile('qwen2.5-coder-0.5B-ax630c_tokenizer.py') ] STATIC_FILES += Glob('mode_*.json') diff --git a/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1b-ax630c.json b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json similarity index 100% rename from projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1b-ax630c.json rename to projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json diff --git a/projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c_tokenizer.py b/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py similarity index 100% rename from projects/llm_framework/main_llm/openbuddy-llama3.2-1b-ax630c_tokenizer.py rename to projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 380d6c42..87fd3ed6 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -175,7 +175,7 @@ class llm_task { if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { throw std::string("img size error"); } - cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); return inference(camera_data, false); } @@ -188,7 +188,7 @@ class llm_task { if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { throw std::string("img size error"); } - cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); return inference(camera_data); } diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 5728b381..8d60d467 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -300,3 +300,6 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_data_deb('llm-yolo11n', data_version, src_folder, revision) create_data_deb('llm-yolo11n-pose', data_version, src_folder, revision) create_data_deb('llm-yolo11n-seg', data_version, src_folder, revision) + create_data_deb('llm-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision) + create_data_deb('llm-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision) + create_data_deb('llm-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision) From 88fdc648a834df03318f4b6b27d6b8265f6334b4 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 4 Dec 2024 17:42:25 +0800 Subject: [PATCH 39/44] [update] add vlm internvl2. rename json name. --- .../llama3.2-1B-prefill-ax630c_tokenizer.py | 4 +- .../mode_openbuddy-llama3.2-1B-ax630c.json | 2 +- .../openbuddy-llama3.2-1B-ax630c_tokenizer.py | 4 +- .../qwen2.5-coder-0.5B-ax630c_tokenizer.py | 4 +- projects/llm_framework/main_vlm/SConstruct | 2 +- ...er.py => internvl2-1B-ax630c_tokenizer.py} | 44 +++++++------------ ...30c.json => mode_internvl2-1B-ax630c.json} | 2 +- 7 files changed, 24 insertions(+), 38 deletions(-) rename projects/llm_framework/main_vlm/{internvl2-1b-ax630c_tokenizer.py => internvl2-1B-ax630c_tokenizer.py} (65%) rename projects/llm_framework/main_vlm/{mode_internvl2-1b-ax630c.json => mode_internvl2-1B-ax630c.json} (96%) diff --git a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py b/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py index 0bf9ec51..712212fc 100644 --- a/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/llama3.2-1B-prefill-ax630c_tokenizer.py @@ -122,8 +122,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json index c435231b..a3229399 100644 --- a/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json +++ b/projects/llm_framework/main_llm/mode_openbuddy-llama3.2-1B-ax630c.json @@ -1,5 +1,5 @@ { - "mode":"openbuddy-llama3.2-1b-ax630c", + "mode":"openbuddy-llama3.2-1B-ax630c", "type":"llm", "capabilities":[ "text_generation", diff --git a/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py b/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py index aae47921..09d7b9b3 100644 --- a/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/openbuddy-llama3.2-1B-ax630c_tokenizer.py @@ -123,8 +123,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py b/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py index 11f98b40..4fded69c 100644 --- a/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py +++ b/projects/llm_framework/main_llm/qwen2.5-coder-0.5B-ax630c_tokenizer.py @@ -122,8 +122,8 @@ def do_POST(self): tokenizer = Tokenizer_Http(args.model_id) - print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) - print(tokenizer.encode("hello world", args.content)) + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1' print('http://%s:%s' % host) diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index 2f399933..b0e5739e 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -49,7 +49,7 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] STATIC_LIB += static_file * 4 -STATIC_FILES += [AFile('internvl2-1b-ax630c_tokenizer.py')] +STATIC_FILES += [AFile('internvl2-1B-ax630c_tokenizer.py')] STATIC_FILES += Glob('mode_*.json') env['COMPONENTS'].append({'target':'llm_vlm', diff --git a/projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py b/projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py similarity index 65% rename from projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py rename to projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py index 19cd9aa7..678cf306 100644 --- a/projects/llm_framework/main_vlm/internvl2-1b-ax630c_tokenizer.py +++ b/projects/llm_framework/main_vlm/internvl2-1B-ax630c_tokenizer.py @@ -6,20 +6,18 @@ class Tokenizer_Http: - def __init__(self): - - path = "internvl2_tokenizer" + def __init__(self, model_id): self.tokenizer = AutoTokenizer.from_pretrained( - path, trust_remote_code=True, use_fast=False + model_id, trust_remote_code=True, use_fast=False ) - def encode(self, content): - prompt = f"<|im_start|>system\n你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。<|im_end|><|im_start|>user\n{content}<|im_end|><|im_start|>assistant\n" + def encode(self, prompt, content): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n{prompt}<|im_end|><|im_start|>assistant\n" input_ids = self.tokenizer.encode(prompt) return input_ids - def encode_vpm(self, content="Please describe the image shortly."): - prompt = f"<|im_start|>system\n你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。<|im_end|><|im_start|>user\n\n{content}<|im_end|><|im_start|>assistant\n" + def encode_vpm(self, prompt, content="Please describe the image shortly."): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n\n{prompt}<|im_end|><|im_start|>assistant\n" input_ids = self.tokenizer.encode(prompt) return input_ids @@ -42,26 +40,6 @@ def bos_token(self): def eos_token(self): return self.tokenizer.eos_token - -tokenizer = Tokenizer_Http() - -print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) -token_ids = tokenizer.encode_vpm() -# [151644, 8948, 198, 56568, 104625, 100633, 104455, 104800, 101101, 32022, 102022, 99602, 100013, 9370, 90286, 21287, 42140, 53772, 35243, 26288, 104949, 3837, 105205, 109641, 67916, 30698, 11, 54851, 46944, 115404, 42192, 99441, 100623, 48692, 100168, 110498, 1773, 151645, 151644, 872, 198, -# 151646, -# 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, 151648, -# 151647, -# 198, 5501, 7512, 279, 2168, 19620, 13, 151645, 151644, 77091, 198] -# 118 -print(token_ids) -print(len(token_ids)) -token_ids = tokenizer.encode("hello world") -# [151644, 8948, 198, 56568, 104625, 100633, 104455, 104800, 101101, 32022, 102022, 99602, 100013, 9370, 90286, 21287, 42140, 53772, 35243, 26288, 104949, 3837, 105205, 109641, 67916, 30698, 11, 54851, 46944, 115404, 42192, 99441, 100623, 48692, 100168, 110498, 1773, 151645, 151644, 872, 198, 14990, 1879, 151645, 151644, 77091, 198] -# 47 -print(token_ids) -print(len(token_ids)) - - class Request(BaseHTTPRequestHandler): # 通过类继承，新定义类 timeout = 5 @@ -117,7 +95,7 @@ def do_POST(self): if b_img_prompt: token_ids = tokenizer.encode_vpm(prompt) else: - token_ids = tokenizer.encode(prompt) + token_ids = tokenizer.encode(prompt, args.content) if token_ids is None: msg = json.dumps({"token_ids": -1}) else: @@ -144,8 +122,16 @@ def do_POST(self): args = argparse.ArgumentParser() args.add_argument("--host", type=str, default="localhost") args.add_argument("--port", type=int, default=8080) + args.add_argument('--model_id', type=str, default='internvl2_tokenizer') + args.add_argument('--content', type=str, default='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。') args = args.parse_args() + tokenizer = Tokenizer_Http(args.model_id) + + + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) + host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1' print("http://%s:%s" % host) server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例 diff --git a/projects/llm_framework/main_vlm/mode_internvl2-1b-ax630c.json b/projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json similarity index 96% rename from projects/llm_framework/main_vlm/mode_internvl2-1b-ax630c.json rename to projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json index a711350f..0818b10c 100644 --- a/projects/llm_framework/main_vlm/mode_internvl2-1b-ax630c.json +++ b/projects/llm_framework/main_vlm/mode_internvl2-1B-ax630c.json @@ -1,5 +1,5 @@ { - "mode":"internvl2-1b-ax630c", + "mode":"internvl2-1B-ax630c", "type":"vlm", "capabilities":[ "text_generation", From aea8f17badb3994d78f3e34c5a166154dd03169a Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 4 Dec 2024 17:43:15 +0800 Subject: [PATCH 40/44] [update] llm_pack.py add vlm internvl2. --- projects/llm_framework/tools/llm_pack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 8d60d467..febaa5ac 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -303,3 +303,4 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): create_data_deb('llm-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision) create_data_deb('llm-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision) create_data_deb('llm-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision) + create_data_deb('llm-internvl2-1B-ax630c', data_version, src_folder, revision) \ No newline at end of file From 23d2adc78fb0835cc348765f972f468b819a5d24 Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 4 Dec 2024 18:31:53 +0800 Subject: [PATCH 41/44] [update] llm-camera --- doc/projects_llm_framework_doc/llm_camera.md | 147 ++++++++++++++++++ .../llm_framework/main_camera/src/main.cpp | 16 +- 2 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 doc/projects_llm_framework_doc/llm_camera.md diff --git a/doc/projects_llm_framework_doc/llm_camera.md b/doc/projects_llm_framework_doc/llm_camera.md new file mode 100644 index 00000000..faf4672f --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_camera.md @@ -0,0 +1,147 @@ +# llm-camera +视频源单元，用于从 USB V4L2 视频设备中获取视频流到内部通道。 + +## setup +配置单元工作。 + +发送 json： +```json +{ + "request_id":"2", + "work_id":"camera", + "action":"setup", + "object":"camera.setup", + "data":{ + "response_format":"camera.raw", + "input":"/dev/video0", + "enoutput":false, + "frame_width":320, + "frame_height":320 + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `camera`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `camera.setup`。 +- response_format：返回结果为 `camera.raw`，是 yuv422 格式。 +- input：读取的设备名。 +- frame_width：输出的视频帧宽。 +- frame_height：输出的视频帧高。 +- enoutput：是否起用用户结果输出。 + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"camera.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "camera.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"camera.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["camera.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"camera.tasklist", + "request_id":"2", + "work_id":"camera" +} +``` + +获取任务运行参数。 + +发送 json： +```json +{ + "request_id": "2", + "work_id": "camera.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "response_format":"camera.raw", + "input":"/dev/video0", + "frame_width":320, + "frame_height":320 + }, + "error":{ + "code":0, + "message":"" + }, + "object":"camera.taskinfo", + "request_id":"2", + "work_id":"camera.1003" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/main.cpp b/projects/llm_framework/main_camera/src/main.cpp index 2884e272..2eb7082d 100644 --- a/projects/llm_framework/main_camera/src/main.cpp +++ b/projects/llm_framework/main_camera/src/main.cpp @@ -37,7 +37,6 @@ class llm_task { public: std::string response_format_; - std::vector inputs_; task_callback_t out_callback_; bool enoutput_; bool enstream_; @@ -95,19 +94,10 @@ class llm_task { try { response_format_ = config_body.at("response_format"); enoutput_ = config_body.at("enoutput"); - devname_ = config_body.at("devname"); + devname_ = config_body.at("input"); frame_width_ = config_body.at("frame_width"); frame_height_ = config_body.at("frame_height"); - if (config_body.contains("input")) { - if (config_body["input"].is_string()) { - inputs_.push_back(config_body["input"].get()); - } else if (config_body["input"].is_array()) { - for (auto _in : config_body["input"]) { - inputs_.push_back(_in.get()); - } - } - } } catch (...) { return true; } @@ -263,7 +253,9 @@ class llm_camera : public StackFlow { auto llm_task_obj = llm_task_[work_id_num]; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["input"] = llm_task_obj->devname_; + req_body["frame_width"] = llm_task_obj->frame_width_; + req_body["frame_height"] = llm_task_obj->frame_height_; send("camera.taskinfo", req_body, LLM_NO_ERROR, work_id); } } From e2f8967595b0af88ebe12f46ed39d3a80d44e99a Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Wed, 4 Dec 2024 18:38:11 +0800 Subject: [PATCH 42/44] [update] yolo doc --- doc/projects_llm_framework_doc/llm_camera.md | 2 +- projects/llm_framework/README.md | 62 ++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/doc/projects_llm_framework_doc/llm_camera.md b/doc/projects_llm_framework_doc/llm_camera.md index faf4672f..891c7cb6 100644 --- a/doc/projects_llm_framework_doc/llm_camera.md +++ b/doc/projects_llm_framework_doc/llm_camera.md @@ -28,7 +28,7 @@ - input：读取的设备名。 - frame_width：输出的视频帧宽。 - frame_height：输出的视频帧高。 -- enoutput：是否起用用户结果输出。 +- enoutput：是否起用用户结果输出。如果不需要获取摄像头图片，请不要开启该参数，视频流会增加信道的通信压力。 响应 json： diff --git a/projects/llm_framework/README.md b/projects/llm_framework/README.md index 3f790874..f59c9045 100644 --- a/projects/llm_framework/README.md +++ b/projects/llm_framework/README.md @@ -167,3 +167,65 @@ waiting return status: ``` **Please enjoy the large model voice assistant.** + + +## cv mode +1、reset + +send : + +```json +{ + "request_id": "11212155", + "work_id": "sys", + "action": "reset" +} +``` +waiting reset over! + +2、init camera unit + +send : + +```json +{ + "request_id":"4", + "work_id":"camera", + "action":"setup", + "object":"camera.setup", + "data":{ + "response_format":"camera.raw", + "input":"/dev/video0", + "enoutput":false, + "frame_width":320, + "frame_height":320 + } +} +``` +waiting return work_id: +```json +{"created":1731488371,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"camera.1000"} +``` + +2、init yolo unit + +send : + +```json +{ + "request_id":"5", + "work_id":"yolo", + "action":"setup", + "object":"yolo.setup", + "data":{ + "model":"yolo11n_anquanmao", + "response_format":"yolo.yolobox", + "input":"camera.1000", + "enoutput":true + } +} +``` +waiting return work_id: +```json +{"created":1731488371,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"5","work_id":"yolo.1001"} +``` From 422ed2f4f8771d0cc142103cb1c3b474d2705bec Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 11 Dec 2024 14:36:42 +0800 Subject: [PATCH 43/44] [update] format code. sys add rmmode method. --- projects/llm_framework/main_asr/src/main.cpp | 3 +- .../llm_framework/main_audio/src/main.cpp | 3 +- projects/llm_framework/main_kws/src/main.cpp | 5 +-- projects/llm_framework/main_llm/src/main.cpp | 5 +-- .../llm_framework/main_melotts/src/main.cpp | 2 +- .../llm_framework/main_sys/src/event_loop.cpp | 31 ++++++++++++------- .../llm_framework/main_sys/src/serial_com.cpp | 5 ++- .../llm_framework/main_sys/src/tcp_com.cpp | 14 ++++----- .../llm_framework/main_tokenizer/src/main.cpp | 4 +-- projects/llm_framework/main_vlm/src/main.cpp | 5 +-- projects/llm_framework/main_yolo/src/main.cpp | 25 ++++++++------- 11 files changed, 56 insertions(+), 46 deletions(-) diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 319bcf7f..0648433b 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -92,7 +92,8 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index 36e6c2cc..e95c1334 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -141,7 +141,8 @@ class llm_audio : public StackFlow { nlohmann::json error_body; std::string base_model_path; std::string base_model_config_path; - std::list config_file_paths = get_config_file_paths(base_model_path, base_model_config_path, "audio"); + std::list config_file_paths = + get_config_file_paths(base_model_path, base_model_config_path, "audio"); try { config_body = nlohmann::json::parse(data); for (auto file_name : config_file_paths) { diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index c4c8f207..930a8bfc 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -98,7 +98,8 @@ class llm_task { } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -486,7 +487,7 @@ class llm_kws : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("kws.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index e4f478d8..b89e94ba 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -85,7 +85,8 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -489,7 +490,7 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 66405a8e..c5573632 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -428,7 +428,7 @@ class llm_tts : public StackFlow { } next_data = &tmp_msg2; } - std::string user_msg = sample_unescapeString(*next_data); + std::string user_msg = sample_unescapeString(*next_data); std::vector tmp_data = llm_task_obj->lexicon_->splitEachChar(user_msg); for (auto cutf8 : tmp_data) { if (is_breakpoint(cutf8)) { diff --git a/projects/llm_framework/main_sys/src/event_loop.cpp b/projects/llm_framework/main_sys/src/event_loop.cpp index e7f129d1..0ac9c037 100644 --- a/projects/llm_framework/main_sys/src/event_loop.cpp +++ b/projects/llm_framework/main_sys/src/event_loop.cpp @@ -160,28 +160,25 @@ int _sys_hwinfo(int com_id, const nlohmann::json &json_obj) std::vector ifcs; ifconfig(ifcs); std::vector jifcs; - for (auto& item : ifcs) - { + for (auto &item : ifcs) { nlohmann::json eth_info; - eth_info["name"] = item.name; - eth_info["ip"] = item.ip; + eth_info["name"] = item.name; + eth_info["ip"] = item.ip; char eth_ip_buff[128] = {0}; sprintf(eth_ip_buff, "/sys/class/net/%s/speed", item.name); FILE *file = fopen(eth_ip_buff, "r"); memset(eth_ip_buff, 0, sizeof(eth_ip_buff)); - if(file != NULL) - { + if (file != NULL) { int size = fread(eth_ip_buff, 1, sizeof(eth_ip_buff), file); - if(size > 0) - eth_ip_buff[size-1] = '\0'; + if (size > 0) eth_ip_buff[size - 1] = '\0'; fclose(file); } eth_info["speed"] = eth_ip_buff; jifcs.push_back(eth_info); } data_body["eth_info"] = jifcs; - out_body["data"] = data_body; - std::string out = out_body.dump(); + out_body["data"] = data_body; + std::string out = out_body.dump(); zmq_com_send(com_id, out); return 0; } @@ -254,6 +251,15 @@ int sys_lsmode(int com_id, const nlohmann::json &json_obj) return out; } +int sys_rmmode(int com_id, const nlohmann::json &json_obj) +{ + std::string rmmode_name = json_obj["data"]; + const std::string command = "dpkg -P llm-" + rmmode_name; + int out = system(command.c_str()); + usr_print_error(json_obj["request_id"], json_obj["work_id"], "{\"code\":0, \"message\":\"\"}", com_id); + return out; +} + int sys_lstask(int com_id, const nlohmann::json &json_obj) { int out; @@ -349,6 +355,7 @@ int sys_update(int com_id, const nlohmann::json &json_obj) "{\"code\":-10, \"message\":\"Not available at the moment.\"}", com_id); return out; } + int sys_upgrade(int com_id, const nlohmann::json &json_obj) { int out; @@ -521,6 +528,7 @@ void server_work() key_sql["sys.reset"] = sys_reset; key_sql["sys.reboot"] = sys_reboot; key_sql["sys.version"] = sys_version; + key_sql["sys.rmmode"] = sys_rmmode; } void server_stop_work() @@ -554,8 +562,7 @@ void unit_action_match(int com_id, const std::string &json_str) usr_print_error("0", "sys", "{\"code\":-2, \"message\":\"json format error\"}", com_id); return; } - if(work_id.empty()) - work_id = "sys"; + if (work_id.empty()) work_id = "sys"; std::string action; error = doc["action"].get_string(action); if (error) { diff --git a/projects/llm_framework/main_sys/src/serial_com.cpp b/projects/llm_framework/main_sys/src/serial_com.cpp index d9952357..6dc46086 100644 --- a/projects/llm_framework/main_sys/src/serial_com.cpp +++ b/projects/llm_framework/main_sys/src/serial_com.cpp @@ -41,8 +41,7 @@ class serial_com : public zmq_bus_com { void send_data(const std::string &data) { - if(exit_flage) - linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); + if (exit_flage) linux_uart_write(uart_fd, data.length(), (void *)data.c_str()); } void reace_data_event() @@ -55,7 +54,7 @@ class serial_com : public zmq_bus_com { struct timeval timeout = {0, 500000}; if ((select(uart_fd + 1, &readfds, NULL, NULL, &timeout) <= 0) || (!FD_ISSET(uart_fd, &readfds))) continue; ssize_t len = linux_uart_read(uart_fd, buff.size(), buff.data()); - if(len <= 0) continue; + if (len <= 0) continue; { try { select_json_str(std::string(buff.data(), len), diff --git a/projects/llm_framework/main_sys/src/tcp_com.cpp b/projects/llm_framework/main_sys/src/tcp_com.cpp index b908f44b..552a4a78 100644 --- a/projects/llm_framework/main_sys/src/tcp_com.cpp +++ b/projects/llm_framework/main_sys/src/tcp_com.cpp @@ -42,7 +42,7 @@ class tcp_com : public zmq_bus_com { void send_data(const std::string& data) { // tcp_server_mutex.lock(); - // if (exit_flage) + // if (exit_flage) // { // auto tcp_channel = channel.lock(); // if(tcp_channel) @@ -52,8 +52,7 @@ class tcp_com : public zmq_bus_com { // } // tcp_server_mutex.unlock(); auto tcp_channel = channel.lock(); - if(tcp_channel) - { + if (tcp_channel) { tcp_channel->write(data); } } @@ -62,7 +61,7 @@ class tcp_com : public zmq_bus_com { void onConnection(const SocketChannelPtr& channel) { if (channel->isConnected()) { - auto p_com = channel->newContextPtr(); + auto p_com = channel->newContextPtr(); p_com->channel = channel; p_com->work(zmq_s_format, counter_port.fetch_add(1)); if (counter_port.load() > 65535) counter_port.store(8000); @@ -77,13 +76,12 @@ void onConnection(const SocketChannelPtr& channel) void onMessage(const SocketChannelPtr& channel, Buffer* buf) { - int len = (int)buf->size(); - char* data = (char*)buf->data(); + int len = (int)buf->size(); + char* data = (char*)buf->data(); auto p_com = channel->getContextPtr(); p_com->tcp_server_mutex.lock(); try { - p_com->select_json_str(std::string(data, len), - std::bind(&tcp_com::on_data, p_com, std::placeholders::_1)); + p_com->select_json_str(std::string(data, len), std::bind(&tcp_com::on_data, p_com, std::placeholders::_1)); } catch (...) { std::string out_str; out_str += "{\"request_id\": \"0\",\"work_id\": \"sys\",\"created\": "; diff --git a/projects/llm_framework/main_tokenizer/src/main.cpp b/projects/llm_framework/main_tokenizer/src/main.cpp index 6a16adba..6b3579c9 100644 --- a/projects/llm_framework/main_tokenizer/src/main.cpp +++ b/projects/llm_framework/main_tokenizer/src/main.cpp @@ -9,13 +9,13 @@ int main() { if (access("./_tokenizer.py", F_OK) == 0) { - char *args[] = {(char*)"python3", (char*)"./_tokenizer.py", NULL}; + char *args[] = {(char *)"python3", (char *)"./_tokenizer.py", NULL}; if (execvp("python3", args) == -1) { perror("execvp"); return 1; } } else if (access("/opt/m5stack/share/_tokenizer.py", F_OK) == 0) { - char *args[] = {(char*)"python3", (char*)"/opt/m5stack/share/_tokenizer.py", NULL}; + char *args[] = {(char *)"python3", (char *)"/opt/m5stack/share/_tokenizer.py", NULL}; if (execvp("python3", args) == -1) { perror("execvp"); return 1; diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index ca014d83..2c422f63 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -89,7 +89,8 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -502,7 +503,7 @@ class llm_llm : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("vlm.taskinfo", req_body, LLM_NO_ERROR, work_id); } } diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 87fd3ed6..9583811c 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -89,7 +89,8 @@ class llm_task { return -1; } nlohmann::json file_body; - std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -160,7 +161,7 @@ class llm_task { if (msg.size() != mode_config_.img_w * mode_config_.img_h * 2) { throw std::string("img size error"); } - cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); + cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); cv::Mat rgb; cv::cvtColor(camera_data, rgb, cv::COLOR_YUV2RGB_YUYV); return inference(rgb, false); @@ -428,8 +429,8 @@ class llm_yolo : public StackFlow { std::string input_url_name = input + ".out_port"; std::string input_url = unit_call("sys", "sql_select", input_url_name); if (!input_url.empty()) { - std::weak_ptr _llm_task_obj = llm_task_obj; - std::weak_ptr _llm_channel = llm_channel; + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; llm_channel->subscriber( input_url, [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { this->task_camera_data(_llm_task_obj, _llm_channel, raw); @@ -471,16 +472,16 @@ class llm_yolo : public StackFlow { std::bind(&llm_yolo::task_user_data, this, std::weak_ptr(llm_task_obj), std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); - } else if (data.find("camera") != std::string::npos){ + } else if (data.find("camera") != std::string::npos) { std::string input_url_name = data + ".out_port"; std::string input_url = unit_call("sys", "sql_select", input_url_name); if (!input_url.empty()) { - std::weak_ptr _llm_task_obj = llm_task_obj; - std::weak_ptr _llm_channel = llm_channel; - llm_channel->subscriber( - input_url, [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { - this->task_camera_data(_llm_task_obj, _llm_channel, raw); - }); + std::weak_ptr _llm_task_obj = llm_task_obj; + std::weak_ptr _llm_channel = llm_channel; + llm_channel->subscriber(input_url, + [this, _llm_task_obj, _llm_channel](pzmq *_pzmq, const std::string &raw) { + this->task_camera_data(_llm_task_obj, _llm_channel, raw); + }); } llm_task_obj->inputs_.push_back(data); } @@ -541,7 +542,7 @@ class llm_yolo : public StackFlow { req_body["model"] = llm_task_obj->model_; req_body["response_format"] = llm_task_obj->response_format_; req_body["enoutput"] = llm_task_obj->enoutput_; - req_body["inputs"] = llm_task_obj->inputs_; + req_body["inputs"] = llm_task_obj->inputs_; send("yolo.taskinfo", req_body, LLM_NO_ERROR, work_id); } } From 13c41021d68fa0bbade7a23936e02137a3a3efea Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Thu, 12 Dec 2024 11:11:38 +0800 Subject: [PATCH 44/44] [update] ax_msp --- .gitignore | 3 ++- ext_components/ax_msp/SConstruct | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 9ec37dd5..cb65ec91 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vscode/settings.json -projects/core135_llm_product_test_ui \ No newline at end of file +projects/core135_llm_product_test_ui +projects/imx678_test \ No newline at end of file diff --git a/ext_components/ax_msp/SConstruct b/ext_components/ax_msp/SConstruct index 82b6590f..7e73e778 100644 --- a/ext_components/ax_msp/SConstruct +++ b/ext_components/ax_msp/SConstruct @@ -17,14 +17,18 @@ if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: LDFLAGS=[] LINK_SEARCH_PATH=[] - INCLUDE += [os.path.join(MSP_PATH, 'out/arm64_glibc/include'), os.path.join(MSP_PATH, 'sample/common')] + env["MSP_PATH"] = MSP_PATH + INCLUDE += [os.path.join(MSP_PATH, 'out/arm64_glibc/include'), os.path.join(MSP_PATH, 'sample'), os.path.join(MSP_PATH, 'sample/common'), os.path.join(MSP_PATH, 'sample/rtsp')] LINK_SEARCH_PATH += [os.path.join(MSP_PATH, 'out/arm64_glibc/lib')] REQUIREMENTS += ['dl', 'm', 'pthread'] - third_party = ['cmdline','drm','faac','fdk-aac','ffmpeg','libsamplerate','live','mp4','opencv','openssl','optee','opus','swupdate','tinyalsa'] + third_party = ['drm','faac','fdk-aac','ffmpeg','libsamplerate','mp4','opencv','opus','tinyalsa'] for dirn in third_party: INCLUDE.append(os.path.join(MSP_PATH,'third-party',dirn,'include')) LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party',dirn,'lib/arm64/glibc')) - + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/arm64/glibc/include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/live/out/arm64/glibc/lib')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/openssl/arm64/include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/openssl/arm64/glibc/lib')) LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'app/lib')) env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']),