diff --git a/examples/main/main.cpp b/examples/main/main.cpp index acf79a89268fb..06cc233924360 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -41,6 +41,7 @@ static std::vector * g_output_tokens; static bool is_interacting = false; static bool need_insert_eot = false; + static void print_usage(int argc, char ** argv) { (void) argc; diff --git a/ggml/src/ggml-tsavorite/CMakeLists.txt b/ggml/src/ggml-tsavorite/CMakeLists.txt index 8aecc0d6778c1..1e156df10eb16 100644 --- a/ggml/src/ggml-tsavorite/CMakeLists.txt +++ b/ggml/src/ggml-tsavorite/CMakeLists.txt @@ -2,7 +2,7 @@ message(STATUS "Tsavorite framework is found") # # tsavorite Kernel Library file(GLOB TLIBS "${RUNTIME_DIR}/lib/*.so" "../../${GGML_TSI_KERNEL_DIR}/host/*.o") - +add_compile_options(--std=c++20) ggml_add_backend_library(ggml-tsavorite ggml-tsavorite.cpp ) diff --git a/ggml/src/ggml-tsavorite/ggml-tsavorite.cpp b/ggml/src/ggml-tsavorite/ggml-tsavorite.cpp index a73e8b244998a..f8bec1b61b9bb 100644 --- a/ggml/src/ggml-tsavorite/ggml-tsavorite.cpp +++ b/ggml/src/ggml-tsavorite/ggml-tsavorite.cpp @@ -20,10 +20,16 @@ #include #include #include - +#include #include "ggml-backend-impl.h" #include "ggml-impl.h" #include "ggml.h" +#include "HostShimCAPI.h" +#include "tsi-rt/utils/Profiler.h" + +using namespace std; +namespace tsirt = ::tsi::runtime; + typedef struct _txe_device_t *txe_device_s; typedef struct _txe_compute_pipeline_state_t *txe_compute_pipeline_state_s; @@ -513,9 +519,12 @@ static struct ggml_backend_tsavorite_context *ggml_tsavorite_init(ggml_backend_d if (tsi_log_setup() == false) return NULL; - // TSI Run time Initalization - tsi_initialize(NUM_OF_TXES); + std::string mainProfilerName = "LLAMA SP Main "; + tsirt::utils::TSIProfiler::initialize(); + tsirt::utils::TSIScopedProfiler mainProfiler(mainProfilerName); + // TSI Run time Initalization + tsi_initialize(NUM_OF_TXES, NULL); // init context struct ggml_backend_tsavorite_context *ctx = (struct ggml_backend_tsavorite_context *)calloc( 1, sizeof(struct ggml_backend_tsavorite_context)); @@ -614,6 +623,12 @@ static void ggml_tsavorite_free(struct ggml_backend_tsavorite_context *ctx) { GGML_TSAVORITE_LOG_INFO("Delaying tsi_finalize for 2 sec"); sleep(2); tsi_finalize(); + tsirt::utils::TSIProfiler::finalize(); + std::cout << "\nLLAMA SP Profiling Results:" << std::endl; + std::cout << tsirt::utils::TSIProfiler::getFormattedResults( + /*truncateFuncNames*/ true) + << std::endl; + //tsi_finalize(); GGML_TSAVORITE_LOG_INFO("End %s\n", __func__); }