diff --git a/examples/mediatek/CMakeLists.txt b/examples/mediatek/CMakeLists.txt index 1018477ec85..966fecb0664 100644 --- a/examples/mediatek/CMakeLists.txt +++ b/examples/mediatek/CMakeLists.txt @@ -105,11 +105,10 @@ if(${ANDROID}) _mtk_llama_executor_runner__srcs ${CMAKE_CURRENT_LIST_DIR}/executor_runner/mtk_llama_executor_runner.cpp ) - # Build ABSL and RE2 - set(LLAMA2_EXAMPLE_MODEL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../examples/models/llama2) - set(THIRD_PARTY_ABSL_DIR ${LLAMA2_EXAMPLE_MODEL_DIR}/third-party/abseil-cpp) - set(THIRD_PARTY_RE2_DIR ${LLAMA2_EXAMPLE_MODEL_DIR}/third-party/re2) + set(EXTENSIONS_LLM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm) + set(THIRD_PARTY_ABSL_DIR ${EXTENSIONS_LLM_DIR}/third-party/abseil-cpp) + set(THIRD_PARTY_RE2_DIR ${EXTENSIONS_LLM_DIR}/third-party/re2) set(ABSL_ENABLE_INSTALL ON) set(ABSL_PROPAGATE_CXX_STD ON) set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE}) @@ -119,7 +118,7 @@ if(${ANDROID}) set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag}) # Build tokenizers - set(LLAMA2_TOKENIZER_DIR ${LLAMA2_EXAMPLE_MODEL_DIR}/tokenizer) + set(LLAMA2_TOKENIZER_DIR ${EXTENSIONS_LLM_DIR}/tokenizer) add_library(tokenizer STATIC) target_include_directories(tokenizer PUBLIC @@ -135,6 +134,7 @@ if(${ANDROID}) PRIVATE ${LLAMA2_TOKENIZER_DIR}/tiktoken.cpp ${LLAMA2_TOKENIZER_DIR}/bpe_tokenizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../models/llama2/tokenizer/llama_tiktoken.cpp ) # Include directory for neuron headers diff --git a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp index 59b7a39e1cc..b605dd13bec 100644 --- a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp +++ b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp @@ -68,8 +68,9 @@ #include "llama_runner/Utils.h" #include "llama_runner/llm_helper/include/llm_types.h" -#include -#include +#include +#include +#include // Llama model options DEFINE_uint64( @@ -316,7 +317,7 @@ std::unique_ptr load_tokenizer() { if (FLAGS_tokenizer_type == "bpe") { tokenizer = std::make_unique(); } else if (FLAGS_tokenizer_type == "tiktoken") { - tokenizer = std::make_unique(); + tokenizer = torch::executor::get_tiktoken_for_llama(); } ET_CHECK_MSG( tokenizer, "Invalid tokenizer type: %s", FLAGS_tokenizer_type.c_str());